1use std::collections::{BTreeMap, BTreeSet};
4use std::sync::atomic::{AtomicBool, Ordering};
5use std::sync::{Arc, Weak};
6use std::time::{Duration, Instant};
7
8use bytes::Bytes;
9use object_store::path::Path as OsPath;
10use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
11use parking_lot::Mutex;
12use serde::{Deserialize, Serialize};
13use sha2::{Digest as _, Sha256};
14use tokio::sync::watch;
15use tokio_stream::StreamExt;
16use uuid::Uuid;
17
18use crate::checkpoint::{
19 AssignmentDrainId, AssignmentDrainTransition, CheckpointAssignmentFence,
20 ClusterRecoveryCapsule, LeaderProof, LeaderProofOwner, RecoveryCapsuleRef,
21 MAX_CHECKPOINT_PARTICIPANTS,
22};
23use crate::checkpoint_decision::{
24 CheckpointDecisionStore, CheckpointOutcome, CheckpointScope, CheckpointVerdict, DecisionError,
25 RecordOutcomeResult,
26};
27use crate::cluster::discovery::NodeId;
28
29use super::catalog_manifest::{
30 CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
31};
32use super::controller::{
33 RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
34 RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
35 MAX_RECOVERY_ANNOUNCEMENT_BYTES,
36};
37use super::lease_deadline::LeaseDeadline;
38use super::process_lease::{ProcessLease, ProcessLeaseFence};
39use super::snapshot::{
40 AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
41};
42
43const LEASE_PREFIX: &str = "control/leader-lease/";
44const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
45const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
46const STORE_CONTRACT_PROBE_V1: &[u8] = b"laminardb-control-store-contract-v1";
47const STORE_CONTRACT_PROBE_V2: &[u8] = b"laminardb-control-store-contract-v2";
48const STORE_CONTRACT_PROBE_STALE: &[u8] = b"laminardb-control-store-contract-stale";
49const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
50const AUTHORITY_RECORD_VERSION: u32 = 9;
51const AUTHORITY_HEAD_VERSION: u32 = 1;
52const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
53const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
54const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
55 + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
56 + 2;
57const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
58const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
59const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
60const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
61const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
62const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
63const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
64const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
65const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
66const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
67const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
68const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
69#[cfg(test)]
70const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
71
72fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
73 match error {
74 SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
75 error => LeaseError::Invalid(format!("{context}: {error}")),
76 }
77}
78
79fn lease_path(sequence: u64) -> OsPath {
80 OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
81}
82
83fn authority_head_path() -> OsPath {
84 OsPath::from(AUTHORITY_HEAD_PATH)
85}
86
87fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
88 OsPath::from(format!(
89 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
90 reference.release.generation(),
91 reference.sha256
92 ))
93}
94
95fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
96 let raw = path
97 .as_ref()
98 .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
99 .and_then(|value| value.strip_prefix("generation="))
100 .ok_or_else(|| {
101 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
102 })?;
103 let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
104 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
105 })?;
106 let digest = digest.strip_suffix(".json").ok_or_else(|| {
107 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
108 })?;
109 if generation.len() != 20
110 || !generation.bytes().all(|byte| byte.is_ascii_digit())
111 || digest.len() != 64
112 || !digest
113 .bytes()
114 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
115 {
116 return Err(LeaseError::Invalid(format!(
117 "invalid recovery release terminal path {path}"
118 )));
119 }
120 let generation = generation.parse::<u64>().map_err(|error| {
121 LeaseError::Invalid(format!(
122 "invalid recovery release terminal generation in {path}: {error}"
123 ))
124 })?;
125 if generation == 0 {
126 return Err(LeaseError::Invalid(format!(
127 "invalid recovery release terminal path {path}"
128 )));
129 }
130 Ok((generation, digest.to_owned()))
131}
132
133fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
134 let raw = path
135 .as_ref()
136 .strip_prefix(LEASE_PREFIX)
137 .and_then(|file| file.strip_prefix('v'))
138 .and_then(|file| file.strip_suffix(".json"))
139 .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
140 if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
141 return Err(LeaseError::Invalid(format!(
142 "invalid leader authority sequence in {path}"
143 )));
144 }
145 let sequence = raw.parse::<u64>().map_err(|error| {
146 LeaseError::Invalid(format!(
147 "invalid leader authority sequence in {path}: {error}"
148 ))
149 })?;
150 if sequence == 0 || lease_path(sequence) != *path {
151 return Err(LeaseError::Invalid(format!(
152 "noncanonical leader authority path {path}"
153 )));
154 }
155 Ok(sequence)
156}
157
158fn consume_live_authority_link(traversed: &mut usize) -> bool {
159 if *traversed == MAX_LIVE_AUTHORITY_LINKS {
160 return false;
161 }
162 *traversed += 1;
163 true
164}
165
166fn now_millis() -> i64 {
167 std::time::SystemTime::now()
168 .duration_since(std::time::UNIX_EPOCH)
169 .ok()
170 .and_then(|duration| i64::try_from(duration.as_millis()).ok())
171 .unwrap_or(i64::MAX)
172}
173
174async fn read_authority_record(
175 store: &dyn ObjectStore,
176 sequence: u64,
177) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
178 let result = match store.get(&lease_path(sequence)).await {
179 Ok(result) => result,
180 Err(object_store::Error::NotFound { .. }) => return Ok(None),
181 Err(error) => return Err(LeaseError::Io(error.to_string())),
182 };
183 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
184 return Err(LeaseError::Invalid(format!(
185 "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
186 result.meta.size
187 )));
188 }
189 let bytes = result
190 .bytes()
191 .await
192 .map_err(|error| LeaseError::Io(error.to_string()))?;
193 let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
194 record.validate()?;
195 if record.lease.seq != sequence {
196 return Err(LeaseError::Invalid(
197 "authority record sequence does not match its object name".into(),
198 ));
199 }
200 let canonical = serde_json::to_vec(&record)?;
201 if canonical.as_slice() != bytes.as_ref() {
202 return Err(LeaseError::Invalid(format!(
203 "leader authority record {sequence} does not use its canonical body"
204 )));
205 }
206 Ok(Some(record))
207}
208
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
210#[serde(deny_unknown_fields)]
211struct AuthorityHeadPointer {
212 version: u32,
213 sequence: u64,
214 nonce: Uuid,
215}
216
217impl AuthorityHeadPointer {
218 fn new(sequence: u64) -> Result<Self, LeaseError> {
219 let pointer = Self {
220 version: AUTHORITY_HEAD_VERSION,
221 sequence,
222 nonce: Uuid::new_v4(),
223 };
224 pointer.validate()?;
225 Ok(pointer)
226 }
227
228 fn validate(self) -> Result<(), LeaseError> {
229 if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
230 return Err(LeaseError::Invalid(
231 "leader authority head has an unsupported version, zero sequence, or nil nonce"
232 .into(),
233 ));
234 }
235 Ok(())
236 }
237}
238
239#[derive(Clone)]
240struct VersionedAuthorityHeadPointer {
241 pointer: AuthorityHeadPointer,
242 update_version: UpdateVersion,
243}
244
245struct PublishedAuthorityHead {
246 record: LeaderAuthorityRecord,
247 pointer: VersionedAuthorityHeadPointer,
248}
249
250async fn read_authority_head_pointer(
251 store: &dyn ObjectStore,
252) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
253 let result = match store.get(&authority_head_path()).await {
254 Ok(result) => result,
255 Err(object_store::Error::NotFound { .. }) => return Ok(None),
256 Err(error) => return Err(LeaseError::Io(error.to_string())),
257 };
258 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
259 return Err(LeaseError::Invalid(format!(
260 "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
261 result.meta.size
262 )));
263 }
264 let update_version = UpdateVersion {
265 e_tag: result.meta.e_tag.clone(),
266 version: result.meta.version.clone(),
267 };
268 let bytes = result
269 .bytes()
270 .await
271 .map_err(|error| LeaseError::Io(error.to_string()))?;
272 let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
273 pointer.validate()?;
274 let canonical = serde_json::to_vec(&pointer)?;
275 if canonical.as_slice() != bytes.as_ref() {
276 return Err(LeaseError::Invalid(
277 "leader authority head does not use its canonical body".into(),
278 ));
279 }
280 Ok(Some(VersionedAuthorityHeadPointer {
281 pointer,
282 update_version,
283 }))
284}
285
286fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
287 let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
288 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
289 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
290 return Err(LeaseError::Invalid(format!(
291 "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
292 encoded.len()
293 )));
294 }
295 Ok(Bytes::from(encoded))
296}
297
298fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
299 record.validate()?;
300 let encoded = serde_json::to_vec(record)?;
301 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
302 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
303 return Err(LeaseError::Invalid(format!(
304 "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
305 encoded.len()
306 )));
307 }
308 if record.recovery_fault_slots.iter().any(|slot| slot.active) {
309 ensure_recovery_fault_authority_headroom(encoded_len)?;
310 }
311 Ok(Bytes::from(encoded))
312}
313
314fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
315 let limit = MAX_AUTHORITY_RECORD_BYTES
316 .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
317 .expect("recovery fault headroom must fit the authority record");
318 if encoded_len > limit {
319 return Err(LeaseError::Invalid(format!(
320 "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
321 )));
322 }
323 Ok(())
324}
325
326#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
328pub struct LeaderLeaseOwner {
329 pub node: NodeId,
331 pub boot: Uuid,
333 pub process_term: u64,
335}
336
337impl LeaderLeaseOwner {
338 fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
339 process
340 .validate(process.node)
341 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
342 Ok(Self {
343 node: process.node,
344 boot: process.owner,
345 process_term: process.term,
346 })
347 }
348
349 fn validate(&self) -> Result<(), LeaseError> {
350 if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
351 return Err(LeaseError::Invalid(
352 "leader owner node, boot identity, and process term must be nonzero".into(),
353 ));
354 }
355 Ok(())
356 }
357
358 fn proof_owner(&self) -> LeaderProofOwner {
359 LeaderProofOwner {
360 node_id: self.node.0,
361 boot_id: self.boot,
362 process_term: self.process_term,
363 }
364 }
365}
366
367#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
369pub struct LeaderLease {
370 pub seq: u64,
372 pub renewal_sequence: u64,
374 pub token: u64,
376 pub owner: LeaderLeaseOwner,
378 pub expires_at_ms: i64,
380 pub catalog_manifest: Option<CatalogManifestRef>,
382}
383
384#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
385#[serde(deny_unknown_fields)]
386struct OutcomeLink {
387 sequence: u64,
388 epoch: u64,
389 checkpoint_id: u64,
390}
391
392impl OutcomeLink {
393 fn validate(self) -> Result<(), LeaseError> {
394 let attempt = crate::state::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
395 if self.sequence == 0 || !attempt.is_canonical() {
396 return Err(LeaseError::Invalid(
397 "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
398 .into(),
399 ));
400 }
401 Ok(())
402 }
403}
404
405#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
406#[serde(deny_unknown_fields)]
407struct AssignmentDecisionLink {
408 sequence: u64,
409 target_version: u64,
410}
411
412#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
413#[serde(deny_unknown_fields)]
414struct AuthorityRecoveryFaultSlot {
415 publisher: RecoveryFaultPublisher,
416 request_sequence: u64,
417 fault_sequence: u64,
418 active: bool,
419}
420
421impl AuthorityRecoveryFaultSlot {
422 fn validate(&self) -> Result<(), LeaseError> {
423 self.publisher.validate().map_err(LeaseError::Invalid)?;
424 if self.request_sequence == 0 || self.fault_sequence == 0 {
425 return Err(LeaseError::Invalid(
426 "recovery fault request and authority sequence must be nonzero".into(),
427 ));
428 }
429 Ok(())
430 }
431
432 fn fault(&self) -> RecoveryFault {
433 RecoveryFault {
434 reporter: NodeId(self.publisher.participant.node_id),
435 sequence: self.fault_sequence,
436 }
437 }
438
439 fn matches_request(&self, publisher: RecoveryFaultPublisher, request_sequence: u64) -> bool {
440 self.publisher == publisher && self.request_sequence == request_sequence
441 }
442}
443
444#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
445#[serde(deny_unknown_fields)]
446pub(crate) struct RecoveryReleaseTerminalRef {
447 release: RecoveryReleaseId,
448 sha256: String,
449 encoded_len: u64,
450}
451
452impl RecoveryReleaseTerminalRef {
453 fn validate(&self) -> Result<(), LeaseError> {
454 if !self.release.is_canonical()
455 || self.sha256.len() != 64
456 || !self
457 .sha256
458 .bytes()
459 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
460 || self.encoded_len == 0
461 || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
462 {
463 return Err(LeaseError::Invalid(
464 "recovery release terminal reference is not canonical".into(),
465 ));
466 }
467 Ok(())
468 }
469
470 pub(crate) fn generation(&self) -> u64 {
471 self.release.generation()
472 }
473}
474
475#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
476#[serde(deny_unknown_fields)]
477struct AuthorityRecoveryReleaseCommit {
478 terminal: RecoveryReleaseTerminalRef,
479 leader_proof: LeaderProof,
480}
481
482impl AuthorityRecoveryReleaseCommit {
483 fn validate(&self) -> Result<(), LeaseError> {
484 self.terminal.validate()?;
485 if !self.leader_proof.is_canonical() {
486 return Err(LeaseError::Invalid(
487 "recovery release commit has no canonical leader proof".into(),
488 ));
489 }
490 Ok(())
491 }
492}
493
494#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
495#[serde(deny_unknown_fields)]
496pub(crate) struct RecoveryReleaseLink {
497 sequence: u64,
498 terminal: RecoveryReleaseTerminalRef,
499}
500
501impl RecoveryReleaseLink {
502 fn validate(&self) -> Result<(), LeaseError> {
503 if self.sequence == 0 {
504 return Err(LeaseError::Invalid(
505 "recovery release authority link has a zero sequence".into(),
506 ));
507 }
508 self.terminal.validate()
509 }
510}
511
512#[derive(Debug, Clone, PartialEq, Eq)]
513pub(crate) enum RecordRecoveryReleaseCommitResult {
514 Created(RecoveryReleaseTerminalRef),
515 Unchanged(RecoveryReleaseTerminalRef),
516 Conflict { winner: RecoveryReleaseTerminalRef },
517 FaultsChanged,
518}
519
520#[derive(Debug, Clone, Copy, PartialEq, Eq)]
521pub(crate) enum RecordRecoveryFaultResult {
522 Active,
523 AlreadyCleared,
524 CoveredByNewerRequest,
525 Superseded,
526}
527
528fn recovery_release_id_for_terminal(
529 terminal: &RecoveryAnnouncement,
530) -> Result<RecoveryReleaseId, LeaseError> {
531 terminal.validate().map_err(LeaseError::Invalid)?;
532 let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
533 return Err(LeaseError::Invalid(
534 "recovery release terminal must carry ReleaseCommitted".into(),
535 ));
536 };
537 RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
538 round: terminal.round.clone(),
539 phase: RecoverPhase::Release { epoch },
540 })
541 .map_err(LeaseError::Invalid)
542}
543
544fn encode_recovery_release_terminal(
545 terminal: &RecoveryAnnouncement,
546) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
547 let release = recovery_release_id_for_terminal(terminal)?;
548 let encoded = serde_json::to_vec(terminal)?;
549 let encoded_len = u64::try_from(encoded.len())
550 .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
551 if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
552 return Err(LeaseError::Invalid(format!(
553 "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
554 )));
555 }
556 let reference = RecoveryReleaseTerminalRef {
557 release,
558 sha256: format!("{:x}", Sha256::digest(&encoded)),
559 encoded_len,
560 };
561 reference.validate()?;
562 Ok((Bytes::from(encoded), reference))
563}
564
565#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
567#[serde(deny_unknown_fields)]
568pub struct AssignmentDrainDecision {
569 pub transition: AssignmentDrainTransition,
571 pub leader_proof: LeaderProof,
573 pub verdict: AssignmentDrainVerdict,
575}
576
577impl AssignmentDrainDecision {
578 pub fn new(
583 transition: &AssignmentDrainTransition,
584 leader_proof: LeaderProof,
585 verdict: AssignmentDrainVerdict,
586 ) -> Result<Self, String> {
587 if !transition.is_canonical()
588 || !leader_proof.is_canonical()
589 || (verdict == AssignmentDrainVerdict::Commit && leader_proof != transition.leader)
590 {
591 return Err("assignment drain decision requires a canonical transition".into());
592 }
593 Ok(Self {
594 transition: transition.clone(),
595 leader_proof,
596 verdict,
597 })
598 }
599
600 #[must_use]
602 pub fn round(&self) -> AssignmentDrainId {
603 self.transition.id()
604 }
605
606 #[must_use]
608 pub fn target_version(&self) -> u64 {
609 self.transition.target.assignment_version
610 }
611
612 fn validate(&self) -> Result<(), LeaseError> {
613 if !self.transition.is_canonical()
614 || !self.leader_proof.is_canonical()
615 || (self.verdict == AssignmentDrainVerdict::Commit
616 && self.leader_proof != self.transition.leader)
617 {
618 return Err(LeaseError::Invalid(
619 "assignment drain decision is not canonical".into(),
620 ));
621 }
622 Ok(())
623 }
624}
625
626#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
628#[serde(rename_all = "snake_case")]
629pub enum AssignmentDrainVerdict {
630 Commit,
632 Abort,
634}
635
636#[derive(Debug, Clone, PartialEq, Eq)]
638pub enum RecordAssignmentDrainDecisionResult {
639 Created(AssignmentDrainDecision),
641 Unchanged(AssignmentDrainDecision),
643 Conflict {
645 winner: AssignmentDrainDecision,
647 },
648}
649
650#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
652#[serde(deny_unknown_fields)]
653pub struct AssignmentRecoveryDecision {
654 pub predecessor: CheckpointAssignmentFence,
656 pub target: CheckpointAssignmentFence,
658 pub proposal: AssignmentSnapshotRef,
660 pub removed_process_fences: Vec<ProcessLeaseFence>,
662 pub leader_proof: LeaderProof,
664}
665
666impl AssignmentRecoveryDecision {
667 pub fn new(
672 predecessor: CheckpointAssignmentFence,
673 target: CheckpointAssignmentFence,
674 proposal: AssignmentSnapshotRef,
675 removed_process_fences: Vec<ProcessLeaseFence>,
676 leader_proof: LeaderProof,
677 ) -> Result<Self, String> {
678 let decision = Self {
679 predecessor,
680 target,
681 proposal,
682 removed_process_fences,
683 leader_proof,
684 };
685 decision.validate().map_err(|error| error.to_string())?;
686 Ok(decision)
687 }
688
689 #[must_use]
691 pub fn target_version(&self) -> u64 {
692 self.target.assignment_version
693 }
694
695 fn validate(&self) -> Result<(), LeaseError> {
696 if !self.predecessor.is_canonical()
697 || !self.target.is_canonical()
698 || !self.leader_proof.is_canonical()
699 || self.predecessor.vnode_count != self.target.vnode_count
700 || self.predecessor.assignment_version.checked_add(1)
701 != Some(self.target.assignment_version)
702 || self.proposal.validate().is_err()
703 || self.proposal.version != self.target.assignment_version
704 {
705 return Err(LeaseError::Invalid(
706 "assignment recovery decision is not a canonical successor".into(),
707 ));
708 }
709
710 let removed: Vec<_> = self
711 .predecessor
712 .participants
713 .iter()
714 .filter(|participant| {
715 self.target.participant_incarnation(participant.node_id)
716 != Some(participant.boot_incarnation)
717 })
718 .collect();
719 if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
720 return Err(LeaseError::Invalid(
721 "assignment recovery decision requires the exact removed predecessor process set"
722 .into(),
723 ));
724 }
725 for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
726 if !fence.is_canonical()
727 || fence.predecessor.node.0 != participant.node_id
728 || fence.predecessor.owner != participant.boot_incarnation
729 {
730 return Err(LeaseError::Invalid(
731 "assignment recovery process fences are incomplete, unsorted, or unrelated"
732 .into(),
733 ));
734 }
735 }
736 Ok(())
737 }
738}
739
740#[derive(Debug, Clone, PartialEq, Eq)]
742pub enum RecordAssignmentRecoveryDecisionResult {
743 Created(AssignmentRecoveryDecision),
745 Unchanged(AssignmentRecoveryDecision),
747 Conflict {
749 winner: AssignmentRecoveryDecision,
751 },
752}
753
754#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
755#[serde(
756 tag = "kind",
757 content = "decision",
758 rename_all = "snake_case",
759 deny_unknown_fields
760)]
761enum AuthorityAssignmentDecision {
762 Drain(AssignmentDrainDecision),
763 Recovery(AssignmentRecoveryDecision),
764}
765
766impl AuthorityAssignmentDecision {
767 fn target_version(&self) -> u64 {
768 match self {
769 Self::Drain(decision) => decision.target_version(),
770 Self::Recovery(decision) => decision.target_version(),
771 }
772 }
773
774 fn leader_proof(&self) -> &LeaderProof {
775 match self {
776 Self::Drain(decision) => &decision.leader_proof,
777 Self::Recovery(decision) => &decision.leader_proof,
778 }
779 }
780
781 fn validate(&self) -> Result<(), LeaseError> {
782 match self {
783 Self::Drain(decision) => decision.validate(),
784 Self::Recovery(decision) => decision.validate(),
785 }
786 }
787}
788
789enum RecordAuthorityAssignmentDecisionResult {
790 Created(AuthorityAssignmentDecision),
791 Unchanged(AuthorityAssignmentDecision),
792 Conflict { winner: AuthorityAssignmentDecision },
793}
794
795#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
796#[serde(deny_unknown_fields)]
797struct AuthorityOutcomeFloor {
798 deployment_id: String,
799 artifact_before_epoch: u64,
800 authority_before_epoch: u64,
801 terminal_anchor: Option<CheckpointOutcome>,
802 terminal_anchor_link: Option<OutcomeLink>,
803 committed_anchor: Option<CheckpointOutcome>,
804 committed_anchor_link: Option<OutcomeLink>,
805}
806
807#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
808#[serde(deny_unknown_fields)]
809struct AuthorityAssignmentDecisionFloor {
810 before_target_version: u64,
811 terminal_anchor: Option<AuthorityAssignmentDecision>,
812 terminal_anchor_link: Option<AssignmentDecisionLink>,
813}
814
815#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
817#[serde(deny_unknown_fields)]
818struct LeaderAuthorityRecord {
819 version: u32,
820 lease: LeaderLease,
821 checkpoint_outcome: Option<CheckpointOutcome>,
823 previous_outcome: Option<OutcomeLink>,
825 outcome_head: Option<OutcomeLink>,
827 previous_commit: Option<OutcomeLink>,
829 commit_head: Option<OutcomeLink>,
831 outcome_floor: Option<AuthorityOutcomeFloor>,
833 assignment_decision: Option<AuthorityAssignmentDecision>,
835 previous_assignment_decision: Option<AssignmentDecisionLink>,
837 assignment_decision_head: Option<AssignmentDecisionLink>,
839 assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
841 recovery_fault_revision: u64,
843 recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
845 recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
847 recovery_release_head: Option<RecoveryReleaseLink>,
849}
850
851impl LeaderLease {
852 fn validate(&self) -> Result<(), LeaseError> {
853 self.owner.validate()?;
854 if self.seq == 0
855 || self.renewal_sequence == 0
856 || self.renewal_sequence > self.seq
857 || self.token == 0
858 {
859 return Err(LeaseError::Invalid(
860 "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
861 .into(),
862 ));
863 }
864 if let Some(reference) = &self.catalog_manifest {
865 reference
866 .validate()
867 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
868 }
869 Ok(())
870 }
871
872 #[must_use]
874 pub fn proof(&self) -> LeaderProof {
875 LeaderProof {
876 owner: self.owner.proof_owner(),
877 fencing_token: self.token,
878 }
879 }
880
881 #[must_use]
883 pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
884 proof.is_canonical()
885 && proof.owner == self.owner.proof_owner()
886 && proof.fencing_token == self.token
887 }
888
889 fn has_same_liveness_identity(&self, other: &Self) -> bool {
890 self.owner == other.owner
891 && self.token == other.token
892 && self.renewal_sequence == other.renewal_sequence
893 }
894}
895
896impl AuthorityOutcomeFloor {
897 fn validate(&self) -> Result<(), LeaseError> {
898 let deployment = Uuid::parse_str(&self.deployment_id)
899 .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
900 if deployment.is_nil()
901 || deployment.to_string() != self.deployment_id
902 || self.authority_before_epoch == 0
903 || self.artifact_before_epoch > self.authority_before_epoch
904 {
905 return Err(LeaseError::Invalid(
906 "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
907 .into(),
908 ));
909 }
910 for link in [self.terminal_anchor_link, self.committed_anchor_link]
911 .into_iter()
912 .flatten()
913 {
914 link.validate()?;
915 }
916 for (name, anchor) in [
917 ("terminal", self.terminal_anchor.as_ref()),
918 ("committed", self.committed_anchor.as_ref()),
919 ] {
920 let Some(anchor) = anchor else { continue };
921 anchor
922 .validate_shape(anchor.epoch)
923 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
924 if anchor.scope != CheckpointScope::Cluster
925 || anchor.deployment_id != self.deployment_id
926 {
927 return Err(LeaseError::Invalid(format!(
928 "outcome floor has an invalid {name} anchor"
929 )));
930 }
931 }
932 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
933 (Some(anchor), Some(link))
934 if anchor.epoch < self.authority_before_epoch
935 && link.sequence != 0
936 && link.epoch == anchor.epoch
937 && link.checkpoint_id == anchor.checkpoint_id => {}
938 (None, None) => {}
939 _ => {
940 return Err(LeaseError::Invalid(
941 "outcome floor terminal anchor does not match its exact authority link".into(),
942 ));
943 }
944 }
945 match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
946 (Some(anchor), Some(link))
947 if self.artifact_before_epoch != 0
948 && anchor.is_commit()
949 && anchor.epoch < self.artifact_before_epoch
950 && link.sequence != 0
951 && link.epoch == anchor.epoch
952 && link.checkpoint_id == anchor.checkpoint_id => {}
953 (None, None) => {}
954 _ => {
955 return Err(LeaseError::Invalid(
956 "outcome floor committed anchor does not match its exact Commit link".into(),
957 ));
958 }
959 }
960 if let (Some(committed), Some(terminal)) = (
961 self.committed_anchor.as_ref(),
962 self.terminal_anchor.as_ref(),
963 ) {
964 let ordered = if committed.epoch == terminal.epoch {
965 committed == terminal
966 } else {
967 committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
968 };
969 if !ordered {
970 return Err(LeaseError::Invalid(
971 "outcome floor anchors are not monotonically ordered".into(),
972 ));
973 }
974 let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
975 (Some(committed_link), Some(terminal_link)) if committed == terminal => {
976 committed_link == terminal_link
977 }
978 (Some(committed_link), Some(terminal_link)) => {
979 committed_link.sequence < terminal_link.sequence
980 }
981 _ => false,
982 };
983 if !links_ordered {
984 return Err(LeaseError::Invalid(
985 "outcome floor anchor links are not ordered with their exact outcomes".into(),
986 ));
987 }
988 } else if self.committed_anchor.is_some() {
989 return Err(LeaseError::Invalid(
990 "outcome floor has a committed anchor without terminal continuity".into(),
991 ));
992 }
993 Ok(())
994 }
995}
996
997impl AuthorityAssignmentDecisionFloor {
998 fn validate(&self) -> Result<(), LeaseError> {
999 if self.before_target_version == 0 {
1000 return Err(LeaseError::Invalid(
1001 "assignment decision floor requires a nonzero target version".into(),
1002 ));
1003 }
1004 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1005 (Some(anchor), Some(link)) => {
1006 anchor.validate()?;
1007 if link.sequence == 0
1008 || link.target_version != anchor.target_version()
1009 || anchor.target_version() >= self.before_target_version
1010 {
1011 return Err(LeaseError::Invalid(
1012 "assignment decision floor anchor does not match its exact authority link"
1013 .into(),
1014 ));
1015 }
1016 }
1017 (None, None) => {}
1018 _ => {
1019 return Err(LeaseError::Invalid(
1020 "assignment decision floor has an incomplete terminal anchor".into(),
1021 ));
1022 }
1023 }
1024 Ok(())
1025 }
1026}
1027
1028impl LeaderAuthorityRecord {
1029 fn initial(lease: LeaderLease) -> Self {
1030 Self {
1031 version: AUTHORITY_RECORD_VERSION,
1032 lease,
1033 checkpoint_outcome: None,
1034 previous_outcome: None,
1035 outcome_head: None,
1036 previous_commit: None,
1037 commit_head: None,
1038 outcome_floor: None,
1039 assignment_decision: None,
1040 previous_assignment_decision: None,
1041 assignment_decision_head: None,
1042 assignment_decision_floor: None,
1043 recovery_fault_revision: 0,
1044 recovery_fault_slots: Vec::new(),
1045 recovery_release_commit: None,
1046 recovery_release_head: None,
1047 }
1048 }
1049
1050 fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1051 Self {
1052 version: AUTHORITY_RECORD_VERSION,
1053 lease,
1054 checkpoint_outcome: None,
1055 previous_outcome: None,
1056 outcome_head: self.outcome_head,
1057 previous_commit: None,
1058 commit_head: self.commit_head,
1059 outcome_floor: self.outcome_floor.clone(),
1060 assignment_decision: None,
1061 previous_assignment_decision: None,
1062 assignment_decision_head: self.assignment_decision_head,
1063 assignment_decision_floor: self.assignment_decision_floor.clone(),
1064 recovery_fault_revision: self.recovery_fault_revision,
1065 recovery_fault_slots: self.recovery_fault_slots.clone(),
1066 recovery_release_commit: None,
1067 recovery_release_head: self.recovery_release_head.clone(),
1068 }
1069 }
1070
1071 fn validate(&self) -> Result<(), LeaseError> {
1072 if self.version != AUTHORITY_RECORD_VERSION {
1073 return Err(LeaseError::Invalid(format!(
1074 "authority record version {} is unsupported",
1075 self.version
1076 )));
1077 }
1078 self.lease.validate()?;
1079 for link in [
1080 self.previous_outcome,
1081 self.outcome_head,
1082 self.previous_commit,
1083 self.commit_head,
1084 ]
1085 .into_iter()
1086 .flatten()
1087 {
1088 link.validate()?;
1089 }
1090 if let Some(floor) = &self.outcome_floor {
1091 floor.validate()?;
1092 }
1093 if let Some(floor) = &self.assignment_decision_floor {
1094 floor.validate()?;
1095 }
1096 self.validate_recovery_fault_inventory()?;
1097 self.validate_checkpoint_outcome_chain()?;
1098 self.validate_outcome_floor()?;
1099 self.validate_assignment_decision_chain()?;
1100 self.validate_assignment_decision_floor()?;
1101 self.validate_recovery_release()?;
1102 Ok(())
1103 }
1104
1105 fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1106 if self.recovery_fault_slots.windows(2).any(|pair| {
1107 pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1108 }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1109 || (self.recovery_fault_revision != 0
1110 && self.recovery_fault_slots.is_empty()
1111 && self
1112 .recovery_release_head
1113 .as_ref()
1114 .map(|head| head.sequence)
1115 != Some(self.recovery_fault_revision))
1116 || self.recovery_fault_revision > self.lease.seq
1117 || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1118 {
1119 return Err(LeaseError::Invalid(
1120 "leader authority recovery-fault inventory is not canonical".into(),
1121 ));
1122 }
1123 let mut fault_sequences = BTreeSet::new();
1124 for slot in &self.recovery_fault_slots {
1125 slot.validate()?;
1126 if slot.fault_sequence > self.recovery_fault_revision
1127 || slot.fault_sequence > self.lease.seq
1128 || !fault_sequences.insert(slot.fault_sequence)
1129 {
1130 return Err(LeaseError::Invalid(
1131 "leader authority recovery-fault sequences are not canonical".into(),
1132 ));
1133 }
1134 }
1135 if [
1136 self.checkpoint_outcome.is_some(),
1137 self.assignment_decision.is_some(),
1138 self.recovery_release_commit.is_some(),
1139 ]
1140 .into_iter()
1141 .filter(|present| *present)
1142 .count()
1143 > 1
1144 {
1145 return Err(LeaseError::Invalid(
1146 "one authority sequence cannot admit two terminal domains".into(),
1147 ));
1148 }
1149 Ok(())
1150 }
1151
1152 fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1153 if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1154 outcome
1155 .validate_shape(outcome.epoch)
1156 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1157 if outcome.scope != CheckpointScope::Cluster {
1158 return Err(LeaseError::Invalid(
1159 "leader authority can only admit cluster checkpoint outcomes".into(),
1160 ));
1161 }
1162 let proof = outcome
1163 .leader_proof
1164 .as_ref()
1165 .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1166 let current_link = OutcomeLink {
1167 sequence: self.lease.seq,
1168 epoch: outcome.epoch,
1169 checkpoint_id: outcome.checkpoint_id,
1170 };
1171 if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1172 return Err(LeaseError::Invalid(
1173 "cluster outcome is not bound to its exact authority sequence and term".into(),
1174 ));
1175 }
1176 if let Some(previous) = self.previous_outcome {
1177 if previous.sequence >= self.lease.seq
1178 || previous.epoch >= outcome.epoch
1179 || previous.checkpoint_id >= outcome.checkpoint_id
1180 {
1181 return Err(LeaseError::Invalid(
1182 "cluster outcome link does not move backward in sequence and epoch".into(),
1183 ));
1184 }
1185 }
1186 if outcome.is_commit() {
1187 if self.commit_head != Some(current_link) {
1188 return Err(LeaseError::Invalid(
1189 "cluster Commit is not bound to its exact Commit-chain head".into(),
1190 ));
1191 }
1192 if let Some(previous) = self.previous_commit {
1193 if previous.sequence >= self.lease.seq
1194 || previous.epoch >= outcome.epoch
1195 || previous.checkpoint_id >= outcome.checkpoint_id
1196 {
1197 return Err(LeaseError::Invalid(
1198 "cluster Commit link does not move backward in sequence and epoch"
1199 .into(),
1200 ));
1201 }
1202 if self.outcome_floor.as_ref().is_some_and(|floor| {
1203 previous.epoch < floor.artifact_before_epoch
1204 && Some(previous) != floor.committed_anchor_link
1205 }) {
1206 return Err(LeaseError::Invalid(
1207 "cluster Commit link crosses the artifact floor without its exact anchor"
1208 .into(),
1209 ));
1210 }
1211 }
1212 } else if self.previous_commit.is_some() {
1213 return Err(LeaseError::Invalid(
1214 "cluster Abort carries a previous-Commit link".into(),
1215 ));
1216 }
1217 if let Some(floor) = &self.outcome_floor {
1218 if outcome.deployment_id != floor.deployment_id
1219 || outcome.epoch < floor.authority_before_epoch
1220 {
1221 return Err(LeaseError::Invalid(
1222 "cluster outcome is below or outside its durable authority floor".into(),
1223 ));
1224 }
1225 }
1226 } else {
1227 if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1228 return Err(LeaseError::Invalid(
1229 "non-outcome authority record carries an outcome-chain link".into(),
1230 ));
1231 }
1232 if self.outcome_head.is_some_and(|head| {
1233 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1234 }) {
1235 return Err(LeaseError::Invalid(
1236 "authority outcome head is outside the durable sequence".into(),
1237 ));
1238 }
1239 }
1240 if self.commit_head.is_some_and(|head| {
1241 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1242 }) {
1243 return Err(LeaseError::Invalid(
1244 "authority Commit head is outside the durable sequence".into(),
1245 ));
1246 }
1247 match (self.commit_head, self.outcome_head) {
1248 (Some(commit), Some(terminal)) => {
1249 let equal = commit == terminal;
1250 let strictly_older = commit.sequence < terminal.sequence
1251 && commit.epoch < terminal.epoch
1252 && commit.checkpoint_id < terminal.checkpoint_id;
1253 if !equal && !strictly_older {
1254 return Err(LeaseError::Invalid(
1255 "authority Commit head is not ordered behind the terminal head".into(),
1256 ));
1257 }
1258 if self
1259 .checkpoint_outcome
1260 .as_ref()
1261 .is_some_and(|outcome| !outcome.is_commit())
1262 && !strictly_older
1263 {
1264 return Err(LeaseError::Invalid(
1265 "cluster Abort did not preserve a strictly older Commit head".into(),
1266 ));
1267 }
1268 }
1269 (Some(_), None) => {
1270 return Err(LeaseError::Invalid(
1271 "authority Commit head exists without terminal continuity".into(),
1272 ));
1273 }
1274 (None, _) => {}
1275 }
1276 Ok(())
1277 }
1278
1279 fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1280 if let Some(floor) = self.outcome_floor.as_ref() {
1281 if floor
1282 .terminal_anchor_link
1283 .is_some_and(|link| link.sequence >= self.lease.seq)
1284 {
1285 return Err(LeaseError::Invalid(
1286 "outcome floor anchor is outside the authority sequence".into(),
1287 ));
1288 }
1289 if floor
1290 .committed_anchor_link
1291 .is_some_and(|link| link.sequence >= self.lease.seq)
1292 {
1293 return Err(LeaseError::Invalid(
1294 "outcome floor Commit anchor is outside the authority sequence".into(),
1295 ));
1296 }
1297 if let Some(head) = self.outcome_head {
1298 if head.epoch < floor.authority_before_epoch
1299 && Some(head) != floor.terminal_anchor_link
1300 {
1301 return Err(LeaseError::Invalid(
1302 "authority outcome head does not meet its durable compaction floor".into(),
1303 ));
1304 }
1305 } else if floor.terminal_anchor_link.is_some() {
1306 return Err(LeaseError::Invalid(
1307 "outcome floor anchor is disconnected from the authority head".into(),
1308 ));
1309 }
1310 if let Some(head) = self.commit_head {
1311 if head.epoch < floor.artifact_before_epoch
1312 && Some(head) != floor.committed_anchor_link
1313 {
1314 return Err(LeaseError::Invalid(
1315 "authority Commit head does not meet its durable artifact floor".into(),
1316 ));
1317 }
1318 } else if floor.committed_anchor_link.is_some() {
1319 return Err(LeaseError::Invalid(
1320 "outcome floor Commit anchor is disconnected from the Commit head".into(),
1321 ));
1322 }
1323 }
1324 Ok(())
1325 }
1326
1327 fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1328 if let Some(decision) = self.assignment_decision.as_ref() {
1329 decision.validate()?;
1330 if self
1331 .assignment_decision_floor
1332 .as_ref()
1333 .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1334 {
1335 return Err(LeaseError::Invalid(
1336 "assignment decision is below its durable authority floor".into(),
1337 ));
1338 }
1339 if !self.lease.matches_proof(decision.leader_proof())
1340 || self.assignment_decision_head
1341 != Some(AssignmentDecisionLink {
1342 sequence: self.lease.seq,
1343 target_version: decision.target_version(),
1344 })
1345 {
1346 return Err(LeaseError::Invalid(
1347 "assignment decision is not bound to its exact authority sequence and term"
1348 .into(),
1349 ));
1350 }
1351 if let Some(previous) = self.previous_assignment_decision {
1352 if previous.sequence >= self.lease.seq
1353 || previous.target_version >= decision.target_version()
1354 {
1355 return Err(LeaseError::Invalid(
1356 "assignment decision link does not move backward".into(),
1357 ));
1358 }
1359 }
1360 } else {
1361 if self.previous_assignment_decision.is_some() {
1362 return Err(LeaseError::Invalid(
1363 "non-decision authority record carries a previous assignment-decision link"
1364 .into(),
1365 ));
1366 }
1367 if self
1368 .assignment_decision_head
1369 .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1370 {
1371 return Err(LeaseError::Invalid(
1372 "authority assignment-decision head is outside the durable sequence".into(),
1373 ));
1374 }
1375 }
1376 Ok(())
1377 }
1378
1379 fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1380 if let Some(floor) = self.assignment_decision_floor.as_ref() {
1381 if floor
1382 .terminal_anchor_link
1383 .is_some_and(|link| link.sequence >= self.lease.seq)
1384 {
1385 return Err(LeaseError::Invalid(
1386 "assignment decision floor anchor is outside the authority sequence".into(),
1387 ));
1388 }
1389 if let Some(head) = self.assignment_decision_head {
1390 if head.target_version < floor.before_target_version
1391 && Some(head) != floor.terminal_anchor_link
1392 {
1393 return Err(LeaseError::Invalid(
1394 "authority assignment-decision head does not meet its durable floor".into(),
1395 ));
1396 }
1397 } else if floor.terminal_anchor_link.is_some() {
1398 return Err(LeaseError::Invalid(
1399 "assignment decision floor anchor is disconnected from the authority head"
1400 .into(),
1401 ));
1402 }
1403 }
1404 Ok(())
1405 }
1406
1407 fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1408 match self.recovery_release_commit.as_ref() {
1409 Some(commit) => {
1410 commit.validate()?;
1411 let expected = RecoveryReleaseLink {
1412 sequence: self.lease.seq,
1413 terminal: commit.terminal.clone(),
1414 };
1415 if !self.lease.matches_proof(&commit.leader_proof)
1416 || self.recovery_release_head.as_ref() != Some(&expected)
1417 || self.recovery_fault_revision != self.lease.seq
1418 || self.recovery_fault_slots.iter().any(|slot| slot.active)
1419 {
1420 return Err(LeaseError::Invalid(
1421 "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1422 .into(),
1423 ));
1424 }
1425 }
1426 None => {
1427 if let Some(head) = self.recovery_release_head.as_ref() {
1428 head.validate()?;
1429 if head.sequence >= self.lease.seq {
1430 return Err(LeaseError::Invalid(
1431 "recovery release head is outside the durable authority sequence"
1432 .into(),
1433 ));
1434 }
1435 if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1436 && self.recovery_fault_revision != head.sequence
1437 {
1438 return Err(LeaseError::Invalid(
1439 "settled recovery fault inventory is not bound to its release head"
1440 .into(),
1441 ));
1442 }
1443 }
1444 }
1445 }
1446 Ok(())
1447 }
1448}
1449
1450#[derive(Debug, Clone, PartialEq, Eq)]
1452pub enum LeaseOutcome {
1453 Acquired(LeaderLease),
1455 Held(LeaderLease),
1457}
1458
1459enum AuthorityCreateOutcome {
1460 Created,
1461 ExistingIdentical,
1462 Contended(Box<LeaderAuthorityRecord>),
1463}
1464
1465#[derive(Clone, Copy)]
1466enum SameOwnerToken {
1467 #[cfg(test)]
1468 Preserve,
1469 Rotate,
1470 Exact(u64),
1471}
1472
1473#[derive(Debug)]
1475pub struct LeaderLeaseObservation {
1476 lease: LeaderLease,
1477 started: Instant,
1478}
1479
1480#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1483pub struct LeaderCandidacy {
1484 eligible: bool,
1485 generation: u64,
1486}
1487
1488impl LeaderCandidacy {
1489 pub(crate) const fn initial(eligible: bool) -> Self {
1490 Self {
1491 eligible,
1492 generation: 1,
1493 }
1494 }
1495
1496 pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1497 let generation = if self.eligible && !eligible {
1498 self.generation.checked_add(1)?
1499 } else {
1500 self.generation
1501 };
1502 Some(Self {
1503 eligible,
1504 generation,
1505 })
1506 }
1507
1508 pub(crate) const fn terminal() -> Self {
1509 Self {
1510 eligible: false,
1511 generation: u64::MAX,
1512 }
1513 }
1514
1515 #[must_use]
1517 pub const fn is_eligible(self) -> bool {
1518 self.eligible
1519 }
1520}
1521
1522#[derive(Debug, thiserror::Error)]
1524pub enum LeaseError {
1525 #[error("object store I/O: {0}")]
1527 Io(String),
1528 #[error("invalid leader lease: {0}")]
1530 Invalid(String),
1531 #[error("leader lease fenced: {0}")]
1533 Fenced(String),
1534 #[error("JSON: {0}")]
1536 Json(#[from] serde_json::Error),
1537}
1538
1539#[derive(Debug, thiserror::Error)]
1541pub enum ClusterCheckpointAuthorityError {
1542 #[error("cluster checkpoint authority is not installed")]
1544 NotConfigured,
1545 #[error("leader authority: {0}")]
1547 Authority(#[from] LeaseError),
1548 #[error("checkpoint decision: {0}")]
1550 Decision(#[from] DecisionError),
1551 #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1553 Fenced,
1554}
1555
1556#[derive(Debug, Clone, PartialEq, Eq)]
1558pub struct ClusterOutcomeRetentionBoundary {
1559 pub artifact_before_epoch: u64,
1561 pub terminal_before_epoch: u64,
1563 pub committed_anchor: Option<CheckpointOutcome>,
1565 pub terminal_anchor: Option<CheckpointOutcome>,
1567}
1568
1569impl ClusterOutcomeRetentionBoundary {
1570 fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1571 floor.map_or(
1572 Self {
1573 artifact_before_epoch: 0,
1574 terminal_before_epoch: 0,
1575 committed_anchor: None,
1576 terminal_anchor: None,
1577 },
1578 |floor| Self {
1579 artifact_before_epoch: floor.artifact_before_epoch,
1580 terminal_before_epoch: floor.authority_before_epoch,
1581 committed_anchor: floor.committed_anchor.clone(),
1582 terminal_anchor: floor.terminal_anchor.clone(),
1583 },
1584 )
1585 }
1586}
1587
1588#[derive(Debug, Clone, PartialEq, Eq)]
1590pub struct ClusterOutcomeInventory {
1591 pub outcomes: Vec<CheckpointOutcome>,
1593 pub retention_boundary: ClusterOutcomeRetentionBoundary,
1595}
1596
1597pub struct LeaderLeaseStore {
1599 store: Arc<dyn ObjectStore>,
1600 ttl_ms: i64,
1601 prune_running: Arc<AtomicBool>,
1602 outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
1603 outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
1604}
1605
1606#[derive(Clone, PartialEq, Eq)]
1607struct ClusterOutcomeAuditKey {
1608 terminal_head: Option<OutcomeLink>,
1609 commit_head: Option<OutcomeLink>,
1610 floor: Option<AuthorityOutcomeFloor>,
1611}
1612
1613struct ClusterOutcomeAuditCache {
1614 key: ClusterOutcomeAuditKey,
1615 authority_sequence: u64,
1616 snapshot: ClusterOutcomeAuditSnapshot,
1617}
1618
1619#[derive(Clone)]
1620struct ClusterOutcomeAuditSnapshot {
1621 outcomes: Arc<[CheckpointOutcome]>,
1622 terminal_links: Arc<[OutcomeLink]>,
1623 commit_links: Arc<[OutcomeLink]>,
1624}
1625
1626struct ClusterOutcomeAuditFlight {
1627 key: ClusterOutcomeAuditKey,
1628 gate: Weak<tokio::sync::Mutex<()>>,
1629}
1630
1631struct PruneLatchGuard(Arc<AtomicBool>);
1632
1633impl Drop for PruneLatchGuard {
1634 fn drop(&mut self) {
1635 self.0.store(false, Ordering::Release);
1636 }
1637}
1638
1639impl std::fmt::Debug for LeaderLeaseStore {
1640 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1641 formatter
1642 .debug_struct("LeaderLeaseStore")
1643 .field("ttl_ms", &self.ttl_ms)
1644 .finish_non_exhaustive()
1645 }
1646}
1647
1648impl LeaderLeaseStore {
1649 #[must_use]
1654 pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
1655 Self {
1656 store,
1657 ttl_ms,
1658 prune_running: Arc::new(AtomicBool::new(false)),
1659 outcome_audit_cache: Mutex::new(None),
1660 outcome_audit_flights: Mutex::new(Vec::new()),
1661 }
1662 }
1663
1664 pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
1673 if timeout.is_zero() {
1674 return Err(LeaseError::Invalid(
1675 "object-store contract probe timeout must be nonzero".into(),
1676 ));
1677 }
1678
1679 let path = OsPath::from(format!(
1680 "{STORE_CONTRACT_PROBE_PREFIX}{}.probe",
1681 Uuid::new_v4()
1682 ));
1683 let validation = tokio::time::timeout(timeout, self.validate_store_contract_at(&path))
1684 .await
1685 .unwrap_or_else(|_| {
1686 Err(LeaseError::Io(format!(
1687 "object-store contract probe exceeded {timeout:?}"
1688 )))
1689 });
1690 let cleanup = tokio::time::timeout(timeout, self.cleanup_store_contract_probe(&path))
1691 .await
1692 .unwrap_or_else(|_| {
1693 Err(LeaseError::Io(format!(
1694 "object-store contract probe cleanup exceeded {timeout:?}"
1695 )))
1696 });
1697
1698 match (validation, cleanup) {
1699 (Ok(()), Ok(())) => Ok(()),
1700 (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
1701 (Err(validation), Err(cleanup)) => Err(LeaseError::Io(format!(
1702 "object-store contract validation failed ({validation}); cleanup failed ({cleanup})"
1703 ))),
1704 }
1705 }
1706
1707 async fn validate_store_contract_at(&self, path: &OsPath) -> Result<(), LeaseError> {
1708 self.store
1709 .put_opts(
1710 path,
1711 PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_V1)),
1712 PutOptions {
1713 mode: PutMode::Create,
1714 ..PutOptions::default()
1715 },
1716 )
1717 .await
1718 .map_err(|error| {
1719 LeaseError::Io(format!(
1720 "object-store contract PutMode::Create failed: {error}"
1721 ))
1722 })?;
1723
1724 self.require_store_contract_create_conflict(path).await?;
1725
1726 let version_v1 = self
1727 .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V1)
1728 .await?;
1729 self.store
1730 .put_opts(
1731 path,
1732 PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_V2)),
1733 PutOptions {
1734 mode: PutMode::Update(version_v1.clone()),
1735 ..PutOptions::default()
1736 },
1737 )
1738 .await
1739 .map_err(|error| {
1740 LeaseError::Io(format!(
1741 "object-store contract PutMode::Update failed: {error}"
1742 ))
1743 })?;
1744
1745 let version_v2 = self
1746 .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V2)
1747 .await?;
1748 if version_v2 == version_v1 {
1749 return Err(LeaseError::Invalid(
1750 "object-store conditional-update version did not change after PutMode::Update"
1751 .into(),
1752 ));
1753 }
1754
1755 match self
1756 .store
1757 .put_opts(
1758 path,
1759 PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_STALE)),
1760 PutOptions {
1761 mode: PutMode::Update(version_v1),
1762 ..PutOptions::default()
1763 },
1764 )
1765 .await
1766 {
1767 Err(object_store::Error::Precondition { .. }) => {}
1768 Ok(_) => {
1769 return Err(LeaseError::Invalid(
1770 "object-store accepted a stale PutMode::Update token".into(),
1771 ));
1772 }
1773 Err(error) => {
1774 return Err(LeaseError::Io(format!(
1775 "object-store stale PutMode::Update failed unexpectedly: {error}"
1776 )));
1777 }
1778 }
1779
1780 let durable_version = self
1781 .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V2)
1782 .await?;
1783 if durable_version != version_v2 {
1784 return Err(LeaseError::Invalid(
1785 "object-store version changed after rejecting stale PutMode::Update".into(),
1786 ));
1787 }
1788 Ok(())
1789 }
1790
1791 async fn require_store_contract_create_conflict(
1792 &self,
1793 path: &OsPath,
1794 ) -> Result<(), LeaseError> {
1795 match self
1796 .store
1797 .put_opts(
1798 path,
1799 PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_STALE)),
1800 PutOptions {
1801 mode: PutMode::Create,
1802 ..PutOptions::default()
1803 },
1804 )
1805 .await
1806 {
1807 Err(
1808 object_store::Error::AlreadyExists { .. }
1809 | object_store::Error::Precondition { .. },
1810 ) => Ok(()),
1811 Ok(_) => Err(LeaseError::Invalid(
1812 "object-store contract PutMode::Create overwrote an existing object".into(),
1813 )),
1814 Err(error) => Err(LeaseError::Io(format!(
1815 "object-store contract duplicate PutMode::Create failed unexpectedly: {error}"
1816 ))),
1817 }
1818 }
1819
1820 async fn read_store_contract_probe(
1821 &self,
1822 path: &OsPath,
1823 expected: &[u8],
1824 ) -> Result<UpdateVersion, LeaseError> {
1825 let result = self.store.get(path).await.map_err(|error| {
1826 LeaseError::Io(format!("object-store contract probe GET failed: {error}"))
1827 })?;
1828 let version = UpdateVersion {
1829 e_tag: result.meta.e_tag.clone(),
1830 version: result.meta.version.clone(),
1831 };
1832 if version.e_tag.is_none() && version.version.is_none() {
1833 return Err(LeaseError::Invalid(
1834 "object-store contract probe GET returned neither ETag nor version".into(),
1835 ));
1836 }
1837 if result.meta.size != u64::try_from(expected.len()).unwrap_or(u64::MAX) {
1838 return Err(LeaseError::Invalid(format!(
1839 "object-store contract probe is {} bytes; expected {}",
1840 result.meta.size,
1841 expected.len()
1842 )));
1843 }
1844 let bytes = result.bytes().await.map_err(|error| {
1845 LeaseError::Io(format!(
1846 "object-store contract probe payload read failed: {error}"
1847 ))
1848 })?;
1849 if bytes.as_ref() != expected {
1850 return Err(LeaseError::Invalid(
1851 "object-store contract probe content changed unexpectedly".into(),
1852 ));
1853 }
1854 Ok(version)
1855 }
1856
1857 async fn cleanup_store_contract_probe(&self, path: &OsPath) -> Result<(), LeaseError> {
1858 let delete_error = match self.store.delete(path).await {
1859 Ok(()) | Err(object_store::Error::NotFound { .. }) => None,
1860 Err(error) => Some(error.to_string()),
1861 };
1862 match self.store.get(path).await {
1863 Err(object_store::Error::NotFound { .. }) => Ok(()),
1864 Ok(_) => Err(LeaseError::Io(delete_error.map_or_else(
1865 || "object-store contract probe remained visible after deletion".into(),
1866 |error| {
1867 format!(
1868 "object-store contract probe deletion failed ({error}) and the object remained visible"
1869 )
1870 },
1871 ))),
1872 Err(error) => Err(LeaseError::Io(delete_error.map_or_else(
1873 || format!("verify object-store contract probe deletion: {error}"),
1874 |delete| {
1875 format!(
1876 "object-store contract probe deletion failed ({delete}); verification failed ({error})"
1877 )
1878 },
1879 ))),
1880 }
1881 }
1882
1883 fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
1884 RecoveryFaultInventory {
1885 revision: record.recovery_fault_revision,
1886 faults: record
1887 .recovery_fault_slots
1888 .iter()
1889 .filter(|slot| slot.active)
1890 .map(AuthorityRecoveryFaultSlot::fault)
1891 .collect(),
1892 }
1893 }
1894
1895 pub(crate) async fn recovery_fault_inventory(
1896 &self,
1897 ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
1898 let record = self
1899 .load_record()
1900 .await?
1901 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
1902 Ok(Self::recovery_fault_inventory_from(&record))
1903 }
1904
1905 pub(crate) async fn record_recovery_fault(
1906 &self,
1907 publisher: RecoveryFaultPublisher,
1908 request_sequence: u64,
1909 ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
1910 publisher.validate().map_err(LeaseError::Invalid)?;
1911 if request_sequence == 0 {
1912 return Err(LeaseError::Invalid(
1913 "recovery fault request sequence must be nonzero".into(),
1914 )
1915 .into());
1916 }
1917
1918 loop {
1919 let head = self
1920 .load_published_authority_head()
1921 .await?
1922 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
1923 let current = &head.record;
1924 let node_id = publisher.participant.node_id;
1925 let slot_index = current
1926 .recovery_fault_slots
1927 .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
1928 let (insert_at, replace) = match slot_index {
1929 Ok(index) => {
1930 let slot = ¤t.recovery_fault_slots[index];
1931 if slot.matches_request(publisher, request_sequence) {
1932 return Ok(if slot.active {
1933 RecordRecoveryFaultResult::Active
1934 } else {
1935 RecordRecoveryFaultResult::AlreadyCleared
1936 });
1937 }
1938 if publisher.process_term < slot.publisher.process_term {
1939 return Ok(RecordRecoveryFaultResult::Superseded);
1940 }
1941 if publisher.process_term == slot.publisher.process_term
1942 && publisher.participant.boot_incarnation
1943 != slot.publisher.participant.boot_incarnation
1944 {
1945 return Err(LeaseError::Invalid(format!(
1946 "stable node {node_id} has two recovery fault publishers for process term {}",
1947 publisher.process_term
1948 ))
1949 .into());
1950 }
1951 if publisher.process_term == slot.publisher.process_term
1952 && request_sequence < slot.request_sequence
1953 {
1954 return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
1955 }
1956 (index, true)
1957 }
1958 Err(index) => (index, false),
1959 };
1960
1961 let sequence =
1962 current.lease.seq.checked_add(1).ok_or_else(|| {
1963 LeaseError::Invalid("leader authority sequence exhausted".into())
1964 })?;
1965 let mut lease = current.lease.clone();
1966 lease.seq = sequence;
1967 let mut candidate = current.preserve_with_lease(lease);
1968 let slot = AuthorityRecoveryFaultSlot {
1969 publisher,
1970 request_sequence,
1971 fault_sequence: sequence,
1975 active: true,
1976 };
1977 if replace {
1978 candidate.recovery_fault_slots[insert_at] = slot;
1979 } else {
1980 candidate.recovery_fault_slots.insert(insert_at, slot);
1981 }
1982 candidate.recovery_fault_revision = sequence;
1983 candidate.validate()?;
1984
1985 match self
1986 .create_authority_record(Some(&head), &candidate)
1987 .await?
1988 {
1989 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
1990 return Ok(RecordRecoveryFaultResult::Active);
1991 }
1992 AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
1993 }
1994 }
1995 }
1996
1997 pub(crate) async fn authorize_recovery_release(
1998 &self,
1999 clearer: RecoveryFaultPublisher,
2000 terminal: &RecoveryAnnouncement,
2001 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2002 clearer.validate().map_err(LeaseError::Invalid)?;
2003 let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2004 let reporter = NodeId(clearer.participant.node_id);
2005 let expected = terminal.round.fault_sequence(reporter);
2006 let current = self
2007 .load_record()
2008 .await?
2009 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2010 let Some(release_head) = current.recovery_release_head.as_ref() else {
2011 return Ok(false);
2012 };
2013 if release_head.terminal != terminal_reference
2014 || current.recovery_fault_revision != release_head.sequence
2015 {
2016 return Ok(false);
2017 }
2018 if current.recovery_fault_slots.iter().any(|slot| slot.active) {
2019 return Ok(false);
2020 }
2021 let slot_index = current
2022 .recovery_fault_slots
2023 .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2024 let Some(expected_sequence) = expected else {
2025 return Ok(match slot_index {
2026 Ok(index) => !current.recovery_fault_slots[index].active,
2027 Err(_) => true,
2028 });
2029 };
2030 let Ok(index) = slot_index else {
2031 return Ok(false);
2032 };
2033 let slot = ¤t.recovery_fault_slots[index];
2034 Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2035 }
2036
2037 pub(crate) async fn stage_recovery_release_terminal(
2038 &self,
2039 terminal: &RecoveryAnnouncement,
2040 ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2041 let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2042 let path = recovery_release_terminal_path(&reference);
2043 let put_error = match self
2044 .store
2045 .put_opts(
2046 &path,
2047 PutPayload::from(encoded),
2048 PutOptions {
2049 mode: PutMode::Create,
2050 ..PutOptions::default()
2051 },
2052 )
2053 .await
2054 {
2055 Ok(_)
2056 | Err(
2057 object_store::Error::AlreadyExists { .. }
2058 | object_store::Error::Precondition { .. },
2059 ) => None,
2060 Err(error) => Some(error),
2061 };
2062 match self.load_recovery_release_terminal(&reference).await {
2063 Ok(stored) if stored == *terminal => Ok(reference),
2064 Ok(_) => Err(LeaseError::Invalid(format!(
2065 "recovery release terminal '{}' differs from the proposed content",
2066 reference.sha256
2067 ))
2068 .into()),
2069 Err(reconcile_error) => {
2070 if let Some(put_error) = put_error {
2071 Err(LeaseError::Io(format!(
2072 "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2073 ))
2074 .into())
2075 } else {
2076 Err(reconcile_error)
2077 }
2078 }
2079 }
2080 }
2081
2082 pub(crate) async fn load_recovery_release_terminal(
2083 &self,
2084 reference: &RecoveryReleaseTerminalRef,
2085 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2086 reference.validate()?;
2087 let result = match self
2088 .store
2089 .get(&recovery_release_terminal_path(reference))
2090 .await
2091 {
2092 Ok(result) => result,
2093 Err(object_store::Error::NotFound { .. }) => {
2094 return Err(LeaseError::Invalid(format!(
2095 "recovery release terminal '{}' is missing",
2096 reference.sha256
2097 ))
2098 .into());
2099 }
2100 Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2101 };
2102 if result.meta.size != reference.encoded_len {
2103 return Err(LeaseError::Invalid(format!(
2104 "recovery release terminal '{}' is {} bytes, expected {}",
2105 reference.sha256, result.meta.size, reference.encoded_len
2106 ))
2107 .into());
2108 }
2109 let bytes = result
2110 .bytes()
2111 .await
2112 .map_err(|error| LeaseError::Io(error.to_string()))?;
2113 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2114 return Err(LeaseError::Invalid(format!(
2115 "recovery release terminal '{}' payload length changed while reading",
2116 reference.sha256
2117 ))
2118 .into());
2119 }
2120 let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2121 LeaseError::Invalid(format!(
2122 "recovery release terminal '{}': {error}",
2123 reference.sha256
2124 ))
2125 })?;
2126 let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2127 if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2128 return Err(LeaseError::Invalid(format!(
2129 "recovery release terminal '{}' does not match its content-addressed reference",
2130 reference.sha256
2131 ))
2132 .into());
2133 }
2134 Ok(terminal)
2135 }
2136
2137 pub(crate) async fn record_recovery_release_commit(
2138 &self,
2139 proof: &LeaderProof,
2140 reference: RecoveryReleaseTerminalRef,
2141 ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2142 reference.validate()?;
2143 let terminal = self.load_recovery_release_terminal(&reference).await?;
2144 if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2145 return Err(ClusterCheckpointAuthorityError::Fenced);
2146 }
2147
2148 loop {
2149 let published = self
2150 .load_published_authority_head()
2151 .await?
2152 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2153 let current = &published.record;
2154 if let Some(winner) = current.recovery_release_head.as_ref() {
2155 if winner.terminal == reference
2156 || winner.terminal.generation() >= reference.generation()
2157 {
2158 self.load_recovery_release_terminal(&winner.terminal)
2159 .await?;
2160 return if winner.terminal == reference {
2161 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2162 } else {
2163 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2164 winner: winner.terminal.clone(),
2165 })
2166 };
2167 }
2168 }
2169 if !current.lease.matches_proof(proof) {
2170 return Err(ClusterCheckpointAuthorityError::Fenced);
2171 }
2172 let fault_inventory = Self::recovery_fault_inventory_from(current);
2173 if fault_inventory.revision != terminal.round.fault_revision()
2174 || fault_inventory.faults != terminal.round.faults
2175 {
2176 return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2177 }
2178
2179 let base_sequence = current.lease.seq;
2180 let sequence = base_sequence
2181 .checked_add(1)
2182 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2183 let mut candidate = current.preserve_with_lease(LeaderLease {
2184 seq: sequence,
2185 renewal_sequence: current.lease.renewal_sequence,
2186 token: current.lease.token,
2187 owner: current.lease.owner.clone(),
2188 expires_at_ms: current.lease.expires_at_ms,
2189 catalog_manifest: current.lease.catalog_manifest.clone(),
2190 });
2191 candidate.recovery_fault_slots.retain(|slot| {
2194 slot.active
2195 && terminal
2196 .round
2197 .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2198 == Some(slot.publisher.participant.boot_incarnation)
2199 });
2200 for slot in &mut candidate.recovery_fault_slots {
2201 slot.active = false;
2202 }
2203 candidate.recovery_fault_revision = sequence;
2204 candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2205 terminal: reference.clone(),
2206 leader_proof: proof.clone(),
2207 });
2208 candidate.recovery_release_head = Some(RecoveryReleaseLink {
2209 sequence,
2210 terminal: reference.clone(),
2211 });
2212 candidate.validate()?;
2213
2214 match self
2215 .create_authority_record(Some(&published), &candidate)
2216 .await?
2217 {
2218 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2219 return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2220 }
2221 AuthorityCreateOutcome::Contended(winner_head) => {
2222 if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2223 if winner.terminal == reference
2224 || winner.terminal.generation() >= reference.generation()
2225 {
2226 self.load_recovery_release_terminal(&winner.terminal)
2227 .await?;
2228 return if winner.terminal == reference {
2229 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2230 } else {
2231 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2232 winner: winner.terminal.clone(),
2233 })
2234 };
2235 }
2236 }
2237 if !winner_head.lease.matches_proof(proof) {
2238 return Err(ClusterCheckpointAuthorityError::Fenced);
2239 }
2240 if winner_head.lease.seq <= base_sequence {
2241 return Err(LeaseError::Invalid(
2242 "recovery release authority contention did not advance the sequence"
2243 .into(),
2244 )
2245 .into());
2246 }
2247 tokio::task::yield_now().await;
2248 }
2249 }
2250 }
2251 }
2252
2253 async fn recovery_release_terminal_from(
2254 &self,
2255 head: &LeaderAuthorityRecord,
2256 link: &RecoveryReleaseLink,
2257 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2258 let admission = if link.sequence == head.lease.seq {
2259 head.clone()
2260 } else {
2261 read_authority_record(self.store.as_ref(), link.sequence)
2262 .await?
2263 .ok_or_else(|| {
2264 LeaseError::Invalid(
2265 "recovery release admission is missing from retained authority history"
2266 .into(),
2267 )
2268 })?
2269 };
2270 let commit = admission
2271 .recovery_release_commit
2272 .as_ref()
2273 .filter(|commit| {
2274 commit.terminal == link.terminal
2275 && admission.recovery_release_head.as_ref() == Some(link)
2276 })
2277 .ok_or_else(|| {
2278 LeaseError::Invalid(
2279 "recovery release head does not match its admitting authority record".into(),
2280 )
2281 })?;
2282 let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2283 if terminal.round.leader_proof != commit.leader_proof {
2284 return Err(LeaseError::Invalid(
2285 "recovery release terminal does not match its admitting leader proof".into(),
2286 )
2287 .into());
2288 }
2289 Ok(terminal)
2290 }
2291
2292 pub(crate) async fn recovery_admission_snapshot(
2293 &self,
2294 ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2295 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2296 let head = self
2297 .load_record()
2298 .await?
2299 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2300 let Some(link) = head.recovery_release_head.clone() else {
2301 return Ok(RecoveryAdmissionSnapshot {
2302 committed_release: None,
2303 fault_inventory: Self::recovery_fault_inventory_from(&head),
2304 authority_sequence: head.lease.seq,
2305 release_head: None,
2306 });
2307 };
2308 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2309 let rechecked = self
2310 .load_record()
2311 .await?
2312 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2313 if rechecked.lease.seq >= head.lease.seq
2314 && rechecked.recovery_release_head == head.recovery_release_head
2315 && rechecked.recovery_fault_revision == head.recovery_fault_revision
2316 && rechecked.recovery_fault_slots == head.recovery_fault_slots
2317 {
2318 return Ok(RecoveryAdmissionSnapshot {
2319 committed_release: Some(terminal?),
2320 fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2321 authority_sequence: rechecked.lease.seq,
2322 release_head: rechecked.recovery_release_head,
2323 });
2324 }
2325 tokio::task::yield_now().await;
2326 }
2327 Err(LeaseError::Io(format!(
2328 "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2329 ))
2330 .into())
2331 }
2332
2333 pub(crate) async fn recovery_admission_is_current(
2334 &self,
2335 snapshot: &RecoveryAdmissionSnapshot,
2336 leader_proof: &LeaderProof,
2337 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2338 let current = self
2339 .load_record()
2340 .await?
2341 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2342 let inventory = Self::recovery_fault_inventory_from(¤t);
2343 Ok(current.lease.matches_proof(leader_proof)
2344 && current.lease.seq >= snapshot.authority_sequence
2345 && current.recovery_release_head == snapshot.release_head
2346 && inventory == snapshot.fault_inventory
2347 && inventory.faults().is_empty())
2348 }
2349
2350 pub(crate) async fn latest_recovery_release_terminal(
2351 &self,
2352 ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2353 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2354 let Some(head) = self.load_record().await? else {
2355 return Ok(None);
2356 };
2357 let Some(link) = head.recovery_release_head.clone() else {
2358 return Ok(None);
2359 };
2360 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2361 let rechecked = self
2362 .load_record()
2363 .await?
2364 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2365 if rechecked.recovery_release_head.as_ref() == Some(&link) {
2366 return Ok(Some(terminal?));
2367 }
2368 tokio::task::yield_now().await;
2369 }
2370 Err(LeaseError::Io(format!(
2371 "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2372 ))
2373 .into())
2374 }
2375
2376 fn ttl(&self) -> Result<Duration, LeaseError> {
2377 let ttl = u64::try_from(self.ttl_ms)
2378 .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2379 if ttl == 0 {
2380 return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2381 }
2382 Ok(Duration::from_millis(ttl))
2383 }
2384
2385 fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2386 now_ms
2387 .checked_add(self.ttl_ms)
2388 .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2389 }
2390
2391 #[cfg(test)]
2392 async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2393 let prefix = OsPath::from(LEASE_PREFIX);
2394 let mut entries = self.store.list(Some(&prefix));
2395 let mut sequences = Vec::new();
2396 while let Some(entry) = entries.next().await {
2397 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2398 if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2399 return Err(LeaseError::Invalid(format!(
2400 "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2401 )));
2402 }
2403 sequences.push(lease_sequence_from_path(&entry.location)?);
2404 }
2405 sequences.sort_unstable();
2406 sequences.dedup();
2407 Ok(sequences)
2408 }
2409
2410 fn schedule_history_prune(&self) {
2411 if self
2412 .prune_running
2413 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2414 .is_err()
2415 {
2416 return;
2417 }
2418 let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2419 self.prune_running.store(false, Ordering::Release);
2420 return;
2421 };
2422 let store = Arc::clone(&self.store);
2423 let prune_running = Arc::clone(&self.prune_running);
2424 let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2425 runtime.spawn(async move {
2426 let _latch = PruneLatchGuard(prune_running);
2427 match tokio::time::timeout(
2428 LEADER_LEASE_PRUNE_TIMEOUT,
2429 Self::prune_history(&store, grace_ms),
2430 )
2431 .await
2432 {
2433 Ok(Ok(())) => {}
2434 Ok(Err(error)) => {
2435 tracing::warn!(%error, "leader lease history prune failed");
2436 }
2437 Err(_) => {
2438 tracing::warn!(
2439 timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2440 "leader lease history prune timed out"
2441 );
2442 }
2443 }
2444 });
2445 }
2446
2447 async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2448 let authority = Self::new(Arc::clone(store), 1);
2449 let Some(head) = authority.load_record().await? else {
2450 return Ok(());
2451 };
2452 let head_sequence = head.lease.seq;
2453 let mut retained = BTreeSet::from([head_sequence]);
2454 if let Some(previous) = head_sequence
2455 .checked_sub(1)
2456 .filter(|sequence| *sequence != 0)
2457 {
2458 retained.insert(previous);
2459 }
2460 let floor = head
2461 .outcome_floor
2462 .as_ref()
2463 .map_or(0, |floor| floor.authority_before_epoch);
2464 let artifact_floor = head
2465 .outcome_floor
2466 .as_ref()
2467 .map_or(0, |floor| floor.artifact_before_epoch);
2468 let mut link = head.outcome_head;
2469 let mut outcome_links = 0;
2470 let mut terminal_commit_links = BTreeSet::new();
2471 let mut expected_commit_head = head.commit_head;
2472 while let Some(current) = link {
2473 if current.epoch < floor {
2474 if head
2475 .outcome_floor
2476 .as_ref()
2477 .and_then(|floor| floor.terminal_anchor_link)
2478 != Some(current)
2479 {
2480 return Err(LeaseError::Invalid(
2481 "outcome floor does not anchor the retained authority chain".into(),
2482 ));
2483 }
2484 break;
2485 }
2486 if !consume_live_authority_link(&mut outcome_links) {
2487 return Err(LeaseError::Invalid(format!(
2488 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2489 )));
2490 }
2491 let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2492 .await?
2493 .filter(|record| {
2494 record.outcome_head == Some(current)
2495 && record
2496 .checkpoint_outcome
2497 .as_ref()
2498 .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2499 == Some((current.epoch, current.checkpoint_id))
2500 })
2501 .ok_or_else(|| {
2502 LeaseError::Invalid("retained outcome authority chain is broken".into())
2503 })?;
2504 if outcome_record.commit_head != expected_commit_head {
2505 return Err(LeaseError::Invalid(
2506 "retained outcome chain did not preserve the exact Commit head".into(),
2507 ));
2508 }
2509 retained.insert(current.sequence);
2510 if outcome_record
2511 .checkpoint_outcome
2512 .as_ref()
2513 .is_some_and(CheckpointOutcome::is_commit)
2514 {
2515 terminal_commit_links.insert(current);
2516 expected_commit_head = outcome_record.previous_commit;
2517 } else {
2518 expected_commit_head = outcome_record.commit_head;
2519 }
2520 link = outcome_record.previous_outcome;
2521 }
2522 let mut commit_link = head.commit_head;
2523 let mut commit_links = 0;
2524 let mut retained_commit_links = BTreeSet::new();
2525 while let Some(current) = commit_link {
2526 if current.epoch < artifact_floor {
2527 break;
2528 }
2529 if !consume_live_authority_link(&mut commit_links) {
2530 return Err(LeaseError::Invalid(format!(
2531 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2532 )));
2533 }
2534 let commit_record = read_authority_record(store.as_ref(), current.sequence)
2535 .await?
2536 .filter(|record| {
2537 record.commit_head == Some(current)
2538 && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2539 outcome.is_commit()
2540 && outcome.epoch == current.epoch
2541 && outcome.checkpoint_id == current.checkpoint_id
2542 })
2543 })
2544 .ok_or_else(|| {
2545 LeaseError::Invalid("retained Commit authority chain is broken".into())
2546 })?;
2547 retained.insert(current.sequence);
2548 retained_commit_links.insert(current);
2549 commit_link = commit_record.previous_commit;
2550 }
2551 match (
2552 commit_link,
2553 head.outcome_floor
2554 .as_ref()
2555 .and_then(|floor| floor.committed_anchor_link),
2556 head.outcome_floor
2557 .as_ref()
2558 .and_then(|floor| floor.committed_anchor.as_ref()),
2559 ) {
2560 (Some(link), Some(anchor_link), Some(anchor))
2561 if link == anchor_link
2562 && link.epoch == anchor.epoch
2563 && link.checkpoint_id == anchor.checkpoint_id => {}
2564 (None, None, None) => {}
2565 _ => {
2566 return Err(LeaseError::Invalid(
2567 "Commit chain does not meet the retained artifact-floor anchor".into(),
2568 ));
2569 }
2570 }
2571 let boundary_commit_head = head
2572 .outcome_floor
2573 .as_ref()
2574 .and_then(|floor| floor.terminal_anchor.as_ref())
2575 .and_then(|anchor| {
2576 retained_commit_links
2577 .iter()
2578 .rev()
2579 .find(|link| link.epoch <= anchor.epoch)
2580 .copied()
2581 .or_else(|| {
2582 head.outcome_floor
2583 .as_ref()
2584 .and_then(|floor| floor.committed_anchor_link)
2585 })
2586 });
2587 if expected_commit_head != boundary_commit_head {
2588 return Err(LeaseError::Invalid(
2589 "retained terminal chain lost Commit continuity at its durable floor".into(),
2590 ));
2591 }
2592 if !terminal_commit_links.is_subset(&retained_commit_links) {
2593 return Err(LeaseError::Invalid(
2594 "terminal Commit records are not linked from the retained Commit chain".into(),
2595 ));
2596 }
2597 if let Some((anchor, anchor_link)) = head.outcome_floor.as_ref().and_then(|floor| {
2598 floor
2599 .terminal_anchor
2600 .as_ref()
2601 .zip(floor.terminal_anchor_link)
2602 }) {
2603 if anchor.is_commit() {
2604 let linked = if anchor.epoch >= artifact_floor {
2605 retained_commit_links.contains(&anchor_link)
2606 } else {
2607 head.outcome_floor.as_ref().is_some_and(|floor| {
2608 floor.committed_anchor.as_ref() == Some(anchor)
2609 && floor.committed_anchor_link == Some(anchor_link)
2610 })
2611 };
2612 if !linked {
2613 return Err(LeaseError::Invalid(
2614 "terminal Commit anchor is not linked from the retained Commit chain"
2615 .into(),
2616 ));
2617 }
2618 }
2619 }
2620 let mut assignment_link = head.assignment_decision_head;
2621 let mut assignment_links = 0;
2622 while let Some(current) = assignment_link {
2623 if !consume_live_authority_link(&mut assignment_links) {
2624 return Err(LeaseError::Invalid(format!(
2625 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2626 )));
2627 }
2628 if head
2629 .assignment_decision_floor
2630 .as_ref()
2631 .is_some_and(|floor| current.target_version < floor.before_target_version)
2632 {
2633 if head
2634 .assignment_decision_floor
2635 .as_ref()
2636 .and_then(|floor| floor.terminal_anchor_link)
2637 != Some(current)
2638 {
2639 return Err(LeaseError::Invalid(
2640 "assignment-decision floor does not anchor the retained authority chain"
2641 .into(),
2642 ));
2643 }
2644 break;
2645 }
2646 let decision_record = read_authority_record(store.as_ref(), current.sequence)
2647 .await?
2648 .filter(|record| {
2649 record.assignment_decision_head == Some(current)
2650 && record
2651 .assignment_decision
2652 .as_ref()
2653 .map(AuthorityAssignmentDecision::target_version)
2654 == Some(current.target_version)
2655 })
2656 .ok_or_else(|| {
2657 LeaseError::Invalid(
2658 "retained assignment-decision authority chain is broken".into(),
2659 )
2660 })?;
2661 retained.insert(current.sequence);
2662 assignment_link = decision_record.previous_assignment_decision;
2663 }
2664 let retained_release = match head.recovery_release_head.as_ref() {
2665 Some(link) => {
2666 let admission = read_authority_record(store.as_ref(), link.sequence)
2667 .await?
2668 .filter(|record| {
2669 record.recovery_release_head.as_ref() == Some(link)
2670 && record
2671 .recovery_release_commit
2672 .as_ref()
2673 .is_some_and(|commit| commit.terminal == link.terminal)
2674 })
2675 .ok_or_else(|| {
2676 LeaseError::Invalid(
2677 "retained recovery release authority link is broken".into(),
2678 )
2679 })?;
2680 if admission.lease.seq != link.sequence {
2681 return Err(LeaseError::Invalid(
2682 "recovery release admission sequence does not match its retained link"
2683 .into(),
2684 ));
2685 }
2686 let terminal = authority
2687 .load_recovery_release_terminal(&link.terminal)
2688 .await
2689 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2690 if admission
2691 .recovery_release_commit
2692 .as_ref()
2693 .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2694 {
2695 return Err(LeaseError::Invalid(
2696 "retained recovery release terminal does not match its admission".into(),
2697 ));
2698 }
2699 retained.insert(link.sequence);
2700 Some(link.terminal.clone())
2701 }
2702 None => None,
2703 };
2704
2705 let mut history_exhausted = false;
2706 for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2707 let (candidates, exhausted) =
2708 Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2709 if candidates.is_empty() {
2710 history_exhausted = true;
2711 break;
2712 }
2713 let deletions =
2714 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2715 let mut results = store.delete_stream(Box::pin(deletions));
2716 while let Some(result) = results.next().await {
2717 if let Err(error) = result {
2718 if !matches!(error, object_store::Error::NotFound { .. }) {
2719 return Err(LeaseError::Io(error.to_string()));
2720 }
2721 }
2722 }
2723 if exhausted {
2724 history_exhausted = true;
2725 break;
2726 }
2727 tokio::task::yield_now().await;
2728 }
2729 if !history_exhausted {
2730 return Err(LeaseError::Io(
2731 "leader lease history still exceeds the bounded prune budget".into(),
2732 ));
2733 }
2734 Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await
2735 }
2736
2737 async fn prune_recovery_release_terminals(
2738 store: &Arc<dyn ObjectStore>,
2739 retained: Option<&RecoveryReleaseTerminalRef>,
2740 grace_ms: i64,
2741 ) -> Result<(), LeaseError> {
2742 let Some(retained) = retained else {
2743 return Ok(());
2744 };
2745 retained.validate()?;
2746 for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
2747 let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
2748 let mut listed = store.list(Some(&prefix));
2749 let now = now_millis();
2750 let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
2751 let mut exhausted = true;
2752 while let Some(entry) = listed.next().await {
2753 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2754 let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
2755 let canonical = OsPath::from(format!(
2756 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
2757 ));
2758 if canonical != entry.location {
2759 return Err(LeaseError::Invalid(format!(
2760 "noncanonical recovery release terminal path {}",
2761 entry.location
2762 )));
2763 }
2764 if entry.location == recovery_release_terminal_path(retained)
2765 || generation > retained.generation()
2766 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2767 {
2768 continue;
2769 }
2770 candidates.push(entry.location);
2771 if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
2772 exhausted = false;
2773 break;
2774 }
2775 }
2776 if candidates.is_empty() {
2777 return Ok(());
2778 }
2779 let deletions =
2780 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2781 let mut results = store.delete_stream(Box::pin(deletions));
2782 while let Some(result) = results.next().await {
2783 if let Err(error) = result {
2784 if !matches!(error, object_store::Error::NotFound { .. }) {
2785 return Err(LeaseError::Io(error.to_string()));
2786 }
2787 }
2788 }
2789 if exhausted {
2790 return Ok(());
2791 }
2792 tokio::task::yield_now().await;
2793 }
2794 Ok(())
2795 }
2796
2797 async fn prune_candidates(
2798 store: &Arc<dyn ObjectStore>,
2799 retained: &BTreeSet<u64>,
2800 snapshot_head_sequence: u64,
2801 grace_ms: i64,
2802 ) -> Result<(Vec<OsPath>, bool), LeaseError> {
2803 let prefix = OsPath::from(LEASE_PREFIX);
2804 let mut listed = store.list(Some(&prefix));
2805 let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
2806 let now = now_millis();
2807 let mut exhausted = true;
2808 while let Some(entry) = listed.next().await {
2809 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2810 let sequence = lease_sequence_from_path(&entry.location)?;
2811 if sequence >= snapshot_head_sequence
2812 || retained.contains(&sequence)
2813 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2814 {
2815 continue;
2816 }
2817 candidates.push(entry.location);
2818 if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
2819 exhausted = false;
2820 break;
2821 }
2822 }
2823 Ok((candidates, exhausted))
2824 }
2825
2826 pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
2831 Ok(self.load_record().await?.map(|record| record.lease))
2832 }
2833
2834 async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
2835 Ok(self
2836 .load_published_authority_head()
2837 .await?
2838 .map(|head| head.record))
2839 }
2840
2841 async fn load_published_authority_head(
2842 &self,
2843 ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
2844 for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2845 let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
2846 let Some(discovered) = self.discover_authority_head().await? else {
2847 if read_authority_head_pointer(self.store.as_ref())
2848 .await?
2849 .is_none()
2850 {
2851 return Ok(None);
2852 }
2853 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2854 tokio::task::yield_now().await;
2855 continue;
2856 }
2857 break;
2858 };
2859 self.publish_authority_head(discovered.lease.seq, None)
2860 .await?;
2861 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2862 tokio::task::yield_now().await;
2863 continue;
2864 }
2865 break;
2866 };
2867
2868 let sequence = pointer.pointer.sequence;
2869 let successor_sequence = sequence.checked_add(1);
2870 let (record, successor) = if let Some(successor_sequence) = successor_sequence {
2871 tokio::try_join!(
2872 read_authority_record(self.store.as_ref(), sequence),
2873 read_authority_record(self.store.as_ref(), successor_sequence)
2874 )?
2875 } else {
2876 (
2877 read_authority_record(self.store.as_ref(), sequence).await?,
2878 None,
2879 )
2880 };
2881 let Some(record) = record else {
2882 let rechecked = read_authority_head_pointer(self.store.as_ref())
2883 .await?
2884 .ok_or_else(|| {
2885 LeaseError::Invalid(
2886 "leader authority head disappeared while reading its target".into(),
2887 )
2888 })?;
2889 if rechecked.pointer.sequence > sequence {
2890 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2891 tokio::task::yield_now().await;
2892 continue;
2893 }
2894 break;
2895 }
2896 if rechecked.pointer.sequence < sequence {
2897 return Err(LeaseError::Invalid(format!(
2898 "leader authority head regressed from sequence {sequence} to {}",
2899 rechecked.pointer.sequence
2900 )));
2901 }
2902 return Err(LeaseError::Invalid(format!(
2903 "leader authority head points ahead to missing sequence {sequence}"
2904 )));
2905 };
2906 let Some(successor_sequence) = successor_sequence else {
2907 return Ok(Some(PublishedAuthorityHead { record, pointer }));
2908 };
2909 if successor.is_none() {
2910 return Ok(Some(PublishedAuthorityHead { record, pointer }));
2911 }
2912
2913 if let Some(after_successor) = successor_sequence.checked_add(1) {
2914 if read_authority_record(self.store.as_ref(), after_successor)
2915 .await?
2916 .is_some()
2917 {
2918 let rechecked = read_authority_head_pointer(self.store.as_ref())
2919 .await?
2920 .ok_or_else(|| {
2921 LeaseError::Invalid(
2922 "leader authority head disappeared while checking pointer lag"
2923 .into(),
2924 )
2925 })?;
2926 if rechecked.pointer.sequence == sequence {
2927 return Err(LeaseError::Invalid(format!(
2928 "leader authority head at sequence {sequence} lags by more than one record"
2929 )));
2930 }
2931 if rechecked.pointer.sequence < sequence {
2932 return Err(LeaseError::Invalid(format!(
2933 "leader authority head regressed from sequence {sequence} to {}",
2934 rechecked.pointer.sequence
2935 )));
2936 }
2937 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2938 tokio::task::yield_now().await;
2939 continue;
2940 }
2941 break;
2942 }
2943 }
2944
2945 self.publish_authority_head(successor_sequence, Some(&pointer))
2946 .await?;
2947 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2948 tokio::task::yield_now().await;
2949 continue;
2950 }
2951 break;
2952 }
2953 Err(LeaseError::Io(format!(
2954 "leader authority head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2955 )))
2956 }
2957
2958 async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
2959 let prefix = OsPath::from(LEASE_PREFIX);
2960 let mut listed = self.store.list(Some(&prefix));
2961 let mut discovered = 0usize;
2962 let mut maximum = None;
2963 while let Some(entry) = listed.next().await {
2964 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2965 discovered = discovered.checked_add(1).ok_or_else(|| {
2966 LeaseError::Invalid("leader authority discovery count overflowed".into())
2967 })?;
2968 if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
2969 return Err(LeaseError::Invalid(format!(
2970 "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
2971 )));
2972 }
2973 let sequence = lease_sequence_from_path(&entry.location)?;
2974 maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
2975 }
2976 let Some(sequence) = maximum else {
2977 return Ok(None);
2978 };
2979 read_authority_record(self.store.as_ref(), sequence)
2980 .await?
2981 .map(Some)
2982 .ok_or_else(|| {
2983 LeaseError::Io(format!(
2984 "discovered leader authority sequence {sequence} vanished before publication"
2985 ))
2986 })
2987 }
2988
2989 async fn publish_authority_head(
2990 &self,
2991 sequence: u64,
2992 expected: Option<&VersionedAuthorityHeadPointer>,
2993 ) -> Result<u64, LeaseError> {
2994 if let Some(expected) = expected {
2995 if expected.pointer.sequence.checked_add(1) != Some(sequence) {
2996 return Err(LeaseError::Invalid(
2997 "leader authority head update is not the exact successor".into(),
2998 ));
2999 }
3000 if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3001 {
3002 return Err(LeaseError::Invalid(
3003 "leader authority store did not provide a native conditional update version"
3004 .into(),
3005 ));
3006 }
3007 }
3008
3009 let options = PutOptions {
3010 mode: expected.map_or(PutMode::Create, |expected| {
3011 PutMode::Update(expected.update_version.clone())
3012 }),
3013 ..PutOptions::default()
3014 };
3015 let result = self
3016 .store
3017 .put_opts(
3018 &authority_head_path(),
3019 PutPayload::from(encode_authority_head_pointer(sequence)?),
3020 options,
3021 )
3022 .await;
3023 if result.is_ok() {
3024 return Ok(sequence);
3025 }
3026 let write_error = result
3027 .expect_err("successful authority head writes return above")
3028 .to_string();
3029 let current = read_authority_head_pointer(self.store.as_ref())
3030 .await?
3031 .ok_or_else(|| {
3032 LeaseError::Io(format!(
3033 "leader authority head write failed ({write_error}) and no pointer was durable"
3034 ))
3035 })?;
3036 if let Some(expected) = expected {
3037 if current.pointer.sequence < expected.pointer.sequence {
3038 return Err(LeaseError::Invalid(format!(
3039 "leader authority head regressed from sequence {} to {}",
3040 expected.pointer.sequence, current.pointer.sequence
3041 )));
3042 }
3043 }
3044 if current.pointer.sequence >= sequence {
3045 if current.pointer.sequence > sequence {
3046 read_authority_record(self.store.as_ref(), current.pointer.sequence)
3047 .await?
3048 .ok_or_else(|| {
3049 LeaseError::Invalid(format!(
3050 "leader authority head points ahead to missing sequence {}",
3051 current.pointer.sequence
3052 ))
3053 })?;
3054 }
3055 return Ok(current.pointer.sequence);
3056 }
3057 Err(LeaseError::Io(format!(
3058 "leader authority head write failed ({write_error}) and remained at sequence {}",
3059 current.pointer.sequence
3060 )))
3061 }
3062
3063 async fn read_catalog_manifest_blob(
3064 &self,
3065 reference: &CatalogManifestRef,
3066 ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3067 reference.validate()?;
3068 let path = reference.object_path();
3069 let result = match self.store.get(&path).await {
3070 Ok(result) => result,
3071 Err(object_store::Error::NotFound { .. }) => {
3072 return Err(CatalogManifestError::Invalid(format!(
3073 "catalog manifest blob '{}' is missing",
3074 reference.sha256
3075 )));
3076 }
3077 Err(error) => {
3078 return Err(CatalogManifestError::Authority(LeaseError::Io(
3079 error.to_string(),
3080 )));
3081 }
3082 };
3083 if result.meta.size != reference.encoded_len {
3084 return Err(CatalogManifestError::Invalid(format!(
3085 "catalog manifest blob '{}' is {} bytes, expected {}",
3086 reference.sha256, result.meta.size, reference.encoded_len
3087 )));
3088 }
3089 let bytes = result
3090 .bytes()
3091 .await
3092 .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3093 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3094 return Err(CatalogManifestError::Invalid(format!(
3095 "catalog manifest blob '{}' payload length changed while reading",
3096 reference.sha256
3097 )));
3098 }
3099 let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3100 let (canonical, actual_reference) = manifest.encode_and_reference()?;
3101 if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3102 return Err(CatalogManifestError::Invalid(format!(
3103 "catalog manifest blob '{}' does not match its sealed reference",
3104 reference.sha256
3105 )));
3106 }
3107 Ok((manifest, bytes))
3108 }
3109
3110 pub(super) async fn load_catalog_manifest(
3111 &self,
3112 reference: &CatalogManifestRef,
3113 ) -> Result<CatalogManifest, CatalogManifestError> {
3114 self.read_catalog_manifest_blob(reference)
3115 .await
3116 .map(|(manifest, _)| manifest)
3117 }
3118
3119 async fn ensure_catalog_manifest_blob(
3120 &self,
3121 encoded: &[u8],
3122 reference: &CatalogManifestRef,
3123 ) -> Result<(), CatalogManifestError> {
3124 reference.validate()?;
3125 let options = PutOptions {
3126 mode: PutMode::Create,
3127 ..PutOptions::default()
3128 };
3129 let path = reference.object_path();
3130 let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3131 let put_error = match self.store.put_opts(&path, payload, options).await {
3132 Ok(_)
3133 | Err(
3134 object_store::Error::AlreadyExists { .. }
3135 | object_store::Error::Precondition { .. },
3136 ) => None,
3137 Err(error) => Some(error),
3138 };
3139
3140 match self.read_catalog_manifest_blob(reference).await {
3141 Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3142 Ok(_) => Err(CatalogManifestError::Invalid(format!(
3143 "catalog manifest blob '{}' differs from the proposed content",
3144 reference.sha256
3145 ))),
3146 Err(error) => {
3147 if let Some(put_error) = put_error {
3148 Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3149 "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3150 ))))
3151 } else {
3152 Err(error)
3153 }
3154 }
3155 }
3156 }
3157
3158 async fn create_authority_record(
3159 &self,
3160 expected: Option<&PublishedAuthorityHead>,
3161 candidate: &LeaderAuthorityRecord,
3162 ) -> Result<AuthorityCreateOutcome, LeaseError> {
3163 let encoded = encode_authority_record(candidate)?;
3164 let expected_pointer = match expected {
3165 None => {
3166 if candidate.lease.seq != 1 {
3167 return Err(LeaseError::Invalid(format!(
3168 "cannot append authority sequence {} to an empty published namespace",
3169 candidate.lease.seq
3170 )));
3171 }
3172 None
3173 }
3174 Some(head) if head.record.lease.seq == candidate.lease.seq => {
3175 return if head.record == *candidate {
3176 Ok(AuthorityCreateOutcome::ExistingIdentical)
3177 } else {
3178 Ok(AuthorityCreateOutcome::Contended(Box::new(
3179 head.record.clone(),
3180 )))
3181 };
3182 }
3183 Some(head) if head.record.lease.seq > candidate.lease.seq => {
3184 return Ok(AuthorityCreateOutcome::Contended(Box::new(
3185 head.record.clone(),
3186 )));
3187 }
3188 Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3189 Some(&head.pointer)
3190 }
3191 Some(head) => {
3192 return Err(LeaseError::Invalid(format!(
3193 "cannot append authority sequence {} after published sequence {}",
3194 candidate.lease.seq, head.record.lease.seq
3195 )));
3196 }
3197 };
3198 let options = PutOptions {
3199 mode: PutMode::Create,
3200 ..PutOptions::default()
3201 };
3202 let result = self
3203 .store
3204 .put_opts(
3205 &lease_path(candidate.lease.seq),
3206 PutPayload::from(encoded),
3207 options,
3208 )
3209 .await;
3210 let created = result.is_ok();
3211 if let Err(error) = result {
3212 let Some(at_sequence) =
3213 read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3214 else {
3215 return Err(LeaseError::Io(format!(
3216 "authority record {} write failed ({error}) and no record was durable",
3217 candidate.lease.seq
3218 )));
3219 };
3220 if at_sequence != *candidate {
3221 let winner = self.load_record().await?.ok_or_else(|| {
3222 LeaseError::Io("authority contender was not published or readable".into())
3223 })?;
3224 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3225 }
3226 }
3227
3228 let published_sequence = self
3229 .publish_authority_head(candidate.lease.seq, expected_pointer)
3230 .await?;
3231 if published_sequence > candidate.lease.seq {
3232 if created {
3233 self.schedule_history_prune();
3234 }
3235 let winner = self
3236 .load_record()
3237 .await?
3238 .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3239 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3240 }
3241 self.schedule_history_prune();
3242 if created {
3243 Ok(AuthorityCreateOutcome::Created)
3244 } else {
3245 Ok(AuthorityCreateOutcome::ExistingIdentical)
3246 }
3247 }
3248
3249 pub(super) async fn seal_catalog(
3258 &self,
3259 proof: &LeaderProof,
3260 manifest: &CatalogManifest,
3261 ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3262 if !proof.is_canonical() {
3263 return Err(CatalogManifestError::Fenced);
3264 }
3265 let (encoded, reference) = manifest.encode_and_reference()?;
3266
3267 let current = self
3268 .load_record()
3269 .await?
3270 .ok_or(CatalogManifestError::Fenced)?;
3271 if let Some(sealed) = ¤t.lease.catalog_manifest {
3272 let durable = self.load_catalog_manifest(sealed).await?;
3273 return if durable == *manifest {
3274 Ok(CatalogSealOutcome::ExistingIdentical)
3275 } else {
3276 Err(CatalogManifestError::Conflict)
3277 };
3278 }
3279 if !current.lease.matches_proof(proof) {
3280 return Err(CatalogManifestError::Fenced);
3281 }
3282 self.ensure_catalog_manifest_blob(&encoded, &reference)
3283 .await?;
3284
3285 loop {
3286 let published = self
3287 .load_published_authority_head()
3288 .await?
3289 .ok_or(CatalogManifestError::Fenced)?;
3290 let current = &published.record;
3291 if let Some(sealed) = ¤t.lease.catalog_manifest {
3292 let durable = self.load_catalog_manifest(sealed).await?;
3293 return if durable == *manifest {
3294 Ok(CatalogSealOutcome::ExistingIdentical)
3295 } else {
3296 Err(CatalogManifestError::Conflict)
3297 };
3298 }
3299 if !current.lease.matches_proof(proof) {
3300 return Err(CatalogManifestError::Fenced);
3301 }
3302
3303 let base_sequence = current.lease.seq;
3304 let candidate_lease = LeaderLease {
3305 seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3306 CatalogManifestError::Authority(LeaseError::Invalid(
3307 "lease sequence exhausted".into(),
3308 ))
3309 })?,
3310 renewal_sequence: current.lease.renewal_sequence,
3311 token: current.lease.token,
3312 owner: current.lease.owner.clone(),
3313 expires_at_ms: current.lease.expires_at_ms,
3314 catalog_manifest: Some(reference.clone()),
3315 };
3316 let candidate = current.preserve_with_lease(candidate_lease);
3317 match self
3318 .create_authority_record(Some(&published), &candidate)
3319 .await?
3320 {
3321 AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3322 AuthorityCreateOutcome::ExistingIdentical => {
3323 return Ok(CatalogSealOutcome::ExistingIdentical);
3324 }
3325 AuthorityCreateOutcome::Contended(winner) => {
3326 if let Some(sealed) = &winner.lease.catalog_manifest {
3327 let durable = self.load_catalog_manifest(sealed).await?;
3328 return if durable == *manifest {
3329 Ok(CatalogSealOutcome::ExistingIdentical)
3330 } else {
3331 Err(CatalogManifestError::Conflict)
3332 };
3333 }
3334 if !winner.lease.matches_proof(proof) {
3335 return Err(CatalogManifestError::Fenced);
3336 }
3337 if winner.lease.seq <= base_sequence {
3338 return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3339 "catalog seal contention did not advance the authority sequence".into(),
3340 )));
3341 }
3342 tokio::task::yield_now().await;
3343 }
3344 }
3345 }
3346 }
3347
3348 async fn audited_cluster_outcomes_from(
3349 &self,
3350 head: &LeaderAuthorityRecord,
3351 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3352 let floor = head.outcome_floor.as_ref();
3353 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3354 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3355 let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3356 let mut terminal_newest_first = Vec::new();
3357 let mut terminal_links = Vec::new();
3358 let mut link = head.outcome_head;
3359 let mut expected_commit_head = head.commit_head;
3360 let mut traversed = 0;
3361 while let Some(current) = link {
3362 if current.epoch < authority_before_epoch {
3363 break;
3364 }
3365 if !consume_live_authority_link(&mut traversed) {
3366 return Err(DecisionError::Conflict(format!(
3367 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3368 ))
3369 .into());
3370 }
3371 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3372 record.clone()
3373 } else {
3374 let record = read_authority_record(self.store.as_ref(), current.sequence)
3375 .await?
3376 .ok_or_else(|| {
3377 DecisionError::InventoryChanged(format!(
3378 "cluster outcome authority record {} disappeared during audit",
3379 current.sequence
3380 ))
3381 })?;
3382 terminal_records.insert(current.sequence, record.clone());
3383 record
3384 };
3385 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3386 DecisionError::Conflict(format!(
3387 "cluster outcome head epoch {} points to non-outcome authority record {}",
3388 current.epoch, current.sequence
3389 ))
3390 })?;
3391 if record.outcome_head != Some(current)
3392 || outcome.epoch != current.epoch
3393 || outcome.checkpoint_id != current.checkpoint_id
3394 {
3395 return Err(DecisionError::Conflict(format!(
3396 "cluster outcome link epoch {} sequence {} does not match its authority record",
3397 current.epoch, current.sequence
3398 ))
3399 .into());
3400 }
3401 if record.commit_head != expected_commit_head {
3402 return Err(DecisionError::Conflict(format!(
3403 "cluster outcome epoch {} did not preserve the exact Commit head",
3404 outcome.epoch
3405 ))
3406 .into());
3407 }
3408 expected_commit_head = if outcome.is_commit() {
3409 record.previous_commit
3410 } else {
3411 record.commit_head
3412 };
3413 terminal_newest_first.push(outcome);
3414 terminal_links.push(current);
3415 link = record.previous_outcome;
3416 }
3417
3418 if let Some(floor) = floor {
3419 match (
3420 link,
3421 floor.terminal_anchor_link,
3422 floor.terminal_anchor.as_ref(),
3423 ) {
3424 (Some(link), Some(anchor_link), Some(anchor))
3425 if link == anchor_link
3426 && link.epoch == anchor.epoch
3427 && link.checkpoint_id == anchor.checkpoint_id => {}
3428 (None, None, None) => {}
3429 _ => {
3430 return Err(DecisionError::Conflict(format!(
3431 "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3432 floor.authority_before_epoch
3433 ))
3434 .into());
3435 }
3436 }
3437 } else if link.is_some() {
3438 return Err(DecisionError::Conflict(
3439 "cluster outcome chain stopped without a durable retention floor".into(),
3440 )
3441 .into());
3442 }
3443
3444 let mut commit_newest_first = Vec::new();
3445 let mut commit_links = Vec::new();
3446 link = head.commit_head;
3447 traversed = 0;
3448 while let Some(current) = link {
3449 if current.epoch < artifact_before_epoch {
3450 break;
3451 }
3452 if !consume_live_authority_link(&mut traversed) {
3453 return Err(DecisionError::Conflict(format!(
3454 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3455 ))
3456 .into());
3457 }
3458 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3459 record.clone()
3460 } else {
3461 read_authority_record(self.store.as_ref(), current.sequence)
3462 .await?
3463 .ok_or_else(|| {
3464 DecisionError::InventoryChanged(format!(
3465 "cluster Commit authority record {} disappeared during audit",
3466 current.sequence
3467 ))
3468 })?
3469 };
3470 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3471 DecisionError::Conflict(format!(
3472 "cluster Commit head epoch {} points to non-outcome authority record {}",
3473 current.epoch, current.sequence
3474 ))
3475 })?;
3476 if !outcome.is_commit()
3477 || record.commit_head != Some(current)
3478 || outcome.epoch != current.epoch
3479 || outcome.checkpoint_id != current.checkpoint_id
3480 {
3481 return Err(DecisionError::Conflict(format!(
3482 "cluster Commit link epoch {} sequence {} does not match its authority record",
3483 current.epoch, current.sequence
3484 ))
3485 .into());
3486 }
3487 commit_newest_first.push(outcome);
3488 commit_links.push(current);
3489 link = record.previous_commit;
3490 }
3491
3492 if let Some(floor) = floor {
3493 match (
3494 link,
3495 floor.committed_anchor_link,
3496 floor.committed_anchor.as_ref(),
3497 ) {
3498 (Some(link), Some(anchor_link), Some(anchor))
3499 if link == anchor_link
3500 && link.epoch == anchor.epoch
3501 && link.checkpoint_id == anchor.checkpoint_id => {}
3502 (None, None, None) => {}
3503 _ => {
3504 return Err(DecisionError::Conflict(format!(
3505 "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3506 floor.artifact_before_epoch
3507 ))
3508 .into());
3509 }
3510 }
3511 } else if link.is_some() {
3512 return Err(DecisionError::Conflict(
3513 "cluster Commit chain stopped without a durable retention floor".into(),
3514 )
3515 .into());
3516 }
3517
3518 terminal_newest_first.reverse();
3519 terminal_links.reverse();
3520 commit_newest_first.reverse();
3521 commit_links.reverse();
3522 let boundary_commit_head = floor
3523 .and_then(|floor| floor.terminal_anchor.as_ref())
3524 .and_then(|anchor| {
3525 commit_links
3526 .iter()
3527 .rev()
3528 .find(|link| link.epoch <= anchor.epoch)
3529 .copied()
3530 .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3531 });
3532 if expected_commit_head != boundary_commit_head {
3533 return Err(DecisionError::Conflict(
3534 "cluster terminal chain does not preserve Commit continuity at its durable floor"
3535 .into(),
3536 )
3537 .into());
3538 }
3539 for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3540 if !outcome.is_commit() {
3541 continue;
3542 }
3543 let commit_link = commit_links
3544 .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3545 .ok()
3546 .map(|index| commit_links[index]);
3547 if commit_link != Some(*terminal_link) {
3548 return Err(DecisionError::Conflict(format!(
3549 "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3550 outcome.epoch
3551 ))
3552 .into());
3553 }
3554 }
3555 if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3556 floor
3557 .terminal_anchor
3558 .as_ref()
3559 .zip(floor.terminal_anchor_link)
3560 }) {
3561 if terminal_anchor.is_commit() {
3562 let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3563 commit_links
3564 .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3565 .ok()
3566 .map(|index| commit_links[index])
3567 } else {
3568 floor.and_then(|floor| {
3569 (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3570 .then_some(floor.committed_anchor_link)
3571 .flatten()
3572 })
3573 };
3574 if commit_link != Some(terminal_anchor_link) {
3575 return Err(DecisionError::Conflict(format!(
3576 "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3577 terminal_anchor.epoch
3578 ))
3579 .into());
3580 }
3581 }
3582 }
3583 let mut outcomes = Vec::with_capacity(
3584 terminal_newest_first
3585 .len()
3586 .saturating_add(commit_newest_first.len())
3587 .saturating_add(2),
3588 );
3589 outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3590 outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3591 outcomes.extend(terminal_newest_first);
3592 outcomes.extend(commit_newest_first);
3593 outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3594 let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3595 for outcome in outcomes {
3596 if let Some(previous) = merged.last() {
3597 if previous.epoch == outcome.epoch {
3598 if previous != &outcome {
3599 return Err(DecisionError::Conflict(format!(
3600 "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3601 outcome.epoch
3602 ))
3603 .into());
3604 }
3605 continue;
3606 }
3607 }
3608 merged.push(outcome);
3609 }
3610
3611 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3612 .load_or_create_deployment_id()
3613 .await?;
3614 for outcome in &merged {
3615 if outcome.deployment_id != expected_deployment {
3616 return Err(DecisionError::Conflict(format!(
3617 "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3618 outcome.epoch, outcome.deployment_id, expected_deployment
3619 ))
3620 .into());
3621 }
3622 }
3623 for pair in merged.windows(2) {
3624 let previous = &pair[0];
3625 let current = &pair[1];
3626 if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3627 return Err(DecisionError::Conflict(format!(
3628 "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3629 previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3630 ))
3631 .into());
3632 }
3633 }
3634 Ok(ClusterOutcomeAuditSnapshot {
3635 outcomes: Arc::from(merged),
3636 terminal_links: Arc::from(terminal_links),
3637 commit_links: Arc::from(commit_links),
3638 })
3639 }
3640
3641 fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3642 ClusterOutcomeAuditKey {
3643 terminal_head: head.outcome_head,
3644 commit_head: head.commit_head,
3645 floor: head.outcome_floor.clone(),
3646 }
3647 }
3648
3649 fn cached_cluster_outcome_audit(
3650 &self,
3651 key: &ClusterOutcomeAuditKey,
3652 authority_sequence: u64,
3653 ) -> Option<ClusterOutcomeAuditSnapshot> {
3654 let mut cache = self.outcome_audit_cache.lock();
3655 let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3656 cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3657 Some(cached.snapshot.clone())
3658 }
3659
3660 fn install_cluster_outcome_audit(
3661 &self,
3662 key: ClusterOutcomeAuditKey,
3663 authority_sequence: u64,
3664 snapshot: ClusterOutcomeAuditSnapshot,
3665 ) {
3666 let mut cache = self.outcome_audit_cache.lock();
3667 if cache.as_ref().is_none_or(|cached| {
3668 cached.authority_sequence < authority_sequence
3669 || (cached.authority_sequence == authority_sequence && cached.key == key)
3670 }) {
3671 *cache = Some(ClusterOutcomeAuditCache {
3672 key,
3673 authority_sequence,
3674 snapshot,
3675 });
3676 }
3677 }
3678
3679 fn cluster_outcome_audit_gate(
3680 &self,
3681 key: &ClusterOutcomeAuditKey,
3682 ) -> Arc<tokio::sync::Mutex<()>> {
3683 let mut flights = self.outcome_audit_flights.lock();
3684 flights.retain(|flight| flight.gate.strong_count() != 0);
3685 if let Some(gate) = flights
3686 .iter()
3687 .find(|flight| flight.key == *key)
3688 .and_then(|flight| flight.gate.upgrade())
3689 {
3690 return gate;
3691 }
3692 let gate = Arc::new(tokio::sync::Mutex::new(()));
3693 flights.push(ClusterOutcomeAuditFlight {
3694 key: key.clone(),
3695 gate: Arc::downgrade(&gate),
3696 });
3697 gate
3698 }
3699
3700 async fn cached_audited_cluster_outcomes_from(
3701 &self,
3702 head: &LeaderAuthorityRecord,
3703 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3704 let key = Self::cluster_outcome_audit_key(head);
3705 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3706 return Ok(snapshot);
3707 }
3708
3709 let gate = self.cluster_outcome_audit_gate(&key);
3710 let _guard = gate.lock().await;
3711 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3712 return Ok(snapshot);
3713 }
3714
3715 let snapshot = self.audited_cluster_outcomes_from(head).await?;
3716 self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
3717 Ok(snapshot)
3718 }
3719
3720 fn outcomes_retained_by_floor(
3721 floor: &AuthorityOutcomeFloor,
3722 snapshot: &ClusterOutcomeAuditSnapshot,
3723 ) -> ClusterOutcomeAuditSnapshot {
3724 let outcomes = &snapshot.outcomes;
3725 let mut retained = Vec::with_capacity(outcomes.len());
3726 if let Some(anchor) = floor.committed_anchor.as_ref() {
3727 retained.push(anchor.clone());
3728 }
3729 if let Some(anchor) = floor.terminal_anchor.as_ref() {
3730 if retained.last() != Some(anchor) {
3731 retained.push(anchor.clone());
3732 }
3733 }
3734 retained.extend(
3735 outcomes
3736 .iter()
3737 .filter(|outcome| {
3738 outcome.epoch >= floor.authority_before_epoch
3739 || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
3740 })
3741 .cloned(),
3742 );
3743 retained.sort_unstable_by_key(|outcome| outcome.epoch);
3744 retained.dedup();
3745 ClusterOutcomeAuditSnapshot {
3746 outcomes: Arc::from(retained),
3747 terminal_links: Arc::from(
3748 snapshot
3749 .terminal_links
3750 .iter()
3751 .filter(|link| link.epoch >= floor.authority_before_epoch)
3752 .copied()
3753 .collect::<Vec<_>>(),
3754 ),
3755 commit_links: Arc::from(
3756 snapshot
3757 .commit_links
3758 .iter()
3759 .filter(|link| link.epoch >= floor.artifact_before_epoch)
3760 .copied()
3761 .collect::<Vec<_>>(),
3762 ),
3763 }
3764 }
3765
3766 async fn build_cluster_outcome_floor(
3767 &self,
3768 current: &LeaderAuthorityRecord,
3769 snapshot: &ClusterOutcomeAuditSnapshot,
3770 artifact_before_epoch: u64,
3771 authority_before_epoch: u64,
3772 ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
3773 let terminal_anchor = snapshot
3774 .outcomes
3775 .iter()
3776 .rev()
3777 .find(|outcome| outcome.epoch < authority_before_epoch)
3778 .cloned();
3779 let terminal_anchor_link = match terminal_anchor.as_ref() {
3780 Some(anchor)
3781 if current
3782 .outcome_floor
3783 .as_ref()
3784 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
3785 {
3786 current
3787 .outcome_floor
3788 .as_ref()
3789 .and_then(|floor| floor.terminal_anchor_link)
3790 }
3791 Some(anchor) => {
3792 let index = snapshot
3793 .terminal_links
3794 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
3795 .map_err(|_| {
3796 DecisionError::Conflict(format!(
3797 "cluster outcome epoch {} has no audited authority link",
3798 anchor.epoch
3799 ))
3800 })?;
3801 let link = snapshot.terminal_links[index];
3802 if link.checkpoint_id != anchor.checkpoint_id {
3803 return Err(DecisionError::Conflict(format!(
3804 "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
3805 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
3806 ))
3807 .into());
3808 }
3809 Some(link)
3810 }
3811 None => None,
3812 };
3813 let committed_anchor = snapshot
3814 .outcomes
3815 .iter()
3816 .rev()
3817 .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
3818 .cloned();
3819 let committed_anchor_link = match committed_anchor.as_ref() {
3820 Some(anchor)
3821 if current
3822 .outcome_floor
3823 .as_ref()
3824 .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
3825 {
3826 current
3827 .outcome_floor
3828 .as_ref()
3829 .and_then(|floor| floor.committed_anchor_link)
3830 }
3831 Some(anchor) => {
3832 let index = snapshot
3833 .commit_links
3834 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
3835 .map_err(|_| {
3836 DecisionError::Conflict(format!(
3837 "cluster Commit epoch {} has no audited Commit link",
3838 anchor.epoch
3839 ))
3840 })?;
3841 let link = snapshot.commit_links[index];
3842 if link.checkpoint_id != anchor.checkpoint_id {
3843 return Err(DecisionError::Conflict(format!(
3844 "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
3845 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
3846 ))
3847 .into());
3848 }
3849 Some(link)
3850 }
3851 None => None,
3852 };
3853 let floor = AuthorityOutcomeFloor {
3854 deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
3855 .load_or_create_deployment_id()
3856 .await?,
3857 artifact_before_epoch,
3858 authority_before_epoch,
3859 terminal_anchor,
3860 terminal_anchor_link,
3861 committed_anchor,
3862 committed_anchor_link,
3863 };
3864 floor.validate()?;
3865 Ok(floor)
3866 }
3867
3868 async fn compact_cluster_outcome_history_before_append(
3869 &self,
3870 proof: &LeaderProof,
3871 current: &LeaderAuthorityRecord,
3872 snapshot: &ClusterOutcomeAuditSnapshot,
3873 ) -> Result<bool, ClusterCheckpointAuthorityError> {
3874 if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
3875 return Ok(false);
3876 }
3877 let authority_before_epoch = snapshot.terminal_links
3878 [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
3879 .epoch;
3880 let artifact_before_epoch = current
3881 .outcome_floor
3882 .as_ref()
3883 .map_or(0, |floor| floor.artifact_before_epoch);
3884 let floor = self
3885 .build_cluster_outcome_floor(
3886 current,
3887 snapshot,
3888 artifact_before_epoch,
3889 authority_before_epoch,
3890 )
3891 .await?;
3892 let published = self
3895 .load_published_authority_head()
3896 .await?
3897 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
3898 let rechecked = &published.record;
3899 if !rechecked.lease.matches_proof(proof) {
3900 return Err(ClusterCheckpointAuthorityError::Fenced);
3901 }
3902 if rechecked.outcome_head != current.outcome_head
3903 || rechecked.commit_head != current.commit_head
3904 || rechecked.outcome_floor != current.outcome_floor
3905 {
3906 return Ok(true);
3907 }
3908
3909 let sequence = rechecked
3910 .lease
3911 .seq
3912 .checked_add(1)
3913 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
3914 let mut lease = rechecked.lease.clone();
3915 lease.seq = sequence;
3916 let mut next = rechecked.preserve_with_lease(lease);
3917 next.outcome_floor = Some(floor.clone());
3918 next.validate()?;
3919 match self
3920 .create_authority_record(Some(&published), &next)
3921 .await?
3922 {
3923 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
3924 self.install_cluster_outcome_audit(
3925 Self::cluster_outcome_audit_key(&next),
3926 next.lease.seq,
3927 Self::outcomes_retained_by_floor(&floor, snapshot),
3928 );
3929 Ok(true)
3930 }
3931 AuthorityCreateOutcome::Contended(winner) => {
3932 if !winner.lease.matches_proof(proof) {
3933 return Err(ClusterCheckpointAuthorityError::Fenced);
3934 }
3935 if winner.lease.seq <= current.lease.seq {
3936 return Err(LeaseError::Invalid(
3937 "outcome history compaction contention did not advance the authority sequence"
3938 .into(),
3939 )
3940 .into());
3941 }
3942 Ok(true)
3943 }
3944 }
3945 }
3946
3947 async fn audited_cluster_outcomes(
3948 &self,
3949 ) -> Result<
3950 (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
3951 ClusterCheckpointAuthorityError,
3952 > {
3953 const AUDIT_RETRIES: usize = 3;
3954 for attempt in 0..AUDIT_RETRIES {
3955 let Some(head) = self.load_record().await? else {
3956 *self.outcome_audit_cache.lock() = None;
3957 return Ok((None, Arc::from([])));
3958 };
3959 match self.cached_audited_cluster_outcomes_from(&head).await {
3960 Err(ClusterCheckpointAuthorityError::Decision(
3961 DecisionError::InventoryChanged(_),
3962 )) if attempt + 1 < AUDIT_RETRIES => {
3963 tokio::task::yield_now().await;
3964 }
3965 Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
3966 Err(error) => return Err(error),
3967 }
3968 }
3969 Err(DecisionError::InventoryChanged(
3970 "cluster outcome audit exhausted stability retries".into(),
3971 )
3972 .into())
3973 }
3974
3975 async fn audited_assignment_decisions_from(
3976 &self,
3977 head: &LeaderAuthorityRecord,
3978 ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
3979 let mut newest_first = Vec::new();
3980 let floor = head.assignment_decision_floor.as_ref();
3981 let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
3982 let mut stopped_at_anchor = false;
3983 let mut link = head.assignment_decision_head;
3984 let mut traversed = 0;
3985 while let Some(current) = link {
3986 if !consume_live_authority_link(&mut traversed) {
3987 return Err(DecisionError::Conflict(format!(
3988 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3989 ))
3990 .into());
3991 }
3992 if current.target_version < before_target_version {
3993 if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
3994 return Err(DecisionError::Conflict(format!(
3995 "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
3996 ))
3997 .into());
3998 }
3999 stopped_at_anchor = true;
4000 break;
4001 }
4002 let record = read_authority_record(self.store.as_ref(), current.sequence)
4003 .await?
4004 .ok_or_else(|| {
4005 DecisionError::InventoryChanged(format!(
4006 "assignment decision authority record {} disappeared during audit",
4007 current.sequence
4008 ))
4009 })?;
4010 let decision = record.assignment_decision.clone().ok_or_else(|| {
4011 DecisionError::Conflict(format!(
4012 "assignment decision head version {} points to non-decision authority record {}",
4013 current.target_version, current.sequence
4014 ))
4015 })?;
4016 if record.assignment_decision_head != Some(current)
4017 || decision.target_version() != current.target_version
4018 {
4019 return Err(DecisionError::Conflict(format!(
4020 "assignment decision link version {} sequence {} does not match its authority record",
4021 current.target_version, current.sequence
4022 ))
4023 .into());
4024 }
4025 newest_first.push(decision);
4026 link = record.previous_assignment_decision;
4027 }
4028 if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4029 return Err(DecisionError::Conflict(
4030 "assignment decision chain stopped without its durable retention anchor".into(),
4031 )
4032 .into());
4033 }
4034 newest_first.reverse();
4035 if let Some(pair) = newest_first
4036 .windows(2)
4037 .find(|pair| pair[0].target_version() >= pair[1].target_version())
4038 {
4039 return Err(DecisionError::Conflict(format!(
4040 "assignment decisions regress from version {} to {}",
4041 pair[0].target_version(),
4042 pair[1].target_version()
4043 ))
4044 .into());
4045 }
4046 Ok(newest_first)
4047 }
4048
4049 async fn exact_assignment_decision_link(
4050 &self,
4051 head: &LeaderAuthorityRecord,
4052 decision: &AuthorityAssignmentDecision,
4053 ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4054 if let Some(floor) = head.assignment_decision_floor.as_ref() {
4055 if floor.terminal_anchor.as_ref() == Some(decision) {
4056 return floor.terminal_anchor_link.ok_or_else(|| {
4057 DecisionError::Conflict(
4058 "assignment decision floor lost its terminal authority link".into(),
4059 )
4060 .into()
4061 });
4062 }
4063 }
4064 let before_target_version = head
4065 .assignment_decision_floor
4066 .as_ref()
4067 .map_or(0, |floor| floor.before_target_version);
4068 let mut link = head.assignment_decision_head;
4069 let mut traversed = 0;
4070 while let Some(current) = link {
4071 if !consume_live_authority_link(&mut traversed) {
4072 return Err(DecisionError::Conflict(format!(
4073 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4074 ))
4075 .into());
4076 }
4077 if current.target_version < before_target_version {
4078 break;
4079 }
4080 let record = read_authority_record(self.store.as_ref(), current.sequence)
4081 .await?
4082 .ok_or_else(|| {
4083 DecisionError::InventoryChanged(format!(
4084 "assignment decision authority record {} disappeared during exact lookup",
4085 current.sequence
4086 ))
4087 })?;
4088 if record.assignment_decision.as_ref() == Some(decision) {
4089 return Ok(current);
4090 }
4091 link = record.previous_assignment_decision;
4092 }
4093 Err(DecisionError::Conflict(format!(
4094 "assignment decision version {} is not linked from the durable authority head",
4095 decision.target_version()
4096 ))
4097 .into())
4098 }
4099
4100 pub async fn record_cluster_outcome(
4109 &self,
4110 proof: &LeaderProof,
4111 epoch: u64,
4112 checkpoint_id: u64,
4113 assignment_fence: CheckpointAssignmentFence,
4114 verdict: CheckpointVerdict,
4115 recovery_capsule: Option<RecoveryCapsuleRef>,
4116 ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4117 if !proof.is_canonical() {
4118 return Err(ClusterCheckpointAuthorityError::Fenced);
4119 }
4120 let attempt = crate::state::CheckpointAttempt::new(epoch, checkpoint_id);
4121 if !attempt.is_canonical() {
4122 return Err(DecisionError::Conflict(
4123 "cluster checkpoint outcomes require one nonzero canonical checkpoint ID".into(),
4124 )
4125 .into());
4126 }
4127 let initial = self
4128 .load_record()
4129 .await?
4130 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4131 if !initial.lease.matches_proof(proof) {
4132 return Err(ClusterCheckpointAuthorityError::Fenced);
4133 }
4134 let candidate = CheckpointDecisionStore::new(Arc::clone(&self.store))
4135 .canonical_outcome(
4136 epoch,
4137 checkpoint_id,
4138 CheckpointScope::Cluster,
4139 Some(assignment_fence),
4140 Some(proof.clone()),
4141 verdict,
4142 recovery_capsule,
4143 )
4144 .await?;
4145
4146 loop {
4147 let published = self
4148 .load_published_authority_head()
4149 .await?
4150 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4151 let current = &published.record;
4152 if !current.lease.matches_proof(proof) {
4153 return Err(ClusterCheckpointAuthorityError::Fenced);
4154 }
4155 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4156 let outcomes = &snapshot.outcomes;
4157 if let Some(winner) = outcomes
4158 .iter()
4159 .find(|outcome| outcome.epoch == candidate.epoch)
4160 {
4161 return if winner == &candidate {
4162 Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4163 } else {
4164 Ok(RecordOutcomeResult::Conflict {
4165 winner: winner.clone(),
4166 })
4167 };
4168 }
4169 if let Some(last) = outcomes.last() {
4170 if candidate.checkpoint_id <= last.checkpoint_id {
4171 return Err(DecisionError::Conflict(format!(
4172 "cluster checkpoint {} does not advance durable checkpoint {}",
4173 candidate.checkpoint_id, last.checkpoint_id
4174 ))
4175 .into());
4176 }
4177 }
4178 if candidate.is_commit() && snapshot.commit_links.len() >= MAX_LIVE_AUTHORITY_LINKS {
4179 return Err(DecisionError::Conflict(format!(
4180 "live Commit retention reached the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound; advance the artifact-retention horizon before admitting another Commit"
4181 ))
4182 .into());
4183 }
4184 if let Some(floor) = current.outcome_floor.as_ref() {
4185 if candidate.deployment_id != floor.deployment_id
4186 || candidate.epoch < floor.authority_before_epoch
4187 {
4188 return Err(DecisionError::Conflict(format!(
4189 "cluster outcome epoch {} is below or outside authority floor {}",
4190 candidate.epoch, floor.authority_before_epoch
4191 ))
4192 .into());
4193 }
4194 }
4195 if Box::pin(
4196 self.compact_cluster_outcome_history_before_append(proof, current, &snapshot),
4197 )
4198 .await?
4199 {
4200 tokio::task::yield_now().await;
4201 continue;
4202 }
4203
4204 let base_sequence = current.lease.seq;
4205 let sequence = base_sequence
4206 .checked_add(1)
4207 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4208 let mut next = current.preserve_with_lease(LeaderLease {
4209 seq: sequence,
4210 renewal_sequence: current.lease.renewal_sequence,
4211 token: current.lease.token,
4212 owner: current.lease.owner.clone(),
4213 expires_at_ms: current.lease.expires_at_ms,
4214 catalog_manifest: current.lease.catalog_manifest.clone(),
4215 });
4216 next.checkpoint_outcome = Some(candidate.clone());
4217 next.previous_outcome = current.outcome_head;
4218 let new_link = OutcomeLink {
4219 sequence,
4220 epoch: candidate.epoch,
4221 checkpoint_id: candidate.checkpoint_id,
4222 };
4223 next.outcome_head = Some(new_link);
4224 if candidate.is_commit() {
4225 next.previous_commit = current.commit_head;
4226 next.commit_head = Some(new_link);
4227 }
4228 next.validate()?;
4229 match self
4230 .create_authority_record(Some(&published), &next)
4231 .await?
4232 {
4233 AuthorityCreateOutcome::Created => {
4234 let mut appended = outcomes.to_vec();
4235 appended.push(candidate.clone());
4236 let mut terminal_links = snapshot.terminal_links.to_vec();
4237 terminal_links.push(new_link);
4238 let mut commit_links = snapshot.commit_links.to_vec();
4239 if candidate.is_commit() {
4240 commit_links.push(new_link);
4241 }
4242 self.install_cluster_outcome_audit(
4243 Self::cluster_outcome_audit_key(&next),
4244 next.lease.seq,
4245 ClusterOutcomeAuditSnapshot {
4246 outcomes: Arc::from(appended),
4247 terminal_links: Arc::from(terminal_links),
4248 commit_links: Arc::from(commit_links),
4249 },
4250 );
4251 return Ok(RecordOutcomeResult::Created(candidate));
4252 }
4253 AuthorityCreateOutcome::ExistingIdentical => {
4254 return Ok(RecordOutcomeResult::Unchanged(candidate));
4255 }
4256 AuthorityCreateOutcome::Contended(winner_head) => {
4257 let winners = self
4258 .cached_audited_cluster_outcomes_from(&winner_head)
4259 .await?;
4260 if let Some(winner) = winners
4261 .outcomes
4262 .iter()
4263 .find(|outcome| outcome.epoch == candidate.epoch)
4264 {
4265 return if winner == &candidate {
4266 Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4267 } else {
4268 Ok(RecordOutcomeResult::Conflict {
4269 winner: winner.clone(),
4270 })
4271 };
4272 }
4273 if !winner_head.lease.matches_proof(proof) {
4274 return Err(ClusterCheckpointAuthorityError::Fenced);
4275 }
4276 if winner_head.lease.seq <= base_sequence {
4277 return Err(LeaseError::Invalid(
4278 "cluster outcome contention did not advance the authority sequence"
4279 .into(),
4280 )
4281 .into());
4282 }
4283 tokio::task::yield_now().await;
4284 }
4285 }
4286 }
4287 }
4288
4289 async fn validate_assignment_recovery_snapshot(
4290 &self,
4291 decision: &AssignmentRecoveryDecision,
4292 ) -> Result<(), ClusterCheckpointAuthorityError> {
4293 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4294 let predecessor = snapshots
4295 .load()
4296 .await
4297 .map_err(|error| {
4298 assignment_snapshot_error("load assignment recovery predecessor", error)
4299 })?
4300 .ok_or_else(|| {
4301 LeaseError::Invalid(
4302 "assignment recovery requires a durable predecessor snapshot".into(),
4303 )
4304 })?;
4305 let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4306 LeaseError::Invalid(format!(
4307 "assignment recovery predecessor has no valid fence: {error}"
4308 ))
4309 })?;
4310 if predecessor.draining || predecessor_fence != decision.predecessor {
4311 return Err(LeaseError::Invalid(
4312 "assignment recovery predecessor is not the exact committed assignment head".into(),
4313 )
4314 .into());
4315 }
4316 let proposal = snapshots
4317 .load_recovery_proposal(&decision.proposal)
4318 .await
4319 .map_err(|error| {
4320 assignment_snapshot_error("load assignment recovery proposal", error)
4321 })?;
4322 let proposal_fence = proposal.assignment_fence().map_err(|error| {
4323 LeaseError::Invalid(format!(
4324 "assignment recovery proposal has no valid fence: {error}"
4325 ))
4326 })?;
4327 if proposal_fence != decision.target {
4328 return Err(LeaseError::Invalid(
4329 "assignment recovery proposal does not match its authorized target fence".into(),
4330 )
4331 .into());
4332 }
4333 Ok(())
4334 }
4335
4336 async fn record_assignment_decision(
4337 &self,
4338 proof: &LeaderProof,
4339 decision: AuthorityAssignmentDecision,
4340 ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
4341 decision.validate()?;
4342 if decision.leader_proof() != proof || !proof.is_canonical() {
4343 return Err(ClusterCheckpointAuthorityError::Fenced);
4344 }
4345
4346 loop {
4347 let published = self
4348 .load_published_authority_head()
4349 .await?
4350 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4351 let current = &published.record;
4352 if !current.lease.matches_proof(proof) {
4353 return Err(ClusterCheckpointAuthorityError::Fenced);
4354 }
4355 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4356 if decision.target_version() < floor.before_target_version {
4357 return Err(DecisionError::Conflict(format!(
4358 "assignment decision version {} is below durable retention floor {}",
4359 decision.target_version(),
4360 floor.before_target_version
4361 ))
4362 .into());
4363 }
4364 }
4365 let decisions = self.audited_assignment_decisions_from(current).await?;
4366 if let Some(winner) = decisions
4367 .iter()
4368 .find(|winner| winner.target_version() == decision.target_version())
4369 {
4370 return if winner == &decision {
4371 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
4372 winner.clone(),
4373 ))
4374 } else {
4375 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
4376 winner: winner.clone(),
4377 })
4378 };
4379 }
4380 if let Some(last) = decisions.last() {
4381 if decision.target_version() <= last.target_version() {
4382 return Err(DecisionError::Conflict(format!(
4383 "assignment decision version {} does not advance durable version {}",
4384 decision.target_version(),
4385 last.target_version()
4386 ))
4387 .into());
4388 }
4389 }
4390 if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
4391 self.validate_assignment_recovery_snapshot(recovery).await?;
4392 }
4393
4394 let base_sequence = current.lease.seq;
4395 let sequence = base_sequence
4396 .checked_add(1)
4397 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4398 let mut candidate = current.preserve_with_lease(LeaderLease {
4399 seq: sequence,
4400 renewal_sequence: current.lease.renewal_sequence,
4401 token: current.lease.token,
4402 owner: current.lease.owner.clone(),
4403 expires_at_ms: current.lease.expires_at_ms,
4404 catalog_manifest: current.lease.catalog_manifest.clone(),
4405 });
4406 candidate.assignment_decision = Some(decision.clone());
4407 candidate.previous_assignment_decision = current.assignment_decision_head;
4408 candidate.assignment_decision_head = Some(AssignmentDecisionLink {
4409 sequence,
4410 target_version: decision.target_version(),
4411 });
4412 candidate.validate()?;
4413
4414 match self
4415 .create_authority_record(Some(&published), &candidate)
4416 .await?
4417 {
4418 AuthorityCreateOutcome::Created => {
4419 return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
4420 }
4421 AuthorityCreateOutcome::ExistingIdentical => {
4422 return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
4423 }
4424 AuthorityCreateOutcome::Contended(winner_head) => {
4425 let winners = self.audited_assignment_decisions_from(&winner_head).await?;
4426 if let Some(winner) = winners
4427 .iter()
4428 .find(|winner| winner.target_version() == decision.target_version())
4429 {
4430 return if winner == &decision {
4431 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
4432 winner.clone(),
4433 ))
4434 } else {
4435 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
4436 winner: winner.clone(),
4437 })
4438 };
4439 }
4440 if !winner_head.lease.matches_proof(proof) {
4441 return Err(ClusterCheckpointAuthorityError::Fenced);
4442 }
4443 if winner_head.lease.seq > base_sequence {
4444 tokio::task::yield_now().await;
4445 continue;
4446 }
4447 return Err(LeaseError::Invalid(
4448 "assignment authority contention did not advance the sequence".into(),
4449 )
4450 .into());
4451 }
4452 }
4453 }
4454 }
4455
4456 pub async fn record_assignment_drain_decision(
4464 &self,
4465 proof: &LeaderProof,
4466 decision: AssignmentDrainDecision,
4467 ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
4468 match Box::pin(
4469 self.record_assignment_decision(proof, AuthorityAssignmentDecision::Drain(decision)),
4470 )
4471 .await?
4472 {
4473 RecordAuthorityAssignmentDecisionResult::Created(
4474 AuthorityAssignmentDecision::Drain(decision),
4475 ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
4476 RecordAuthorityAssignmentDecisionResult::Unchanged(
4477 AuthorityAssignmentDecision::Drain(decision),
4478 ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
4479 RecordAuthorityAssignmentDecisionResult::Conflict {
4480 winner: AuthorityAssignmentDecision::Drain(winner),
4481 } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
4482 RecordAuthorityAssignmentDecisionResult::Conflict {
4483 winner: AuthorityAssignmentDecision::Recovery(winner),
4484 } => Err(DecisionError::Conflict(format!(
4485 "assignment recovery decision already settled target version {}",
4486 winner.target_version()
4487 ))
4488 .into()),
4489 RecordAuthorityAssignmentDecisionResult::Created(_)
4490 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
4491 "assignment decision recorder returned the wrong decision kind".into(),
4492 )
4493 .into()),
4494 }
4495 }
4496
4497 pub(crate) async fn record_assignment_recovery_decision(
4502 &self,
4503 proof: &LeaderProof,
4504 decision: AssignmentRecoveryDecision,
4505 ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
4506 decision.validate()?;
4507 if &decision.leader_proof != proof || !proof.is_canonical() {
4508 return Err(ClusterCheckpointAuthorityError::Fenced);
4509 }
4510 let current = self
4511 .load_record()
4512 .await?
4513 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4514 if !current.lease.matches_proof(proof) {
4515 return Err(ClusterCheckpointAuthorityError::Fenced);
4516 }
4517 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4518 if decision.target_version() < floor.before_target_version {
4519 return Err(DecisionError::Conflict(format!(
4520 "assignment decision version {} is below durable retention floor {}",
4521 decision.target_version(),
4522 floor.before_target_version
4523 ))
4524 .into());
4525 }
4526 }
4527 if let Some(winner) = self
4528 .audited_assignment_decisions_from(¤t)
4529 .await?
4530 .into_iter()
4531 .find(|winner| winner.target_version() == decision.target_version())
4532 {
4533 return match winner {
4534 AuthorityAssignmentDecision::Recovery(winner) if winner == decision => {
4535 Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(winner))
4536 }
4537 AuthorityAssignmentDecision::Recovery(winner) => {
4538 Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner })
4539 }
4540 AuthorityAssignmentDecision::Drain(winner) => {
4541 Err(DecisionError::Conflict(format!(
4542 "assignment drain decision already settled target version {}",
4543 winner.target_version()
4544 ))
4545 .into())
4546 }
4547 };
4548 }
4549 match Box::pin(
4550 self.record_assignment_decision(proof, AuthorityAssignmentDecision::Recovery(decision)),
4551 )
4552 .await?
4553 {
4554 RecordAuthorityAssignmentDecisionResult::Created(
4555 AuthorityAssignmentDecision::Recovery(decision),
4556 ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
4557 RecordAuthorityAssignmentDecisionResult::Unchanged(
4558 AuthorityAssignmentDecision::Recovery(decision),
4559 ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
4560 RecordAuthorityAssignmentDecisionResult::Conflict {
4561 winner: AuthorityAssignmentDecision::Recovery(winner),
4562 } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
4563 RecordAuthorityAssignmentDecisionResult::Conflict {
4564 winner: AuthorityAssignmentDecision::Drain(winner),
4565 } => Err(DecisionError::Conflict(format!(
4566 "assignment drain decision already settled target version {}",
4567 winner.target_version()
4568 ))
4569 .into()),
4570 RecordAuthorityAssignmentDecisionResult::Created(_)
4571 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
4572 "assignment decision recorder returned the wrong decision kind".into(),
4573 )
4574 .into()),
4575 }
4576 }
4577
4578 async fn assignment_decision(
4579 &self,
4580 target_version: u64,
4581 ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4582 if target_version == 0 {
4583 return Err(LeaseError::Invalid(
4584 "assignment decision target version must be nonzero".into(),
4585 )
4586 .into());
4587 }
4588 let Some(head) = self.load_record().await? else {
4589 return Ok(None);
4590 };
4591 if let Some(floor) = head.assignment_decision_floor.as_ref() {
4592 if target_version < floor.before_target_version {
4593 return Err(DecisionError::Conflict(format!(
4594 "assignment decision version {target_version} is below durable retention floor {}",
4595 floor.before_target_version
4596 ))
4597 .into());
4598 }
4599 }
4600 Ok(self
4601 .audited_assignment_decisions_from(&head)
4602 .await?
4603 .into_iter()
4604 .find(|decision| decision.target_version() == target_version))
4605 }
4606
4607 pub async fn assignment_drain_decision(
4612 &self,
4613 target_version: u64,
4614 ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
4615 match self.assignment_decision(target_version).await? {
4616 Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
4617 Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
4618 format!("target version {target_version} was settled by assignment recovery"),
4619 )
4620 .into()),
4621 None => Ok(None),
4622 }
4623 }
4624
4625 pub async fn assignment_recovery_decision(
4630 &self,
4631 target_version: u64,
4632 ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
4633 match self.assignment_decision(target_version).await? {
4634 Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
4635 Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
4636 "target version {target_version} was settled by assignment drain"
4637 ))
4638 .into()),
4639 None => Ok(None),
4640 }
4641 }
4642
4643 pub async fn materialize_assignment_recovery(
4651 &self,
4652 target_version: u64,
4653 ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
4654 let decision = self
4655 .assignment_recovery_decision(target_version)
4656 .await?
4657 .ok_or_else(|| {
4658 DecisionError::Conflict(format!(
4659 "assignment recovery target {target_version} has no authority decision"
4660 ))
4661 })?;
4662 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4663 let proposal = snapshots
4664 .load_recovery_proposal(&decision.proposal)
4665 .await
4666 .map_err(|error| {
4667 assignment_snapshot_error("load authorized assignment recovery proposal", error)
4668 })?;
4669 if proposal.assignment_fence().map_err(|error| {
4670 LeaseError::Invalid(format!(
4671 "authorized assignment recovery proposal has no valid fence: {error}"
4672 ))
4673 })? != decision.target
4674 {
4675 return Err(LeaseError::Invalid(
4676 "authorized assignment recovery proposal does not match its target".into(),
4677 )
4678 .into());
4679 }
4680 snapshots
4681 .materialize_recovery(&decision.proposal)
4682 .await
4683 .map_err(|error| {
4684 assignment_snapshot_error("materialize authorized assignment recovery", error)
4685 .into()
4686 })
4687 }
4688
4689 pub async fn prune_assignment_drain_decisions_before(
4701 &self,
4702 proof: &LeaderProof,
4703 before_target_version: u64,
4704 ) -> Result<u64, ClusterCheckpointAuthorityError> {
4705 if before_target_version == 0 {
4706 return Ok(self
4707 .load_record()
4708 .await?
4709 .and_then(|head| head.assignment_decision_floor)
4710 .map_or(0, |floor| floor.before_target_version));
4711 }
4712 if !proof.is_canonical() {
4713 return Err(ClusterCheckpointAuthorityError::Fenced);
4714 }
4715 loop {
4716 let published = self
4717 .load_published_authority_head()
4718 .await?
4719 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4720 let current = &published.record;
4721 if !current.lease.matches_proof(proof) {
4722 return Err(ClusterCheckpointAuthorityError::Fenced);
4723 }
4724 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4725 if floor.before_target_version >= before_target_version {
4726 self.schedule_history_prune();
4727 return Ok(floor.before_target_version);
4728 }
4729 }
4730
4731 let decisions = self.audited_assignment_decisions_from(current).await?;
4732 let terminal_anchor = decisions
4733 .iter()
4734 .rev()
4735 .find(|decision| decision.target_version() < before_target_version)
4736 .cloned()
4737 .or_else(|| {
4738 current
4739 .assignment_decision_floor
4740 .as_ref()
4741 .and_then(|floor| floor.terminal_anchor.clone())
4742 });
4743 let terminal_anchor_link = match terminal_anchor.as_ref() {
4744 Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
4745 None => None,
4746 };
4747 let floor = AuthorityAssignmentDecisionFloor {
4748 before_target_version,
4749 terminal_anchor,
4750 terminal_anchor_link,
4751 };
4752 floor.validate()?;
4753
4754 let base_sequence = current.lease.seq;
4755 let sequence = base_sequence
4756 .checked_add(1)
4757 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4758 let mut next = current.preserve_with_lease(LeaderLease {
4759 seq: sequence,
4760 renewal_sequence: current.lease.renewal_sequence,
4761 token: current.lease.token,
4762 owner: current.lease.owner.clone(),
4763 expires_at_ms: current.lease.expires_at_ms,
4764 catalog_manifest: current.lease.catalog_manifest.clone(),
4765 });
4766 next.assignment_decision_floor = Some(floor);
4767 next.validate()?;
4768
4769 match self
4770 .create_authority_record(Some(&published), &next)
4771 .await?
4772 {
4773 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4774 return Ok(before_target_version);
4775 }
4776 AuthorityCreateOutcome::Contended(winner) => {
4777 if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
4778 if winner_floor.before_target_version >= before_target_version {
4779 self.schedule_history_prune();
4780 return Ok(winner_floor.before_target_version);
4781 }
4782 }
4783 if !winner.lease.matches_proof(proof) {
4784 return Err(ClusterCheckpointAuthorityError::Fenced);
4785 }
4786 if winner.lease.seq > base_sequence {
4787 tokio::task::yield_now().await;
4788 continue;
4789 }
4790 return Err(LeaseError::Invalid(
4791 "assignment decision floor contention did not advance the sequence".into(),
4792 )
4793 .into());
4794 }
4795 }
4796 }
4797 }
4798
4799 async fn cluster_outcome_from_snapshot(
4800 &self,
4801 head: &LeaderAuthorityRecord,
4802 epoch: u64,
4803 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4804 let floor = head.outcome_floor.as_ref();
4805 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
4806 if epoch < artifact_before_epoch {
4807 return Ok(None);
4808 }
4809 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
4810 let cache_key = Self::cluster_outcome_audit_key(head);
4811 if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
4812 return Ok(snapshot
4813 .outcomes
4814 .binary_search_by_key(&epoch, |outcome| outcome.epoch)
4815 .ok()
4816 .map(|index| snapshot.outcomes[index].clone())
4817 .filter(|outcome| {
4818 outcome.epoch >= authority_before_epoch
4819 || outcome.is_commit()
4820 || floor
4821 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
4822 }));
4823 }
4824 if let Some(anchor) = floor
4825 .and_then(|floor| floor.terminal_anchor.as_ref())
4826 .filter(|anchor| anchor.epoch == epoch)
4827 {
4828 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
4829 .load_or_create_deployment_id()
4830 .await?;
4831 if anchor.deployment_id != expected_deployment {
4832 return Err(DecisionError::Conflict(format!(
4833 "cluster terminal anchor epoch {} does not belong to current deployment {}",
4834 anchor.epoch, expected_deployment
4835 ))
4836 .into());
4837 }
4838 return Ok(Some(anchor.clone()));
4839 }
4840
4841 let commit_only = epoch < authority_before_epoch;
4842 let before_epoch = if commit_only {
4843 artifact_before_epoch
4844 } else {
4845 authority_before_epoch
4846 };
4847 let mut current = if commit_only {
4848 head.commit_head
4849 } else {
4850 head.outcome_head
4851 };
4852 let expected_anchor = if commit_only {
4853 floor.and_then(|floor| floor.committed_anchor_link)
4854 } else {
4855 floor.and_then(|floor| floor.terminal_anchor_link)
4856 };
4857 let expected_anchor_outcome = if commit_only {
4858 floor.and_then(|floor| floor.committed_anchor.as_ref())
4859 } else {
4860 floor.and_then(|floor| floor.terminal_anchor.as_ref())
4861 };
4862 let mut traversed = 0;
4863 while let Some(link) = current {
4864 if link.epoch < before_epoch {
4865 match (expected_anchor, expected_anchor_outcome) {
4866 (Some(anchor_link), Some(anchor))
4867 if link == anchor_link
4868 && link.epoch == anchor.epoch
4869 && link.checkpoint_id == anchor.checkpoint_id =>
4870 {
4871 return Ok(None)
4872 }
4873 (None, None) => return Ok(None),
4874 _ => {
4875 return Err(DecisionError::Conflict(format!(
4876 "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
4877 if commit_only { "Commit" } else { "terminal outcome" }
4878 ))
4879 .into());
4880 }
4881 }
4882 }
4883 if link.epoch < epoch {
4884 return Ok(None);
4885 }
4886 if !consume_live_authority_link(&mut traversed) {
4887 return Err(DecisionError::Conflict(format!(
4888 "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
4889 if commit_only { "Commit" } else { "outcome" }
4890 ))
4891 .into());
4892 }
4893
4894 let admission;
4895 let record = if link.sequence == head.lease.seq {
4896 head
4897 } else {
4898 admission = read_authority_record(self.store.as_ref(), link.sequence)
4899 .await?
4900 .ok_or_else(|| {
4901 DecisionError::InventoryChanged(format!(
4902 "cluster {} authority record {} disappeared during exact lookup",
4903 if commit_only { "Commit" } else { "outcome" },
4904 link.sequence
4905 ))
4906 })?;
4907 &admission
4908 };
4909 let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
4910 DecisionError::Conflict(format!(
4911 "cluster {} head epoch {} points to non-outcome authority record {}",
4912 if commit_only { "Commit" } else { "outcome" },
4913 link.epoch,
4914 link.sequence
4915 ))
4916 })?;
4917 let record_head = if commit_only {
4918 record.commit_head
4919 } else {
4920 record.outcome_head
4921 };
4922 if record_head != Some(link)
4923 || (commit_only && !outcome.is_commit())
4924 || outcome.epoch != link.epoch
4925 || outcome.checkpoint_id != link.checkpoint_id
4926 {
4927 return Err(DecisionError::Conflict(format!(
4928 "cluster {} link epoch {} sequence {} does not match its authority record",
4929 if commit_only { "Commit" } else { "outcome" },
4930 link.epoch,
4931 link.sequence
4932 ))
4933 .into());
4934 }
4935 if outcome.epoch == epoch {
4936 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
4937 .load_or_create_deployment_id()
4938 .await?;
4939 if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
4940 || outcome.deployment_id != expected_deployment
4941 {
4942 return Err(DecisionError::Conflict(format!(
4943 "cluster outcome epoch {} does not belong to current deployment {}",
4944 outcome.epoch, expected_deployment
4945 ))
4946 .into());
4947 }
4948 return Ok(Some(outcome.clone()));
4949 }
4950 current = if commit_only {
4951 record.previous_commit
4952 } else {
4953 record.previous_outcome
4954 };
4955 }
4956 if expected_anchor.is_some() {
4957 return Err(DecisionError::Conflict(format!(
4958 "cluster {} chain stopped without its durable floor anchor",
4959 if commit_only { "Commit" } else { "outcome" }
4960 ))
4961 .into());
4962 }
4963 Ok(None)
4964 }
4965
4966 async fn stable_cluster_outcome(
4967 &self,
4968 epoch: u64,
4969 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4970 const LOOKUP_RETRIES: usize = 3;
4971 for attempt in 0..LOOKUP_RETRIES {
4972 let Some(head) = self.load_record().await? else {
4973 return Ok(None);
4974 };
4975 match self.cluster_outcome_from_snapshot(&head, epoch).await {
4976 Err(ClusterCheckpointAuthorityError::Decision(
4977 DecisionError::InventoryChanged(_),
4978 )) if attempt + 1 < LOOKUP_RETRIES => {
4979 tokio::task::yield_now().await;
4980 }
4981 result => return result,
4982 }
4983 }
4984 Err(DecisionError::InventoryChanged(
4985 "cluster outcome exact lookup exhausted stability retries".into(),
4986 )
4987 .into())
4988 }
4989
4990 pub async fn cluster_outcome(
4996 &self,
4997 epoch: u64,
4998 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4999 self.stable_cluster_outcome(epoch).await
5000 }
5001
5002 pub async fn cluster_outcome_with_recovery_capsule(
5009 &self,
5010 epoch: u64,
5011 ) -> Result<
5012 Option<(CheckpointOutcome, Option<ClusterRecoveryCapsule>)>,
5013 ClusterCheckpointAuthorityError,
5014 > {
5015 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5016 let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5017 return Ok(None);
5018 };
5019 let capsule = if outcome.is_commit() {
5020 let reference = outcome.recovery_capsule.as_ref().ok_or_else(|| {
5021 DecisionError::Conflict(format!(
5022 "cluster Commit epoch {} checkpoint {} has no recovery capsule",
5023 outcome.epoch, outcome.checkpoint_id
5024 ))
5025 })?;
5026 let capsule = decisions.load_recovery_capsule(reference).await?;
5027 if capsule.attempt.epoch != outcome.epoch
5028 || capsule.attempt.checkpoint_id != outcome.checkpoint_id
5029 || Some(&capsule.assignment_fence) != outcome.assignment_fence.as_ref()
5030 || capsule.deployment_id != outcome.deployment_id
5031 {
5032 return Err(DecisionError::Conflict(format!(
5033 "cluster Commit epoch {} checkpoint {} does not match recovery capsule '{}'",
5034 outcome.epoch, outcome.checkpoint_id, reference.sha256
5035 ))
5036 .into());
5037 }
5038 Some(capsule)
5039 } else {
5040 None
5041 };
5042 Ok(Some((outcome, capsule)))
5043 }
5044
5045 pub async fn cluster_outcome_inventory(
5051 &self,
5052 ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5053 let (head, outcomes) = self.audited_cluster_outcomes().await?;
5054 Ok(Self::cluster_outcome_inventory_from_audit(
5055 head.as_ref(),
5056 &outcomes,
5057 ))
5058 }
5059
5060 fn cluster_outcome_inventory_from_audit(
5061 head: Option<&LeaderAuthorityRecord>,
5062 outcomes: &[CheckpointOutcome],
5063 ) -> ClusterOutcomeInventory {
5064 let floor = head.and_then(|head| head.outcome_floor.as_ref());
5065 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5066 ClusterOutcomeInventory {
5067 outcomes: outcomes
5068 .iter()
5069 .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5070 .cloned()
5071 .collect(),
5072 retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5073 }
5074 }
5075
5076 pub async fn cluster_outcomes(
5082 &self,
5083 ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5084 Ok(self.cluster_outcome_inventory().await?.outcomes)
5085 }
5086
5087 pub async fn highest_cluster_committed_outcome(
5093 &self,
5094 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5095 Ok(self
5096 .cluster_outcomes()
5097 .await?
5098 .into_iter()
5099 .rev()
5100 .find(CheckpointOutcome::is_commit))
5101 }
5102
5103 pub async fn highest_cluster_terminal_outcome(
5109 &self,
5110 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5111 Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5112 }
5113
5114 pub async fn cluster_attempt_settlement(
5122 &self,
5123 attempt: crate::state::CheckpointAttempt,
5124 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5125 if !attempt.is_canonical() {
5126 return Err(DecisionError::Conflict(
5127 "cluster checkpoint settlement requires one nonzero canonical checkpoint ID".into(),
5128 )
5129 .into());
5130 }
5131 let outcomes = self.audited_cluster_outcomes().await?.1;
5132 if let Ok(index) = outcomes.binary_search_by_key(&attempt.epoch, |outcome| outcome.epoch) {
5133 return Ok(Some(outcomes[index].clone()));
5134 }
5135 let Some(highest) = outcomes.last() else {
5136 return Ok(None);
5137 };
5138 if highest.checkpoint_id > attempt.checkpoint_id {
5139 Ok(Some(highest.clone()))
5140 } else {
5141 Ok(None)
5142 }
5143 }
5144
5145 async fn validate_cluster_recovery_cut(
5146 &self,
5147 floor: &AuthorityOutcomeFloor,
5148 audited_outcomes: &[CheckpointOutcome],
5149 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5150 if floor.artifact_before_epoch == 0 {
5151 return Ok(None);
5152 }
5153 let recovery_cut = audited_outcomes
5154 .iter()
5155 .rev()
5156 .find(|outcome| outcome.epoch >= floor.artifact_before_epoch && outcome.is_commit())
5157 .ok_or_else(|| {
5158 DecisionError::Conflict(format!(
5159 "cluster outcome floor {} has no live commit recovery cut",
5160 floor.artifact_before_epoch
5161 ))
5162 })?;
5163 CheckpointDecisionStore::new(Arc::clone(&self.store))
5164 .validate_recovery_capsule_for_outcome(recovery_cut)
5165 .await?;
5166 Ok(Some(recovery_cut.clone()))
5167 }
5168
5169 async fn preflight_cluster_recovery_cut<V, Fut>(
5170 &self,
5171 floor: &AuthorityOutcomeFloor,
5172 audited_outcomes: &[CheckpointOutcome],
5173 validate_artifacts: &V,
5174 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError>
5175 where
5176 V: Fn(CheckpointOutcome) -> Fut,
5177 Fut: std::future::Future<Output = Result<(), String>>,
5178 {
5179 let recovery_cut = self
5180 .validate_cluster_recovery_cut(floor, audited_outcomes)
5181 .await?;
5182 if let Some(recovery_cut) = recovery_cut.as_ref() {
5183 validate_artifacts(recovery_cut.clone())
5184 .await
5185 .map_err(|error| {
5186 DecisionError::Conflict(format!(
5187 "cluster recovery cut epoch {} checkpoint {} failed durable recovery metadata preflight: {error}",
5188 recovery_cut.epoch, recovery_cut.checkpoint_id
5189 ))
5190 })?;
5191 }
5192 Ok(recovery_cut)
5193 }
5194
5195 pub async fn cluster_outcome_retention_boundary(
5201 &self,
5202 ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
5203 let head = self.load_record().await?;
5204 Ok(ClusterOutcomeRetentionBoundary::from_floor(
5205 head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
5206 ))
5207 }
5208
5209 pub async fn audited_cluster_outcome_retention_boundary(
5216 &self,
5217 ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
5218 Ok(self
5219 .validated_cluster_outcome_inventory(|_| async { Ok(()) })
5220 .await?
5221 .retention_boundary)
5222 }
5223
5224 pub async fn validated_cluster_outcome_inventory<V, Fut>(
5232 &self,
5233 validate_artifacts: V,
5234 ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError>
5235 where
5236 V: Fn(CheckpointOutcome) -> Fut,
5237 Fut: std::future::Future<Output = Result<(), String>>,
5238 {
5239 loop {
5240 let current = self
5241 .load_record()
5242 .await?
5243 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5244 let snapshot = self.cached_audited_cluster_outcomes_from(¤t).await?;
5245 if let Some(floor) = current.outcome_floor.as_ref() {
5246 self.preflight_cluster_recovery_cut(floor, &snapshot.outcomes, &validate_artifacts)
5247 .await?;
5248 }
5249 let rechecked = self
5250 .load_record()
5251 .await?
5252 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5253 if rechecked.outcome_head == current.outcome_head
5254 && rechecked.commit_head == current.commit_head
5255 && rechecked.outcome_floor == current.outcome_floor
5256 {
5257 return Ok(Self::cluster_outcome_inventory_from_audit(
5258 Some(¤t),
5259 &snapshot.outcomes,
5260 ));
5261 }
5262 tokio::task::yield_now().await;
5263 }
5264 }
5265
5266 pub async fn maintain_cluster_recovery_capsules(
5275 &self,
5276 ) -> Result<crate::checkpoint_decision::RecoveryCapsuleGcStep, ClusterCheckpointAuthorityError>
5277 {
5278 let current = self
5279 .load_record()
5280 .await?
5281 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5282 let audited = self.cached_audited_cluster_outcomes_from(¤t).await?;
5283 let Some(floor) = current.outcome_floor.as_ref() else {
5284 return Ok(crate::checkpoint_decision::RecoveryCapsuleGcStep {
5285 examined: 0,
5286 deleted: 0,
5287 quarantined: 0,
5288 pending: false,
5289 });
5290 };
5291 if floor.artifact_before_epoch == 0 {
5292 return Ok(crate::checkpoint_decision::RecoveryCapsuleGcStep {
5293 examined: 0,
5294 deleted: 0,
5295 quarantined: 0,
5296 pending: false,
5297 });
5298 }
5299 let mut known_live_digests = BTreeSet::new();
5300 known_live_digests.extend(
5301 audited
5302 .outcomes
5303 .iter()
5304 .filter(|outcome| outcome.epoch >= floor.artifact_before_epoch)
5305 .filter_map(|outcome| outcome.recovery_capsule.as_ref())
5306 .map(|reference| reference.sha256.clone()),
5307 );
5308 CheckpointDecisionStore::new(Arc::clone(&self.store))
5309 .sweep_recovery_capsules_step(floor.artifact_before_epoch, &known_live_digests)
5310 .await
5311 .map_err(Into::into)
5312 }
5313
5314 pub async fn prune_cluster_outcomes_before<V, Fut>(
5324 &self,
5325 proof: &LeaderProof,
5326 before_epoch: u64,
5327 validate_artifacts: V,
5328 ) -> Result<u64, ClusterCheckpointAuthorityError>
5329 where
5330 V: Fn(CheckpointOutcome) -> Fut,
5331 Fut: std::future::Future<Output = Result<(), String>>,
5332 {
5333 if before_epoch == 0 {
5334 return Ok(self
5335 .cluster_outcome_retention_boundary()
5336 .await?
5337 .artifact_before_epoch);
5338 }
5339 if !proof.is_canonical() {
5340 return Err(ClusterCheckpointAuthorityError::Fenced);
5341 }
5342 loop {
5343 let current = self
5344 .load_record()
5345 .await?
5346 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5347 if !current.lease.matches_proof(proof) {
5348 return Err(ClusterCheckpointAuthorityError::Fenced);
5349 }
5350 if let Some(floor) = current
5351 .outcome_floor
5352 .as_ref()
5353 .filter(|floor| floor.artifact_before_epoch >= before_epoch)
5354 {
5355 self.schedule_history_prune();
5356 return Ok(floor.artifact_before_epoch);
5357 }
5358 let snapshot = self.cached_audited_cluster_outcomes_from(¤t).await?;
5359 if !snapshot
5360 .outcomes
5361 .iter()
5362 .any(|outcome| outcome.epoch >= before_epoch && outcome.is_commit())
5363 {
5364 return Err(DecisionError::Conflict(format!(
5365 "cannot advance cluster outcome floor to {before_epoch}: no live commit recovery cut would remain"
5366 ))
5367 .into());
5368 }
5369 let authority_before_epoch = current
5370 .outcome_floor
5371 .as_ref()
5372 .map_or(before_epoch, |floor| {
5373 floor.authority_before_epoch.max(before_epoch)
5374 });
5375 let floor = self
5376 .build_cluster_outcome_floor(
5377 ¤t,
5378 &snapshot,
5379 before_epoch,
5380 authority_before_epoch,
5381 )
5382 .await?;
5383 self.preflight_cluster_recovery_cut(&floor, &snapshot.outcomes, &validate_artifacts)
5384 .await?;
5385
5386 let published = self
5390 .load_published_authority_head()
5391 .await?
5392 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5393 let rechecked = &published.record;
5394 if !rechecked.lease.matches_proof(proof) {
5395 return Err(ClusterCheckpointAuthorityError::Fenced);
5396 }
5397 if rechecked.outcome_head != current.outcome_head
5398 || rechecked.commit_head != current.commit_head
5399 || rechecked.outcome_floor != current.outcome_floor
5400 {
5401 tokio::task::yield_now().await;
5402 continue;
5403 }
5404 let base_sequence = rechecked.lease.seq;
5405 let sequence = base_sequence
5406 .checked_add(1)
5407 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5408 let mut next = rechecked.preserve_with_lease(LeaderLease {
5409 seq: sequence,
5410 renewal_sequence: rechecked.lease.renewal_sequence,
5411 token: rechecked.lease.token,
5412 owner: rechecked.lease.owner.clone(),
5413 expires_at_ms: rechecked.lease.expires_at_ms,
5414 catalog_manifest: rechecked.lease.catalog_manifest.clone(),
5415 });
5416 next.outcome_floor = Some(floor.clone());
5417 next.validate()?;
5418 match self
5419 .create_authority_record(Some(&published), &next)
5420 .await?
5421 {
5422 AuthorityCreateOutcome::Created => {
5423 self.install_cluster_outcome_audit(
5424 Self::cluster_outcome_audit_key(&next),
5425 next.lease.seq,
5426 Self::outcomes_retained_by_floor(&floor, &snapshot),
5427 );
5428 return Ok(before_epoch);
5429 }
5430 AuthorityCreateOutcome::ExistingIdentical => {
5431 let winner_floor = next.outcome_floor.as_ref().ok_or_else(|| {
5432 LeaseError::Invalid(
5433 "durable floor winner lost its retention boundary".into(),
5434 )
5435 })?;
5436 let winner_snapshot = self.cached_audited_cluster_outcomes_from(&next).await?;
5437 self.preflight_cluster_recovery_cut(
5438 winner_floor,
5439 &winner_snapshot.outcomes,
5440 &validate_artifacts,
5441 )
5442 .await?;
5443 let confirmed = self
5444 .load_record()
5445 .await?
5446 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5447 if !confirmed.lease.matches_proof(proof) {
5448 return Err(ClusterCheckpointAuthorityError::Fenced);
5449 }
5450 if confirmed.outcome_head == next.outcome_head
5451 && confirmed.commit_head == next.commit_head
5452 && confirmed.outcome_floor == next.outcome_floor
5453 {
5454 return Ok(before_epoch);
5455 }
5456 tokio::task::yield_now().await;
5457 }
5458 AuthorityCreateOutcome::Contended(winner) => {
5459 if !winner.lease.matches_proof(proof) {
5460 return Err(ClusterCheckpointAuthorityError::Fenced);
5461 }
5462 if winner.lease.seq <= base_sequence {
5463 return Err(LeaseError::Invalid(
5464 "outcome floor contention did not advance the authority sequence"
5465 .into(),
5466 )
5467 .into());
5468 }
5469 tokio::task::yield_now().await;
5470 }
5471 }
5472 }
5473 }
5474
5475 pub async fn begin_new_term(
5482 &self,
5483 owner: &LeaderLeaseOwner,
5484 now_ms: i64,
5485 ) -> Result<LeaseOutcome, LeaseError> {
5486 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
5487 owner,
5488 now_ms,
5489 SameOwnerToken::Rotate,
5490 ))
5491 .await
5492 }
5493
5494 pub async fn renew_exact(
5500 &self,
5501 owner: &LeaderLeaseOwner,
5502 token: u64,
5503 now_ms: i64,
5504 ) -> Result<LeaseOutcome, LeaseError> {
5505 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
5506 owner,
5507 now_ms,
5508 SameOwnerToken::Exact(token),
5509 ))
5510 .await
5511 }
5512
5513 #[cfg(test)]
5514 pub(crate) async fn acquire_or_renew_current_term_for_test(
5515 &self,
5516 owner: &LeaderLeaseOwner,
5517 now_ms: i64,
5518 ) -> Result<LeaseOutcome, LeaseError> {
5519 self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
5520 .await
5521 }
5522
5523 async fn acquire_or_renew_current_term_for_test_inner(
5524 &self,
5525 owner: &LeaderLeaseOwner,
5526 now_ms: i64,
5527 same_owner_token: SameOwnerToken,
5528 ) -> Result<LeaseOutcome, LeaseError> {
5529 owner.validate()?;
5530 self.ttl()?;
5531 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
5532 loop {
5533 let published = self.load_published_authority_head().await?;
5534 let candidate = match published.as_ref() {
5535 None => {
5536 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5537 return Err(LeaseError::Fenced(
5538 "exact leader renewal lost the durable authority head".into(),
5539 ));
5540 }
5541 LeaderAuthorityRecord::initial(LeaderLease {
5542 seq: 1,
5543 renewal_sequence: 1,
5544 token: 1,
5545 owner: owner.clone(),
5546 expires_at_ms,
5547 catalog_manifest: None,
5548 })
5549 }
5550 Some(head) if head.record.lease.owner == *owner => {
5551 let record = &head.record;
5552 let token = match same_owner_token {
5553 #[cfg(test)]
5554 SameOwnerToken::Preserve => record.lease.token,
5555 SameOwnerToken::Rotate => {
5556 record.lease.token.checked_add(1).ok_or_else(|| {
5557 LeaseError::Invalid("leader fencing token exhausted".into())
5558 })?
5559 }
5560 SameOwnerToken::Exact(expected) if expected == record.lease.token => {
5561 expected
5562 }
5563 SameOwnerToken::Exact(_) => {
5564 return Err(LeaseError::Fenced(
5565 "exact leader renewal was superseded by a newer local term".into(),
5566 ));
5567 }
5568 };
5569 let lease = LeaderLease {
5570 seq: record.lease.seq.checked_add(1).ok_or_else(|| {
5571 LeaseError::Invalid("lease sequence exhausted".into())
5572 })?,
5573 renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
5574 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
5575 )?,
5576 token,
5577 owner: owner.clone(),
5578 expires_at_ms,
5579 catalog_manifest: record.lease.catalog_manifest.clone(),
5580 };
5581 record.preserve_with_lease(lease)
5582 }
5583 Some(head) => {
5584 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5585 return Err(LeaseError::Fenced(
5586 "exact leader renewal was superseded by a rival owner".into(),
5587 ));
5588 }
5589 return Ok(LeaseOutcome::Held(head.record.lease.clone()));
5590 }
5591 };
5592 match self
5593 .create_authority_record(published.as_ref(), &candidate)
5594 .await?
5595 {
5596 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5597 return Ok(LeaseOutcome::Acquired(candidate.lease));
5598 }
5599 AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
5600 if matches!(
5601 same_owner_token,
5602 SameOwnerToken::Exact(expected) if winner.lease.token != expected
5603 ) {
5604 return Err(LeaseError::Fenced(
5605 "exact leader renewal lost a same-owner CAS race".into(),
5606 ));
5607 }
5608 tokio::task::yield_now().await;
5609 }
5610 AuthorityCreateOutcome::Contended(winner) => {
5611 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5612 return Err(LeaseError::Fenced(
5613 "exact leader renewal lost a rival-owner CAS race".into(),
5614 ));
5615 }
5616 return Ok(LeaseOutcome::Held(winner.lease));
5617 }
5618 }
5619 }
5620 }
5621
5622 pub fn observe_rival(
5627 &self,
5628 owner: &LeaderLeaseOwner,
5629 lease: &LeaderLease,
5630 ) -> Result<LeaderLeaseObservation, LeaseError> {
5631 owner.validate()?;
5632 lease.validate()?;
5633 self.ttl()?;
5634 if lease.owner == *owner {
5635 return Err(LeaseError::Invalid(
5636 "leader takeover observation must belong to a rival".into(),
5637 ));
5638 }
5639 Ok(LeaderLeaseObservation {
5640 lease: lease.clone(),
5641 started: Instant::now(),
5642 })
5643 }
5644
5645 pub async fn try_takeover(
5651 &self,
5652 owner: &LeaderLeaseOwner,
5653 observation: &LeaderLeaseObservation,
5654 now_ms: i64,
5655 ) -> Result<LeaseOutcome, LeaseError> {
5656 owner.validate()?;
5657 observation.lease.validate()?;
5658 let ttl = self.ttl()?;
5659 if observation.lease.owner == *owner {
5660 return Err(LeaseError::Invalid(
5661 "leader takeover observation must belong to a rival".into(),
5662 ));
5663 }
5664 if observation.started.elapsed() < ttl {
5665 return Ok(LeaseOutcome::Held(observation.lease.clone()));
5666 }
5667 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
5668 loop {
5669 let published = self
5670 .load_published_authority_head()
5671 .await?
5672 .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
5673 let current = &published.record;
5674 if !current.lease.has_same_liveness_identity(&observation.lease) {
5675 return Ok(LeaseOutcome::Held(current.lease.clone()));
5676 }
5677 let candidate_lease = LeaderLease {
5678 seq: current
5679 .lease
5680 .seq
5681 .checked_add(1)
5682 .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
5683 renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
5684 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
5685 )?,
5686 token: current
5687 .lease
5688 .token
5689 .checked_add(1)
5690 .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
5691 owner: owner.clone(),
5692 expires_at_ms,
5693 catalog_manifest: current.lease.catalog_manifest.clone(),
5694 };
5695 let candidate = current.preserve_with_lease(candidate_lease);
5696 match self
5697 .create_authority_record(Some(&published), &candidate)
5698 .await?
5699 {
5700 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5701 return Ok(LeaseOutcome::Acquired(candidate.lease));
5702 }
5703 AuthorityCreateOutcome::Contended(winner)
5704 if winner.lease.has_same_liveness_identity(&observation.lease) =>
5705 {
5706 tokio::task::yield_now().await;
5707 }
5708 AuthorityCreateOutcome::Contended(winner) => {
5709 return Ok(LeaseOutcome::Held(winner.lease));
5710 }
5711 }
5712 }
5713 }
5714}
5715
5716#[derive(Debug, Clone, Copy)]
5718pub struct LeaderLeaseConfig {
5719 pub ttl: Duration,
5721 pub renew_interval: Duration,
5723}
5724
5725impl Default for LeaderLeaseConfig {
5726 fn default() -> Self {
5727 Self {
5728 ttl: Duration::from_secs(5),
5729 renew_interval: Duration::from_secs(2),
5730 }
5731 }
5732}
5733
5734#[must_use]
5736pub fn lease_grants_leadership(
5737 lease: &Option<LeaderLease>,
5738 owner: &LeaderLeaseOwner,
5739 deadline: &LeaseDeadline,
5740) -> bool {
5741 deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
5742}
5743
5744#[must_use]
5746pub fn lease_grants_proof(
5747 lease: &Option<LeaderLease>,
5748 owner: &LeaderLeaseOwner,
5749 deadline: &LeaseDeadline,
5750 proof: &LeaderProof,
5751) -> bool {
5752 deadline.is_live()
5753 && proof.owner == owner.proof_owner()
5754 && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
5755}
5756
5757pub struct LeaderLeaseManager {
5759 store: Arc<LeaderLeaseStore>,
5760 owner: LeaderLeaseOwner,
5761 config: LeaderLeaseConfig,
5762 lease_tx: watch::Sender<Option<LeaderLease>>,
5763 deadline: Arc<LeaseDeadline>,
5764}
5765
5766#[cfg(feature = "cluster")]
5767enum LeaseOperationEvent {
5768 Shutdown,
5769 Candidacy(Result<(), watch::error::RecvError>),
5770 Deadline,
5771 Completed {
5772 result: Result<LeaseOutcome, LeaseError>,
5773 valid_until: tokio::time::Instant,
5774 },
5775}
5776
5777#[cfg(feature = "cluster")]
5778async fn wait_for_deadline(deadline: Option<tokio::time::Instant>) {
5779 if let Some(deadline) = deadline {
5780 tokio::time::sleep_until(deadline).await;
5781 } else {
5782 std::future::pending::<()>().await;
5783 }
5784}
5785
5786impl std::fmt::Debug for LeaderLeaseManager {
5787 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
5788 formatter
5789 .debug_struct("LeaderLeaseManager")
5790 .field("owner", &self.owner)
5791 .field("config", &self.config)
5792 .finish_non_exhaustive()
5793 }
5794}
5795
5796impl LeaderLeaseManager {
5797 pub fn new(
5802 store: Arc<LeaderLeaseStore>,
5803 process_lease: &ProcessLease,
5804 config: LeaderLeaseConfig,
5805 ) -> Result<Self, LeaseError> {
5806 let owner = LeaderLeaseOwner::from_process_lease(process_lease)?;
5807 let ttl_ms = i64::try_from(config.ttl.as_millis())
5808 .map_err(|_| LeaseError::Invalid("lease TTL exceeds diagnostic range".into()))?;
5809 let exact_ttl = u64::try_from(ttl_ms)
5810 .ok()
5811 .map(Duration::from_millis)
5812 .is_some_and(|ttl| ttl == config.ttl);
5813 if ttl_ms <= 0
5814 || !exact_ttl
5815 || config.renew_interval.is_zero()
5816 || config.renew_interval >= config.ttl
5817 || ttl_ms != store.ttl_ms
5818 {
5819 return Err(LeaseError::Invalid(
5820 "manager requires a renewal interval below the store's matching TTL".into(),
5821 ));
5822 }
5823 let (lease_tx, _lease_rx) = watch::channel(None);
5824 Ok(Self {
5825 store,
5826 owner,
5827 config,
5828 lease_tx,
5829 deadline: Arc::new(LeaseDeadline::uninitialized()),
5830 })
5831 }
5832
5833 #[must_use]
5835 pub fn owner(&self) -> &LeaderLeaseOwner {
5836 &self.owner
5837 }
5838
5839 #[must_use]
5841 pub fn lease_watch(&self) -> watch::Receiver<Option<LeaderLease>> {
5842 self.lease_tx.subscribe()
5843 }
5844
5845 #[must_use]
5847 pub fn deadline(&self) -> Arc<LeaseDeadline> {
5848 Arc::clone(&self.deadline)
5849 }
5850
5851 #[cfg(feature = "cluster")]
5852 fn withdraw(&self) {
5853 self.deadline.withdraw();
5854 self.lease_tx.send_replace(None);
5855 }
5856
5857 #[cfg(feature = "cluster")]
5858 fn fence(&self) {
5859 self.deadline.fence();
5860 self.lease_tx.send_replace(None);
5861 }
5862
5863 #[cfg(feature = "cluster")]
5864 async fn attempt_lease(
5865 &self,
5866 shutdown: &tokio_util::sync::CancellationToken,
5867 candidate: &mut watch::Receiver<LeaderCandidacy>,
5868 valid_until: Option<tokio::time::Instant>,
5869 observation: Option<&LeaderLeaseObservation>,
5870 held_token: Option<u64>,
5871 ) -> LeaseOperationEvent {
5872 let Some(attempt_valid_until) = tokio::time::Instant::now().checked_add(self.config.ttl)
5873 else {
5874 return LeaseOperationEvent::Deadline;
5875 };
5876 let operation_deadline = valid_until.map_or(attempt_valid_until, |current| {
5877 current.min(attempt_valid_until)
5878 });
5879 let operation = async {
5880 if let Some(observation) = observation {
5881 Box::pin(
5882 self.store
5883 .try_takeover(&self.owner, observation, now_millis()),
5884 )
5885 .await
5886 } else if let Some(token) = held_token {
5887 Box::pin(self.store.renew_exact(&self.owner, token, now_millis())).await
5888 } else {
5889 Box::pin(self.store.begin_new_term(&self.owner, now_millis())).await
5890 }
5891 };
5892 tokio::select! {
5893 biased;
5894 () = shutdown.cancelled() => LeaseOperationEvent::Shutdown,
5895 changed = candidate.changed() => LeaseOperationEvent::Candidacy(changed),
5896 () = wait_for_deadline(Some(operation_deadline)) => LeaseOperationEvent::Deadline,
5897 result = operation => LeaseOperationEvent::Completed {
5898 result,
5899 valid_until: attempt_valid_until,
5900 },
5901 }
5902 }
5903
5904 #[cfg(feature = "cluster")]
5905 async fn wait_for_candidacy_change(
5906 &self,
5907 shutdown: &tokio_util::sync::CancellationToken,
5908 candidate: &mut watch::Receiver<LeaderCandidacy>,
5909 ) -> bool {
5910 tokio::select! {
5911 biased;
5912 () = shutdown.cancelled() => {
5913 self.fence();
5914 false
5915 }
5916 changed = candidate.changed() => {
5917 if changed.is_err() {
5918 self.fence();
5919 return false;
5920 }
5921 true
5922 }
5923 }
5924 }
5925
5926 #[cfg(feature = "cluster")]
5927 async fn run(
5928 self,
5929 shutdown: tokio_util::sync::CancellationToken,
5930 mut candidate: watch::Receiver<LeaderCandidacy>,
5931 ) {
5932 let mut ticker = tokio::time::interval(self.config.renew_interval);
5933 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
5934 let mut valid_until: Option<tokio::time::Instant> = None;
5935 let mut observation: Option<LeaderLeaseObservation> = None;
5936 let mut held_token = None;
5939 let mut candidacy_generation = candidate.borrow().generation;
5940
5941 loop {
5942 let candidacy = *candidate.borrow_and_update();
5943 if candidacy.generation != candidacy_generation {
5944 self.withdraw();
5945 observation = None;
5946 valid_until = None;
5947 held_token = None;
5948 candidacy_generation = candidacy.generation;
5949 }
5950 if !candidacy.eligible {
5951 self.withdraw();
5952 observation = None;
5953 valid_until = None;
5954 held_token = None;
5955 if !self
5956 .wait_for_candidacy_change(&shutdown, &mut candidate)
5957 .await
5958 {
5959 return;
5960 }
5961 continue;
5962 }
5963
5964 tokio::select! {
5965 biased;
5966 () = shutdown.cancelled() => {
5967 self.fence();
5968 return;
5969 }
5970 changed = candidate.changed() => {
5971 if changed.is_err() {
5972 self.fence();
5973 return;
5974 }
5975 continue;
5976 }
5977 () = wait_for_deadline(valid_until) => {
5978 self.fence();
5979 return;
5980 }
5981 _ = ticker.tick() => {}
5982 }
5983
5984 let (result, attempt_valid_until) = match Box::pin(self.attempt_lease(
5985 &shutdown,
5986 &mut candidate,
5987 valid_until,
5988 observation.as_ref(),
5989 held_token,
5990 ))
5991 .await
5992 {
5993 LeaseOperationEvent::Shutdown | LeaseOperationEvent::Deadline => {
5994 self.fence();
5995 return;
5996 }
5997 LeaseOperationEvent::Candidacy(changed) => {
5998 if changed.is_err() {
5999 self.fence();
6000 return;
6001 }
6002 continue;
6003 }
6004 LeaseOperationEvent::Completed {
6005 result,
6006 valid_until: attempt_valid_until,
6007 } => {
6008 let response_at = tokio::time::Instant::now();
6009 if response_at >= attempt_valid_until
6010 || valid_until.is_some_and(|current| response_at >= current)
6011 {
6012 self.fence();
6013 return;
6014 }
6015 (result, attempt_valid_until)
6016 }
6017 };
6018
6019 match result {
6020 Ok(LeaseOutcome::Acquired(lease)) if lease.owner == self.owner => {
6021 let publication_at = tokio::time::Instant::now();
6022 if publication_at >= attempt_valid_until
6023 || valid_until.is_some_and(|current| publication_at >= current)
6024 {
6025 self.fence();
6026 return;
6027 }
6028 observation = None;
6029 valid_until = Some(attempt_valid_until);
6030 held_token = Some(lease.token);
6031 self.deadline.extend_until(attempt_valid_until.into_std());
6032 self.lease_tx.send_replace(Some(lease));
6033 }
6034 Ok(LeaseOutcome::Acquired(_)) => {
6035 self.fence();
6036 return;
6037 }
6038 Ok(LeaseOutcome::Held(rival)) => {
6039 self.withdraw();
6040 valid_until = None;
6041 held_token = None;
6042 let unchanged = observation
6043 .as_ref()
6044 .is_some_and(|observed| observed.lease.has_same_liveness_identity(&rival));
6045 if !unchanged {
6046 match self.store.observe_rival(&self.owner, &rival) {
6047 Ok(new_observation) => observation = Some(new_observation),
6048 Err(error) => {
6049 tracing::warn!(%error, "leader lease observation rejected");
6050 observation = None;
6051 }
6052 }
6053 }
6054 }
6055 Err(LeaseError::Fenced(error)) => {
6056 tracing::warn!(%error, "leader lease renewal was fenced");
6057 self.fence();
6058 return;
6059 }
6060 Err(error) => {
6061 tracing::warn!(%error, "leader lease operation failed");
6062 }
6063 }
6064 }
6065 }
6066
6067 #[cfg(feature = "cluster")]
6071 #[must_use]
6072 pub fn spawn(
6073 self,
6074 shutdown: tokio_util::sync::CancellationToken,
6075 candidate: watch::Receiver<LeaderCandidacy>,
6076 ) -> tokio::task::JoinHandle<()> {
6077 tokio::spawn(self.run(shutdown, candidate))
6078 }
6079}
6080
6081#[cfg(test)]
6082mod tests;