1mod artifact_admission;
4mod attempt_status;
5mod subscription_replay;
6
7pub use attempt_status::ClusterAttemptStatus;
8pub use subscription_replay::{
9 SubscriptionReplayPin, SubscriptionReplayPinAcquire, SUBSCRIPTION_REPLAY_PIN_RENEW_INTERVAL,
10};
11
12use std::collections::{BTreeMap, BTreeSet};
13use std::sync::atomic::{AtomicBool, Ordering};
14use std::sync::{Arc, Weak};
15use std::time::{Duration, Instant};
16
17use bytes::Bytes;
18use object_store::path::Path as OsPath;
19use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
20use parking_lot::Mutex;
21use serde::{Deserialize, Serialize};
22use sha2::{Digest as _, Sha256};
23use tokio::sync::watch;
24use tokio_stream::StreamExt;
25use uuid::Uuid;
26
27use crate::checkpoint::{
28 probe_object_store_conditional_update, AssignmentDrainId, AssignmentDrainTransition,
29 CheckpointAssignmentFence, CheckpointStoreError, CommittedCheckpointIndex,
30 CommittedCheckpointRef, LeaderProof, LeaderProofOwner, MAX_CHECKPOINT_PARTICIPANTS,
31};
32use crate::checkpoint_decision::{
33 CheckpointArtifactInventory, CheckpointDecisionStore, CheckpointOutcome, CheckpointScope,
34 CheckpointVerdict, DecisionError, RecordOutcomeResult,
35};
36use crate::cluster::discovery::NodeId;
37
38use super::catalog_manifest::{
39 CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
40};
41use super::controller::{
42 RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
43 RecoveryFaultDisposition, RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
44 MAX_RECOVERY_ANNOUNCEMENT_BYTES,
45};
46use super::lease_deadline::LeaseDeadline;
47use super::process_lease::{ProcessLease, ProcessLeaseFence};
48use super::snapshot::{
49 AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
50};
51
52const LEASE_PREFIX: &str = "control/leader-lease/";
53const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
54const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
55const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
56const AUTHORITY_RECORD_VERSION: u32 = 12;
57const AUTHORITY_HEAD_VERSION: u32 = 1;
58const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
59const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
60const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
61 + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
62 + 2;
63const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
64const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
65const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
66const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
67const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
68const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
69const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
70const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
71const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
72const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
73const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
74const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
75#[cfg(test)]
76const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
77
78fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
79 match error {
80 SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
81 error => LeaseError::Invalid(format!("{context}: {error}")),
82 }
83}
84
85fn lease_path(sequence: u64) -> OsPath {
86 OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
87}
88
89fn authority_head_path() -> OsPath {
90 OsPath::from(AUTHORITY_HEAD_PATH)
91}
92
93fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
94 OsPath::from(format!(
95 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
96 reference.release.generation(),
97 reference.sha256
98 ))
99}
100
101fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
102 let raw = path
103 .as_ref()
104 .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
105 .and_then(|value| value.strip_prefix("generation="))
106 .ok_or_else(|| {
107 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
108 })?;
109 let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
110 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
111 })?;
112 let digest = digest.strip_suffix(".json").ok_or_else(|| {
113 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
114 })?;
115 if generation.len() != 20
116 || !generation.bytes().all(|byte| byte.is_ascii_digit())
117 || digest.len() != 64
118 || !digest
119 .bytes()
120 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
121 {
122 return Err(LeaseError::Invalid(format!(
123 "invalid recovery release terminal path {path}"
124 )));
125 }
126 let generation = generation.parse::<u64>().map_err(|error| {
127 LeaseError::Invalid(format!(
128 "invalid recovery release terminal generation in {path}: {error}"
129 ))
130 })?;
131 if generation == 0 {
132 return Err(LeaseError::Invalid(format!(
133 "invalid recovery release terminal path {path}"
134 )));
135 }
136 Ok((generation, digest.to_owned()))
137}
138
139fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
140 let raw = path
141 .as_ref()
142 .strip_prefix(LEASE_PREFIX)
143 .and_then(|file| file.strip_prefix('v'))
144 .and_then(|file| file.strip_suffix(".json"))
145 .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
146 if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
147 return Err(LeaseError::Invalid(format!(
148 "invalid leader authority sequence in {path}"
149 )));
150 }
151 let sequence = raw.parse::<u64>().map_err(|error| {
152 LeaseError::Invalid(format!(
153 "invalid leader authority sequence in {path}: {error}"
154 ))
155 })?;
156 if sequence == 0 || lease_path(sequence) != *path {
157 return Err(LeaseError::Invalid(format!(
158 "noncanonical leader authority path {path}"
159 )));
160 }
161 Ok(sequence)
162}
163
164fn consume_live_authority_link(traversed: &mut usize) -> bool {
165 if *traversed == MAX_LIVE_AUTHORITY_LINKS {
166 return false;
167 }
168 *traversed += 1;
169 true
170}
171
172fn now_millis() -> i64 {
173 std::time::SystemTime::now()
174 .duration_since(std::time::UNIX_EPOCH)
175 .ok()
176 .and_then(|duration| i64::try_from(duration.as_millis()).ok())
177 .unwrap_or(i64::MAX)
178}
179
180async fn read_authority_record(
181 store: &dyn ObjectStore,
182 sequence: u64,
183) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
184 let result = match store.get(&lease_path(sequence)).await {
185 Ok(result) => result,
186 Err(object_store::Error::NotFound { .. }) => return Ok(None),
187 Err(error) => return Err(LeaseError::Io(error.to_string())),
188 };
189 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
190 return Err(LeaseError::Invalid(format!(
191 "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
192 result.meta.size
193 )));
194 }
195 let bytes = result
196 .bytes()
197 .await
198 .map_err(|error| LeaseError::Io(error.to_string()))?;
199 let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
200 record.validate()?;
201 if record.lease.seq != sequence {
202 return Err(LeaseError::Invalid(
203 "authority record sequence does not match its object name".into(),
204 ));
205 }
206 let canonical = serde_json::to_vec(&record)?;
207 if canonical.as_slice() != bytes.as_ref() {
208 return Err(LeaseError::Invalid(format!(
209 "leader authority record {sequence} does not use its canonical body"
210 )));
211 }
212 Ok(Some(record))
213}
214
215#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
216#[serde(deny_unknown_fields)]
217struct AuthorityHeadPointer {
218 version: u32,
219 sequence: u64,
220 nonce: Uuid,
221}
222
223impl AuthorityHeadPointer {
224 fn new(sequence: u64) -> Result<Self, LeaseError> {
225 let pointer = Self {
226 version: AUTHORITY_HEAD_VERSION,
227 sequence,
228 nonce: Uuid::new_v4(),
229 };
230 pointer.validate()?;
231 Ok(pointer)
232 }
233
234 fn validate(self) -> Result<(), LeaseError> {
235 if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
236 return Err(LeaseError::Invalid(
237 "leader authority head has an unsupported version, zero sequence, or nil nonce"
238 .into(),
239 ));
240 }
241 Ok(())
242 }
243}
244
245#[derive(Clone)]
246struct VersionedAuthorityHeadPointer {
247 pointer: AuthorityHeadPointer,
248 update_version: UpdateVersion,
249}
250
251struct PublishedAuthorityHead {
252 record: LeaderAuthorityRecord,
253 pointer: VersionedAuthorityHeadPointer,
254}
255
256async fn read_authority_head_pointer(
257 store: &dyn ObjectStore,
258) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
259 let result = match store.get(&authority_head_path()).await {
260 Ok(result) => result,
261 Err(object_store::Error::NotFound { .. }) => return Ok(None),
262 Err(error) => return Err(LeaseError::Io(error.to_string())),
263 };
264 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
265 return Err(LeaseError::Invalid(format!(
266 "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
267 result.meta.size
268 )));
269 }
270 let update_version = UpdateVersion {
271 e_tag: result.meta.e_tag.clone(),
272 version: result.meta.version.clone(),
273 };
274 let bytes = result
275 .bytes()
276 .await
277 .map_err(|error| LeaseError::Io(error.to_string()))?;
278 let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
279 pointer.validate()?;
280 let canonical = serde_json::to_vec(&pointer)?;
281 if canonical.as_slice() != bytes.as_ref() {
282 return Err(LeaseError::Invalid(
283 "leader authority head does not use its canonical body".into(),
284 ));
285 }
286 Ok(Some(VersionedAuthorityHeadPointer {
287 pointer,
288 update_version,
289 }))
290}
291
292fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
293 let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
294 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
295 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
296 return Err(LeaseError::Invalid(format!(
297 "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
298 encoded.len()
299 )));
300 }
301 Ok(Bytes::from(encoded))
302}
303
304fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
305 record.validate()?;
306 let encoded = serde_json::to_vec(record)?;
307 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
308 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
309 return Err(LeaseError::Invalid(format!(
310 "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
311 encoded.len()
312 )));
313 }
314 if record.recovery_fault_slots.iter().any(|slot| slot.active) {
315 ensure_recovery_fault_authority_headroom(encoded_len)?;
316 }
317 Ok(Bytes::from(encoded))
318}
319
320fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
321 let limit = MAX_AUTHORITY_RECORD_BYTES
322 .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
323 .expect("recovery fault headroom must fit the authority record");
324 if encoded_len > limit {
325 return Err(LeaseError::Invalid(format!(
326 "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
327 )));
328 }
329 Ok(())
330}
331
332#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
334pub struct LeaderLeaseOwner {
335 pub node: NodeId,
337 pub boot: Uuid,
339 pub process_term: u64,
341}
342
343impl LeaderLeaseOwner {
344 fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
345 process
346 .validate(process.node)
347 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
348 Ok(Self {
349 node: process.node,
350 boot: process.owner,
351 process_term: process.term,
352 })
353 }
354
355 fn validate(&self) -> Result<(), LeaseError> {
356 if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
357 return Err(LeaseError::Invalid(
358 "leader owner node, boot identity, and process term must be nonzero".into(),
359 ));
360 }
361 Ok(())
362 }
363
364 fn proof_owner(&self) -> LeaderProofOwner {
365 LeaderProofOwner {
366 node_id: self.node.0,
367 boot_id: self.boot,
368 process_term: self.process_term,
369 }
370 }
371}
372
373#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
375pub struct LeaderLease {
376 pub seq: u64,
378 pub renewal_sequence: u64,
380 pub token: u64,
382 pub owner: LeaderLeaseOwner,
384 pub expires_at_ms: i64,
386 pub catalog_manifest: Option<CatalogManifestRef>,
388}
389
390#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
391#[serde(deny_unknown_fields)]
392struct OutcomeLink {
393 sequence: u64,
394 epoch: u64,
395 checkpoint_id: u64,
396}
397
398impl OutcomeLink {
399 fn validate(self) -> Result<(), LeaseError> {
400 let attempt = crate::checkpoint::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
401 if self.sequence == 0 || !attempt.is_canonical() {
402 return Err(LeaseError::Invalid(
403 "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
404 .into(),
405 ));
406 }
407 Ok(())
408 }
409}
410
411#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
412#[serde(deny_unknown_fields)]
413struct AssignmentDecisionLink {
414 sequence: u64,
415 target_version: u64,
416}
417
418#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
419#[serde(deny_unknown_fields)]
420struct AuthorityRecoveryFaultSlot {
421 publisher: RecoveryFaultPublisher,
422 request_sequence: u64,
423 fault_sequence: u64,
424 #[serde(
425 default,
426 skip_serializing_if = "RecoveryFaultDisposition::is_recoverable"
427 )]
428 disposition: RecoveryFaultDisposition,
429 active: bool,
430}
431
432impl AuthorityRecoveryFaultSlot {
433 fn validate(&self) -> Result<(), LeaseError> {
434 self.publisher.validate().map_err(LeaseError::Invalid)?;
435 if self.request_sequence == 0 || self.fault_sequence == 0 {
436 return Err(LeaseError::Invalid(
437 "recovery fault request and authority sequence must be nonzero".into(),
438 ));
439 }
440 if self.disposition == RecoveryFaultDisposition::Terminal && !self.active {
441 return Err(LeaseError::Invalid(
442 "terminal recovery fault authority cannot be tombstoned".into(),
443 ));
444 }
445 Ok(())
446 }
447
448 fn fault(&self) -> RecoveryFault {
449 RecoveryFault {
450 reporter: NodeId(self.publisher.participant.node_id),
451 sequence: self.fault_sequence,
452 disposition: self.disposition,
453 }
454 }
455
456 fn matches_request(
457 &self,
458 publisher: RecoveryFaultPublisher,
459 request_sequence: u64,
460 disposition: RecoveryFaultDisposition,
461 ) -> bool {
462 self.publisher == publisher
463 && self.request_sequence == request_sequence
464 && self.disposition == disposition
465 }
466}
467
468#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
469#[serde(deny_unknown_fields)]
470pub(crate) struct RecoveryReleaseTerminalRef {
471 release: RecoveryReleaseId,
472 sha256: String,
473 encoded_len: u64,
474}
475
476impl RecoveryReleaseTerminalRef {
477 fn validate(&self) -> Result<(), LeaseError> {
478 if !self.release.is_canonical()
479 || self.sha256.len() != 64
480 || !self
481 .sha256
482 .bytes()
483 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
484 || self.encoded_len == 0
485 || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
486 {
487 return Err(LeaseError::Invalid(
488 "recovery release terminal reference is not canonical".into(),
489 ));
490 }
491 Ok(())
492 }
493
494 pub(crate) fn generation(&self) -> u64 {
495 self.release.generation()
496 }
497}
498
499#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
500#[serde(deny_unknown_fields)]
501struct AuthorityRecoveryReleaseCommit {
502 terminal: RecoveryReleaseTerminalRef,
503 leader_proof: LeaderProof,
504}
505
506impl AuthorityRecoveryReleaseCommit {
507 fn validate(&self) -> Result<(), LeaseError> {
508 self.terminal.validate()?;
509 if !self.leader_proof.is_canonical() {
510 return Err(LeaseError::Invalid(
511 "recovery release commit has no canonical leader proof".into(),
512 ));
513 }
514 Ok(())
515 }
516}
517
518#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
519#[serde(deny_unknown_fields)]
520pub(crate) struct RecoveryReleaseLink {
521 sequence: u64,
522 terminal: RecoveryReleaseTerminalRef,
523}
524
525impl RecoveryReleaseLink {
526 fn validate(&self) -> Result<(), LeaseError> {
527 if self.sequence == 0 {
528 return Err(LeaseError::Invalid(
529 "recovery release authority link has a zero sequence".into(),
530 ));
531 }
532 self.terminal.validate()
533 }
534}
535
536#[derive(Debug, Clone, PartialEq, Eq)]
537pub(crate) enum RecordRecoveryReleaseCommitResult {
538 Created(RecoveryReleaseTerminalRef),
539 Unchanged(RecoveryReleaseTerminalRef),
540 Conflict { winner: RecoveryReleaseTerminalRef },
541 FaultsChanged,
542}
543
544#[derive(Debug, Clone, Copy, PartialEq, Eq)]
545pub(crate) enum RecordRecoveryFaultResult {
546 Active,
547 AlreadyCleared,
548 CoveredByNewerRequest,
549 TerminalFenceActive,
550 Superseded,
551}
552
553fn recovery_release_id_for_terminal(
554 terminal: &RecoveryAnnouncement,
555) -> Result<RecoveryReleaseId, LeaseError> {
556 terminal.validate().map_err(LeaseError::Invalid)?;
557 let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
558 return Err(LeaseError::Invalid(
559 "recovery release terminal must carry ReleaseCommitted".into(),
560 ));
561 };
562 RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
563 round: terminal.round.clone(),
564 phase: RecoverPhase::Release { epoch },
565 })
566 .map_err(LeaseError::Invalid)
567}
568
569fn encode_recovery_release_terminal(
570 terminal: &RecoveryAnnouncement,
571) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
572 let release = recovery_release_id_for_terminal(terminal)?;
573 let encoded = serde_json::to_vec(terminal)?;
574 let encoded_len = u64::try_from(encoded.len())
575 .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
576 if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
577 return Err(LeaseError::Invalid(format!(
578 "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
579 )));
580 }
581 let reference = RecoveryReleaseTerminalRef {
582 release,
583 sha256: format!("{:x}", Sha256::digest(&encoded)),
584 encoded_len,
585 };
586 reference.validate()?;
587 Ok((Bytes::from(encoded), reference))
588}
589
590#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
592#[serde(deny_unknown_fields)]
593pub struct AssignmentDrainDecision {
594 pub transition: AssignmentDrainTransition,
596 pub leader_proof: LeaderProof,
598 pub verdict: AssignmentDrainVerdict,
600 pub handoff_checkpoint: Option<CommittedCheckpointRef>,
603}
604
605impl AssignmentDrainDecision {
606 pub fn commit(
611 transition: &AssignmentDrainTransition,
612 leader_proof: LeaderProof,
613 handoff_checkpoint: CommittedCheckpointRef,
614 ) -> Result<Self, String> {
615 handoff_checkpoint.validate()?;
616 if !transition.is_canonical() || leader_proof != transition.leader {
617 return Err("assignment drain decision requires a canonical transition".into());
618 }
619 Ok(Self {
620 transition: transition.clone(),
621 leader_proof,
622 verdict: AssignmentDrainVerdict::Commit,
623 handoff_checkpoint: Some(handoff_checkpoint),
624 })
625 }
626
627 pub fn abort(
632 transition: &AssignmentDrainTransition,
633 leader_proof: LeaderProof,
634 ) -> Result<Self, String> {
635 if !transition.is_canonical() || !leader_proof.is_canonical() {
636 return Err("assignment drain decision requires a canonical transition".into());
637 }
638 Ok(Self {
639 transition: transition.clone(),
640 leader_proof,
641 verdict: AssignmentDrainVerdict::Abort,
642 handoff_checkpoint: None,
643 })
644 }
645
646 #[must_use]
648 pub fn round(&self) -> AssignmentDrainId {
649 self.transition.id()
650 }
651
652 #[must_use]
654 pub fn target_version(&self) -> u64 {
655 self.transition.target.assignment_version
656 }
657
658 fn validate(&self) -> Result<(), LeaseError> {
659 if !self.transition.is_canonical()
660 || !self.leader_proof.is_canonical()
661 || (self.verdict == AssignmentDrainVerdict::Commit
662 && self.leader_proof != self.transition.leader)
663 {
664 return Err(LeaseError::Invalid(
665 "assignment drain decision is not canonical".into(),
666 ));
667 }
668 match (self.verdict, self.handoff_checkpoint.as_ref()) {
669 (AssignmentDrainVerdict::Commit, Some(reference)) => {
670 reference.validate().map_err(LeaseError::Invalid)?;
671 }
672 (AssignmentDrainVerdict::Abort, None) => {}
673 _ => {
674 return Err(LeaseError::Invalid(
675 "assignment drain decision has an invalid handoff checkpoint".into(),
676 ));
677 }
678 }
679 Ok(())
680 }
681}
682
683#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
685#[serde(rename_all = "snake_case")]
686pub enum AssignmentDrainVerdict {
687 Commit,
689 Abort,
691}
692
693#[derive(Debug, Clone, PartialEq, Eq)]
695pub enum RecordAssignmentDrainDecisionResult {
696 Created(AssignmentDrainDecision),
698 Unchanged(AssignmentDrainDecision),
700 Conflict {
702 winner: AssignmentDrainDecision,
704 },
705}
706
707#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
709#[serde(deny_unknown_fields)]
710pub struct AssignmentRecoveryDecision {
711 pub predecessor: CheckpointAssignmentFence,
713 pub target: CheckpointAssignmentFence,
715 pub proposal: AssignmentSnapshotRef,
717 pub removed_process_fences: Vec<ProcessLeaseFence>,
719 pub recovery_checkpoint: CommittedCheckpointRef,
721 pub leader_proof: LeaderProof,
723}
724
725impl AssignmentRecoveryDecision {
726 pub fn new(
731 predecessor: CheckpointAssignmentFence,
732 target: CheckpointAssignmentFence,
733 proposal: AssignmentSnapshotRef,
734 removed_process_fences: Vec<ProcessLeaseFence>,
735 recovery_checkpoint: CommittedCheckpointRef,
736 leader_proof: LeaderProof,
737 ) -> Result<Self, String> {
738 let decision = Self {
739 predecessor,
740 target,
741 proposal,
742 removed_process_fences,
743 recovery_checkpoint,
744 leader_proof,
745 };
746 decision.validate().map_err(|error| error.to_string())?;
747 Ok(decision)
748 }
749
750 #[must_use]
752 pub fn target_version(&self) -> u64 {
753 self.target.assignment_version
754 }
755
756 fn validate(&self) -> Result<(), LeaseError> {
757 if !self.predecessor.is_canonical()
758 || !self.target.is_canonical()
759 || !self.leader_proof.is_canonical()
760 || self.predecessor.vnode_count != self.target.vnode_count
761 || self.predecessor.assignment_version.checked_add(1)
762 != Some(self.target.assignment_version)
763 || self.proposal.validate().is_err()
764 || self.proposal.version != self.target.assignment_version
765 || self.recovery_checkpoint.validate().is_err()
766 {
767 return Err(LeaseError::Invalid(
768 "assignment recovery decision is not a canonical successor".into(),
769 ));
770 }
771
772 let removed: Vec<_> = self
773 .predecessor
774 .participants
775 .iter()
776 .filter(|participant| {
777 self.target.participant_incarnation(participant.node_id)
778 != Some(participant.boot_incarnation)
779 })
780 .collect();
781 if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
782 return Err(LeaseError::Invalid(
783 "assignment recovery decision requires the exact removed predecessor process set"
784 .into(),
785 ));
786 }
787 for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
788 if !fence.is_canonical()
789 || fence.predecessor.node.0 != participant.node_id
790 || fence.predecessor.owner != participant.boot_incarnation
791 {
792 return Err(LeaseError::Invalid(
793 "assignment recovery process fences are incomplete, unsorted, or unrelated"
794 .into(),
795 ));
796 }
797 }
798 Ok(())
799 }
800}
801
802#[derive(Debug, Clone, PartialEq, Eq)]
804pub enum RecordAssignmentRecoveryDecisionResult {
805 Created(AssignmentRecoveryDecision),
807 Unchanged(AssignmentRecoveryDecision),
809 Conflict {
811 winner: AssignmentRecoveryDecision,
813 },
814}
815
816#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
817#[serde(
818 tag = "kind",
819 content = "decision",
820 rename_all = "snake_case",
821 deny_unknown_fields
822)]
823enum AuthorityAssignmentDecision {
824 Drain(AssignmentDrainDecision),
825 Recovery(AssignmentRecoveryDecision),
826}
827
828impl AuthorityAssignmentDecision {
829 fn target_version(&self) -> u64 {
830 match self {
831 Self::Drain(decision) => decision.target_version(),
832 Self::Recovery(decision) => decision.target_version(),
833 }
834 }
835
836 fn predecessor(&self) -> &CheckpointAssignmentFence {
837 match self {
838 Self::Drain(decision) => &decision.transition.predecessor,
839 Self::Recovery(decision) => &decision.predecessor,
840 }
841 }
842
843 fn materialized_target(&self) -> CheckpointAssignmentFence {
844 match self {
845 Self::Drain(decision) => match decision.verdict {
846 AssignmentDrainVerdict::Commit => decision.transition.target.clone(),
847 AssignmentDrainVerdict::Abort => {
848 let mut target = decision.transition.predecessor.clone();
849 target.assignment_version = decision.transition.target.assignment_version;
850 target
851 }
852 },
853 Self::Recovery(decision) => decision.target.clone(),
854 }
855 }
856
857 fn leader_proof(&self) -> &LeaderProof {
858 match self {
859 Self::Drain(decision) => &decision.leader_proof,
860 Self::Recovery(decision) => &decision.leader_proof,
861 }
862 }
863
864 fn validate(&self) -> Result<(), LeaseError> {
865 match self {
866 Self::Drain(decision) => decision.validate(),
867 Self::Recovery(decision) => decision.validate(),
868 }
869 }
870}
871
872enum RecordAuthorityAssignmentDecisionResult {
873 Created(AuthorityAssignmentDecision),
874 Unchanged(AuthorityAssignmentDecision),
875 Conflict { winner: AuthorityAssignmentDecision },
876}
877
878#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
880#[serde(rename_all = "snake_case")]
881pub enum ClusterArtifactCleanupPhase {
882 DeleteData,
884 DeleteMetadata,
886}
887
888#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
890#[serde(deny_unknown_fields)]
891pub struct ClusterArtifactCleanupCursor {
892 pub protected: CommittedCheckpointRef,
894 pub current: CommittedCheckpointRef,
896 pub next: Option<CommittedCheckpointRef>,
898 pub stop_before: Option<CommittedCheckpointRef>,
900 pub participant_ids: Vec<u64>,
902 pub phase: ClusterArtifactCleanupPhase,
904}
905
906impl ClusterArtifactCleanupCursor {
907 fn validate(&self) -> Result<(), LeaseError> {
908 for (name, reference) in [
909 ("protected", Some(&self.protected)),
910 ("current", Some(&self.current)),
911 ("next", self.next.as_ref()),
912 ("stop-before", self.stop_before.as_ref()),
913 ] {
914 if let Some(reference) = reference {
915 reference.validate().map_err(|error| {
916 LeaseError::Invalid(format!(
917 "cluster artifact cleanup {name} reference: {error}"
918 ))
919 })?;
920 }
921 }
922 if self.current.epoch >= self.protected.epoch
923 || self.current.checkpoint_id >= self.protected.checkpoint_id
924 {
925 return Err(LeaseError::Invalid(
926 "cluster artifact cleanup current checkpoint is not older than its protected cut"
927 .into(),
928 ));
929 }
930 if let Some(next) = self.next.as_ref() {
931 if next.epoch >= self.current.epoch || next.checkpoint_id >= self.current.checkpoint_id
932 {
933 return Err(LeaseError::Invalid(
934 "cluster artifact cleanup next checkpoint is not older than its current checkpoint"
935 .into(),
936 ));
937 }
938 }
939 if let Some(stop) = self.stop_before.as_ref() {
940 if stop.epoch >= self.current.epoch || stop.checkpoint_id >= self.current.checkpoint_id
941 {
942 return Err(LeaseError::Invalid(
943 "cluster artifact cleanup stop boundary is not older than its current checkpoint"
944 .into(),
945 ));
946 }
947 let next = self.next.as_ref().ok_or_else(|| {
948 LeaseError::Invalid(
949 "cluster artifact cleanup lost the path to its stop boundary".into(),
950 )
951 })?;
952 if next.epoch < stop.epoch || next.checkpoint_id < stop.checkpoint_id {
953 return Err(LeaseError::Invalid(
954 "cluster artifact cleanup next checkpoint crosses its stop boundary".into(),
955 ));
956 }
957 }
958 if self.participant_ids.is_empty()
959 || self.participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
960 || self.participant_ids[0] == 0
961 || !self
962 .participant_ids
963 .windows(2)
964 .all(|pair| pair[0] < pair[1])
965 {
966 return Err(LeaseError::Invalid(
967 "cluster artifact cleanup participants are not canonical".into(),
968 ));
969 }
970 Ok(())
971 }
972}
973
974#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
975#[serde(deny_unknown_fields)]
976struct AuthorityOutcomeFloor {
977 deployment_id: String,
978 artifact_before_epoch: u64,
979 authority_before_epoch: u64,
980 terminal_anchor: Option<CheckpointOutcome>,
981 terminal_anchor_link: Option<OutcomeLink>,
982 committed_anchor: Option<CheckpointOutcome>,
983 committed_anchor_link: Option<OutcomeLink>,
984}
985
986#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
987#[serde(deny_unknown_fields)]
988struct AuthorityAssignmentDecisionFloor {
989 before_target_version: u64,
990 terminal_anchor: Option<AuthorityAssignmentDecision>,
991 terminal_anchor_link: Option<AssignmentDecisionLink>,
992}
993
994#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
995#[serde(deny_unknown_fields)]
996struct AssignmentHandoffPin {
997 target: CheckpointAssignmentFence,
998 checkpoint: CommittedCheckpointRef,
999}
1000
1001impl AssignmentHandoffPin {
1002 fn validate(&self) -> Result<(), LeaseError> {
1003 if !self.target.is_canonical() {
1004 return Err(LeaseError::Invalid(
1005 "assignment handoff pin has an invalid target fence".into(),
1006 ));
1007 }
1008 self.checkpoint.validate().map_err(LeaseError::Invalid)
1009 }
1010}
1011
1012#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1014#[serde(deny_unknown_fields)]
1015struct LeaderAuthorityRecord {
1016 version: u32,
1017 lease: LeaderLease,
1018 checkpoint_outcome: Option<CheckpointOutcome>,
1020 previous_outcome: Option<OutcomeLink>,
1022 outcome_head: Option<OutcomeLink>,
1024 previous_commit: Option<OutcomeLink>,
1026 commit_head: Option<OutcomeLink>,
1028 outcome_floor: Option<AuthorityOutcomeFloor>,
1030 artifact_cleanup: Option<ClusterArtifactCleanupCursor>,
1032 #[serde(default, skip_serializing_if = "Option::is_none")]
1034 subscription_cleanup_commit: Option<subscription_replay::SubscriptionCleanupCommit>,
1035 active_checkpoint_artifacts: Option<CheckpointArtifactInventory>,
1037 active_checkpoint_artifact_leader_proof: Option<LeaderProof>,
1039 assignment_decision: Option<AuthorityAssignmentDecision>,
1041 previous_assignment_decision: Option<AssignmentDecisionLink>,
1043 assignment_decision_head: Option<AssignmentDecisionLink>,
1045 assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
1047 assignment_handoff_pin: Option<AssignmentHandoffPin>,
1049 recovery_fault_revision: u64,
1051 recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
1053 recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
1055 recovery_release_head: Option<RecoveryReleaseLink>,
1057}
1058
1059impl LeaderLease {
1060 fn validate(&self) -> Result<(), LeaseError> {
1061 self.owner.validate()?;
1062 if self.seq == 0
1063 || self.renewal_sequence == 0
1064 || self.renewal_sequence > self.seq
1065 || self.token == 0
1066 {
1067 return Err(LeaseError::Invalid(
1068 "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
1069 .into(),
1070 ));
1071 }
1072 if let Some(reference) = &self.catalog_manifest {
1073 reference
1074 .validate()
1075 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1076 }
1077 Ok(())
1078 }
1079
1080 #[must_use]
1082 pub fn proof(&self) -> LeaderProof {
1083 LeaderProof {
1084 owner: self.owner.proof_owner(),
1085 fencing_token: self.token,
1086 }
1087 }
1088
1089 #[must_use]
1091 pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
1092 proof.is_canonical()
1093 && proof.owner == self.owner.proof_owner()
1094 && proof.fencing_token == self.token
1095 }
1096
1097 fn has_same_liveness_identity(&self, other: &Self) -> bool {
1098 self.owner == other.owner
1099 && self.token == other.token
1100 && self.renewal_sequence == other.renewal_sequence
1101 }
1102}
1103
1104impl AuthorityOutcomeFloor {
1105 fn validate(&self) -> Result<(), LeaseError> {
1106 let deployment = Uuid::parse_str(&self.deployment_id)
1107 .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
1108 if deployment.is_nil()
1109 || deployment.to_string() != self.deployment_id
1110 || self.authority_before_epoch == 0
1111 || self.artifact_before_epoch > self.authority_before_epoch
1112 {
1113 return Err(LeaseError::Invalid(
1114 "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
1115 .into(),
1116 ));
1117 }
1118 for link in [self.terminal_anchor_link, self.committed_anchor_link]
1119 .into_iter()
1120 .flatten()
1121 {
1122 link.validate()?;
1123 }
1124 for (name, anchor) in [
1125 ("terminal", self.terminal_anchor.as_ref()),
1126 ("committed", self.committed_anchor.as_ref()),
1127 ] {
1128 let Some(anchor) = anchor else { continue };
1129 anchor
1130 .validate_shape(anchor.epoch)
1131 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1132 if anchor.scope != CheckpointScope::Cluster
1133 || anchor.deployment_id != self.deployment_id
1134 {
1135 return Err(LeaseError::Invalid(format!(
1136 "outcome floor has an invalid {name} anchor"
1137 )));
1138 }
1139 }
1140 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1141 (Some(anchor), Some(link))
1142 if anchor.epoch < self.authority_before_epoch
1143 && link.sequence != 0
1144 && link.epoch == anchor.epoch
1145 && link.checkpoint_id == anchor.checkpoint_id => {}
1146 (None, None) => {}
1147 _ => {
1148 return Err(LeaseError::Invalid(
1149 "outcome floor terminal anchor does not match its exact authority link".into(),
1150 ));
1151 }
1152 }
1153 match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
1154 (Some(anchor), Some(link))
1155 if self.artifact_before_epoch != 0
1156 && anchor.is_commit()
1157 && anchor.epoch < self.artifact_before_epoch
1158 && link.sequence != 0
1159 && link.epoch == anchor.epoch
1160 && link.checkpoint_id == anchor.checkpoint_id => {}
1161 (None, None) => {}
1162 _ => {
1163 return Err(LeaseError::Invalid(
1164 "outcome floor committed anchor does not match its exact Commit link".into(),
1165 ));
1166 }
1167 }
1168 if let (Some(committed), Some(terminal)) = (
1169 self.committed_anchor.as_ref(),
1170 self.terminal_anchor.as_ref(),
1171 ) {
1172 let ordered = if committed.epoch == terminal.epoch {
1173 committed == terminal
1174 } else {
1175 committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
1176 };
1177 if !ordered {
1178 return Err(LeaseError::Invalid(
1179 "outcome floor anchors are not monotonically ordered".into(),
1180 ));
1181 }
1182 let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
1183 (Some(committed_link), Some(terminal_link)) if committed == terminal => {
1184 committed_link == terminal_link
1185 }
1186 (Some(committed_link), Some(terminal_link)) => {
1187 committed_link.sequence < terminal_link.sequence
1188 }
1189 _ => false,
1190 };
1191 if !links_ordered {
1192 return Err(LeaseError::Invalid(
1193 "outcome floor anchor links are not ordered with their exact outcomes".into(),
1194 ));
1195 }
1196 } else if self.committed_anchor.is_some() {
1197 return Err(LeaseError::Invalid(
1198 "outcome floor has a committed anchor without terminal continuity".into(),
1199 ));
1200 }
1201 Ok(())
1202 }
1203}
1204
1205impl AuthorityAssignmentDecisionFloor {
1206 fn validate(&self) -> Result<(), LeaseError> {
1207 if self.before_target_version == 0 {
1208 return Err(LeaseError::Invalid(
1209 "assignment decision floor requires a nonzero target version".into(),
1210 ));
1211 }
1212 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1213 (Some(anchor), Some(link)) => {
1214 anchor.validate()?;
1215 if link.sequence == 0
1216 || link.target_version != anchor.target_version()
1217 || anchor.target_version() >= self.before_target_version
1218 {
1219 return Err(LeaseError::Invalid(
1220 "assignment decision floor anchor does not match its exact authority link"
1221 .into(),
1222 ));
1223 }
1224 }
1225 (None, None) => {}
1226 _ => {
1227 return Err(LeaseError::Invalid(
1228 "assignment decision floor has an incomplete terminal anchor".into(),
1229 ));
1230 }
1231 }
1232 Ok(())
1233 }
1234}
1235
1236impl LeaderAuthorityRecord {
1237 fn initial(lease: LeaderLease) -> Self {
1238 Self {
1239 version: AUTHORITY_RECORD_VERSION,
1240 lease,
1241 checkpoint_outcome: None,
1242 previous_outcome: None,
1243 outcome_head: None,
1244 previous_commit: None,
1245 commit_head: None,
1246 outcome_floor: None,
1247 artifact_cleanup: None,
1248 subscription_cleanup_commit: None,
1249 active_checkpoint_artifacts: None,
1250 active_checkpoint_artifact_leader_proof: None,
1251 assignment_decision: None,
1252 previous_assignment_decision: None,
1253 assignment_decision_head: None,
1254 assignment_decision_floor: None,
1255 assignment_handoff_pin: None,
1256 recovery_fault_revision: 0,
1257 recovery_fault_slots: Vec::new(),
1258 recovery_release_commit: None,
1259 recovery_release_head: None,
1260 }
1261 }
1262
1263 fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1264 Self {
1265 version: AUTHORITY_RECORD_VERSION,
1266 lease,
1267 checkpoint_outcome: None,
1268 previous_outcome: None,
1269 outcome_head: self.outcome_head,
1270 previous_commit: None,
1271 commit_head: self.commit_head,
1272 outcome_floor: self.outcome_floor.clone(),
1273 artifact_cleanup: self.artifact_cleanup.clone(),
1274 subscription_cleanup_commit: self.subscription_cleanup_commit.clone(),
1275 active_checkpoint_artifacts: self.active_checkpoint_artifacts.clone(),
1276 active_checkpoint_artifact_leader_proof: self
1277 .active_checkpoint_artifact_leader_proof
1278 .clone(),
1279 assignment_decision: None,
1280 previous_assignment_decision: None,
1281 assignment_decision_head: self.assignment_decision_head,
1282 assignment_decision_floor: self.assignment_decision_floor.clone(),
1283 assignment_handoff_pin: self.assignment_handoff_pin.clone(),
1284 recovery_fault_revision: self.recovery_fault_revision,
1285 recovery_fault_slots: self.recovery_fault_slots.clone(),
1286 recovery_release_commit: None,
1287 recovery_release_head: self.recovery_release_head.clone(),
1288 }
1289 }
1290
1291 fn validate(&self) -> Result<(), LeaseError> {
1292 if self.version != AUTHORITY_RECORD_VERSION {
1293 return Err(LeaseError::Invalid(format!(
1294 "authority record version {} is unsupported",
1295 self.version
1296 )));
1297 }
1298 self.lease.validate()?;
1299 for link in [
1300 self.previous_outcome,
1301 self.outcome_head,
1302 self.previous_commit,
1303 self.commit_head,
1304 ]
1305 .into_iter()
1306 .flatten()
1307 {
1308 link.validate()?;
1309 }
1310 self.subscription_cleanup_commit
1311 .iter()
1312 .try_for_each(subscription_replay::SubscriptionCleanupCommit::validate)?;
1313 self.validate_recovery_fault_inventory()?;
1314 self.validate_checkpoint_outcome_chain()?;
1315 self.validate_outcome_floor()?;
1316 self.validate_artifact_cleanup()?;
1317 self.validate_checkpoint_artifact_inventory()?;
1318 self.validate_assignment_decision_chain()?;
1319 self.validate_assignment_decision_floor()?;
1320 self.validate_assignment_handoff_pin()?;
1321 self.validate_recovery_release()?;
1322 Ok(())
1323 }
1324
1325 fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1326 if self.recovery_fault_slots.windows(2).any(|pair| {
1327 pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1328 }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1329 || (self.recovery_fault_revision != 0
1330 && self.recovery_fault_slots.is_empty()
1331 && self
1332 .recovery_release_head
1333 .as_ref()
1334 .map(|head| head.sequence)
1335 != Some(self.recovery_fault_revision))
1336 || self.recovery_fault_revision > self.lease.seq
1337 || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1338 {
1339 return Err(LeaseError::Invalid(
1340 "leader authority recovery-fault inventory is not canonical".into(),
1341 ));
1342 }
1343 let mut fault_sequences = BTreeSet::new();
1344 for slot in &self.recovery_fault_slots {
1345 slot.validate()?;
1346 if slot.fault_sequence > self.recovery_fault_revision
1347 || slot.fault_sequence > self.lease.seq
1348 || !fault_sequences.insert(slot.fault_sequence)
1349 {
1350 return Err(LeaseError::Invalid(
1351 "leader authority recovery-fault sequences are not canonical".into(),
1352 ));
1353 }
1354 }
1355 if [
1356 self.checkpoint_outcome.is_some(),
1357 self.assignment_decision.is_some(),
1358 self.recovery_release_commit.is_some(),
1359 ]
1360 .into_iter()
1361 .filter(|present| *present)
1362 .count()
1363 > 1
1364 {
1365 return Err(LeaseError::Invalid(
1366 "one authority sequence cannot admit two terminal domains".into(),
1367 ));
1368 }
1369 Ok(())
1370 }
1371
1372 fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1373 if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1374 outcome
1375 .validate_shape(outcome.epoch)
1376 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1377 if outcome.scope != CheckpointScope::Cluster {
1378 return Err(LeaseError::Invalid(
1379 "leader authority can only admit cluster checkpoint outcomes".into(),
1380 ));
1381 }
1382 let proof = outcome
1383 .leader_proof
1384 .as_ref()
1385 .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1386 let current_link = OutcomeLink {
1387 sequence: self.lease.seq,
1388 epoch: outcome.epoch,
1389 checkpoint_id: outcome.checkpoint_id,
1390 };
1391 if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1392 return Err(LeaseError::Invalid(
1393 "cluster outcome is not bound to its exact authority sequence and term".into(),
1394 ));
1395 }
1396 if let Some(previous) = self.previous_outcome {
1397 if previous.sequence >= self.lease.seq
1398 || previous.epoch >= outcome.epoch
1399 || previous.checkpoint_id >= outcome.checkpoint_id
1400 {
1401 return Err(LeaseError::Invalid(
1402 "cluster outcome link does not move backward in sequence and epoch".into(),
1403 ));
1404 }
1405 }
1406 if outcome.is_commit() {
1407 if self.commit_head != Some(current_link) {
1408 return Err(LeaseError::Invalid(
1409 "cluster Commit is not bound to its exact Commit-chain head".into(),
1410 ));
1411 }
1412 if let Some(previous) = self.previous_commit {
1413 if previous.sequence >= self.lease.seq
1414 || previous.epoch >= outcome.epoch
1415 || previous.checkpoint_id >= outcome.checkpoint_id
1416 {
1417 return Err(LeaseError::Invalid(
1418 "cluster Commit link does not move backward in sequence and epoch"
1419 .into(),
1420 ));
1421 }
1422 if self.outcome_floor.as_ref().is_some_and(|floor| {
1423 previous.epoch < floor.artifact_before_epoch
1424 && Some(previous) != floor.committed_anchor_link
1425 }) {
1426 return Err(LeaseError::Invalid(
1427 "cluster Commit link crosses the artifact floor without its exact anchor"
1428 .into(),
1429 ));
1430 }
1431 }
1432 } else if self.previous_commit.is_some() {
1433 return Err(LeaseError::Invalid(
1434 "cluster Abort carries a previous-Commit link".into(),
1435 ));
1436 }
1437 if let Some(floor) = &self.outcome_floor {
1438 if outcome.deployment_id != floor.deployment_id
1439 || outcome.epoch < floor.authority_before_epoch
1440 {
1441 return Err(LeaseError::Invalid(
1442 "cluster outcome is below or outside its durable authority floor".into(),
1443 ));
1444 }
1445 }
1446 } else {
1447 if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1448 return Err(LeaseError::Invalid(
1449 "non-outcome authority record carries an outcome-chain link".into(),
1450 ));
1451 }
1452 if self.outcome_head.is_some_and(|head| {
1453 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1454 }) {
1455 return Err(LeaseError::Invalid(
1456 "authority outcome head is outside the durable sequence".into(),
1457 ));
1458 }
1459 }
1460 if self.commit_head.is_some_and(|head| {
1461 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1462 }) {
1463 return Err(LeaseError::Invalid(
1464 "authority Commit head is outside the durable sequence".into(),
1465 ));
1466 }
1467 match (self.commit_head, self.outcome_head) {
1468 (Some(commit), Some(terminal)) => {
1469 let equal = commit == terminal;
1470 let strictly_older = commit.sequence < terminal.sequence
1471 && commit.epoch < terminal.epoch
1472 && commit.checkpoint_id < terminal.checkpoint_id;
1473 if !equal && !strictly_older {
1474 return Err(LeaseError::Invalid(
1475 "authority Commit head is not ordered behind the terminal head".into(),
1476 ));
1477 }
1478 if self
1479 .checkpoint_outcome
1480 .as_ref()
1481 .is_some_and(|outcome| !outcome.is_commit())
1482 && !strictly_older
1483 {
1484 return Err(LeaseError::Invalid(
1485 "cluster Abort did not preserve a strictly older Commit head".into(),
1486 ));
1487 }
1488 }
1489 (Some(_), None) => {
1490 return Err(LeaseError::Invalid(
1491 "authority Commit head exists without terminal continuity".into(),
1492 ));
1493 }
1494 (None, _) => {}
1495 }
1496 Ok(())
1497 }
1498
1499 fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1500 if let Some(floor) = self.outcome_floor.as_ref() {
1501 floor.validate()?;
1502 if floor
1503 .terminal_anchor_link
1504 .is_some_and(|link| link.sequence >= self.lease.seq)
1505 {
1506 return Err(LeaseError::Invalid(
1507 "outcome floor anchor is outside the authority sequence".into(),
1508 ));
1509 }
1510 if floor
1511 .committed_anchor_link
1512 .is_some_and(|link| link.sequence >= self.lease.seq)
1513 {
1514 return Err(LeaseError::Invalid(
1515 "outcome floor Commit anchor is outside the authority sequence".into(),
1516 ));
1517 }
1518 if let Some(head) = self.outcome_head {
1519 if head.epoch < floor.authority_before_epoch
1520 && Some(head) != floor.terminal_anchor_link
1521 {
1522 return Err(LeaseError::Invalid(
1523 "authority outcome head does not meet its durable compaction floor".into(),
1524 ));
1525 }
1526 } else if floor.terminal_anchor_link.is_some() {
1527 return Err(LeaseError::Invalid(
1528 "outcome floor anchor is disconnected from the authority head".into(),
1529 ));
1530 }
1531 if let Some(head) = self.commit_head {
1532 if head.epoch < floor.artifact_before_epoch
1533 && Some(head) != floor.committed_anchor_link
1534 {
1535 return Err(LeaseError::Invalid(
1536 "authority Commit head does not meet its durable artifact floor".into(),
1537 ));
1538 }
1539 } else if floor.committed_anchor_link.is_some() {
1540 return Err(LeaseError::Invalid(
1541 "outcome floor Commit anchor is disconnected from the Commit head".into(),
1542 ));
1543 }
1544 }
1545 Ok(())
1546 }
1547
1548 fn validate_artifact_cleanup(&self) -> Result<(), LeaseError> {
1549 let Some(cursor) = self.artifact_cleanup.as_ref() else {
1550 return Ok(());
1551 };
1552 cursor.validate()?;
1553 let floor = self.outcome_floor.as_ref().ok_or_else(|| {
1554 LeaseError::Invalid(
1555 "cluster artifact cleanup exists without an outcome retention floor".into(),
1556 )
1557 })?;
1558 if floor.artifact_before_epoch != cursor.protected.epoch
1559 || self
1560 .commit_head
1561 .is_none_or(|head| head.epoch < cursor.protected.epoch)
1562 {
1563 return Err(LeaseError::Invalid(
1564 "cluster artifact cleanup is not bound to its protected retention cut".into(),
1565 ));
1566 }
1567 Ok(())
1568 }
1569
1570 fn validate_checkpoint_artifact_inventory(&self) -> Result<(), LeaseError> {
1571 let (Some(inventory), Some(admitting_proof)) = (
1572 self.active_checkpoint_artifacts.as_ref(),
1573 self.active_checkpoint_artifact_leader_proof.as_ref(),
1574 ) else {
1575 if self.active_checkpoint_artifacts.is_some()
1576 || self.active_checkpoint_artifact_leader_proof.is_some()
1577 {
1578 return Err(LeaseError::Invalid(
1579 "cluster checkpoint artifact inventory has incomplete leader authority".into(),
1580 ));
1581 }
1582 return Ok(());
1583 };
1584
1585 inventory.validate().map_err(|error| {
1586 LeaseError::Invalid(format!("cluster checkpoint artifact inventory: {error}"))
1587 })?;
1588 let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
1589 LeaseError::Invalid(
1590 "cluster checkpoint artifact inventory has no assignment fence".into(),
1591 )
1592 })?;
1593 if !admitting_proof.is_canonical()
1594 || assignment_fence.participant_incarnation(admitting_proof.owner.node_id)
1595 != Some(admitting_proof.owner.boot_id)
1596 {
1597 return Err(LeaseError::Invalid(
1598 "cluster checkpoint artifact inventory has no canonical admitting leader".into(),
1599 ));
1600 }
1601 if self
1602 .outcome_floor
1603 .as_ref()
1604 .is_some_and(|floor| floor.deployment_id != inventory.deployment_id)
1605 {
1606 return Err(LeaseError::Invalid(
1607 "cluster checkpoint artifact inventory belongs to a foreign deployment".into(),
1608 ));
1609 }
1610 if let Some(head) = self.outcome_head {
1611 let attempt = crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
1612 if matches!(
1613 inventory.attempt.relation_to(attempt),
1614 crate::checkpoint::CheckpointAttemptRelation::Older
1615 | crate::checkpoint::CheckpointAttemptRelation::Conflict
1616 ) {
1617 return Err(LeaseError::Invalid(
1618 "cluster checkpoint artifact inventory is behind the terminal outcome head"
1619 .into(),
1620 ));
1621 }
1622 }
1623 if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1624 let exact = outcome.deployment_id == inventory.deployment_id
1625 && outcome.epoch == inventory.attempt.epoch
1626 && outcome.checkpoint_id == inventory.attempt.checkpoint_id
1627 && outcome.assignment_fence.as_ref() == inventory.assignment_fence.as_ref();
1628 if outcome.is_commit() || !exact {
1629 return Err(LeaseError::Invalid(
1630 "terminal cluster outcome has inconsistent checkpoint artifact inventory"
1631 .into(),
1632 ));
1633 }
1634 }
1635 Ok(())
1636 }
1637
1638 fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1639 if let Some(decision) = self.assignment_decision.as_ref() {
1640 decision.validate()?;
1641 if self
1642 .assignment_decision_floor
1643 .as_ref()
1644 .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1645 {
1646 return Err(LeaseError::Invalid(
1647 "assignment decision is below its durable authority floor".into(),
1648 ));
1649 }
1650 if !self.lease.matches_proof(decision.leader_proof())
1651 || self.assignment_decision_head
1652 != Some(AssignmentDecisionLink {
1653 sequence: self.lease.seq,
1654 target_version: decision.target_version(),
1655 })
1656 {
1657 return Err(LeaseError::Invalid(
1658 "assignment decision is not bound to its exact authority sequence and term"
1659 .into(),
1660 ));
1661 }
1662 if let Some(previous) = self.previous_assignment_decision {
1663 if previous.sequence >= self.lease.seq
1664 || previous.target_version >= decision.target_version()
1665 {
1666 return Err(LeaseError::Invalid(
1667 "assignment decision link does not move backward".into(),
1668 ));
1669 }
1670 }
1671 } else {
1672 if self.previous_assignment_decision.is_some() {
1673 return Err(LeaseError::Invalid(
1674 "non-decision authority record carries a previous assignment-decision link"
1675 .into(),
1676 ));
1677 }
1678 if self
1679 .assignment_decision_head
1680 .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1681 {
1682 return Err(LeaseError::Invalid(
1683 "authority assignment-decision head is outside the durable sequence".into(),
1684 ));
1685 }
1686 }
1687 Ok(())
1688 }
1689
1690 fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1691 if let Some(floor) = self.assignment_decision_floor.as_ref() {
1692 floor.validate()?;
1693 if floor
1694 .terminal_anchor_link
1695 .is_some_and(|link| link.sequence >= self.lease.seq)
1696 {
1697 return Err(LeaseError::Invalid(
1698 "assignment decision floor anchor is outside the authority sequence".into(),
1699 ));
1700 }
1701 if let Some(head) = self.assignment_decision_head {
1702 if head.target_version < floor.before_target_version
1703 && Some(head) != floor.terminal_anchor_link
1704 {
1705 return Err(LeaseError::Invalid(
1706 "authority assignment-decision head does not meet its durable floor".into(),
1707 ));
1708 }
1709 } else if floor.terminal_anchor_link.is_some() {
1710 return Err(LeaseError::Invalid(
1711 "assignment decision floor anchor is disconnected from the authority head"
1712 .into(),
1713 ));
1714 }
1715 }
1716 Ok(())
1717 }
1718
1719 fn validate_assignment_handoff_pin(&self) -> Result<(), LeaseError> {
1720 let Some(pin) = self.assignment_handoff_pin.as_ref() else {
1721 return Ok(());
1722 };
1723 pin.validate()?;
1724 if self.commit_head.is_none_or(|head| {
1725 head.epoch != pin.checkpoint.epoch || head.checkpoint_id != pin.checkpoint.checkpoint_id
1726 }) || self
1727 .outcome_floor
1728 .as_ref()
1729 .is_some_and(|floor| floor.artifact_before_epoch > pin.checkpoint.epoch)
1730 || self
1731 .artifact_cleanup
1732 .as_ref()
1733 .is_some_and(|cursor| cursor.protected.epoch > pin.checkpoint.epoch)
1734 {
1735 return Err(LeaseError::Invalid(
1736 "assignment handoff pin is outside its live checkpoint retention range".into(),
1737 ));
1738 }
1739 Ok(())
1740 }
1741
1742 fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1743 match self.recovery_release_commit.as_ref() {
1744 Some(commit) => {
1745 commit.validate()?;
1746 let expected = RecoveryReleaseLink {
1747 sequence: self.lease.seq,
1748 terminal: commit.terminal.clone(),
1749 };
1750 if !self.lease.matches_proof(&commit.leader_proof)
1751 || self.recovery_release_head.as_ref() != Some(&expected)
1752 || self.recovery_fault_revision != self.lease.seq
1753 || self.recovery_fault_slots.iter().any(|slot| {
1754 slot.active || slot.disposition == RecoveryFaultDisposition::Terminal
1755 })
1756 {
1757 return Err(LeaseError::Invalid(
1758 "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1759 .into(),
1760 ));
1761 }
1762 }
1763 None => {
1764 if let Some(head) = self.recovery_release_head.as_ref() {
1765 head.validate()?;
1766 if head.sequence >= self.lease.seq {
1767 return Err(LeaseError::Invalid(
1768 "recovery release head is outside the durable authority sequence"
1769 .into(),
1770 ));
1771 }
1772 if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1773 && self.recovery_fault_revision != head.sequence
1774 {
1775 return Err(LeaseError::Invalid(
1776 "settled recovery fault inventory is not bound to its release head"
1777 .into(),
1778 ));
1779 }
1780 }
1781 }
1782 }
1783 Ok(())
1784 }
1785}
1786
1787#[derive(Debug, Clone, PartialEq, Eq)]
1789pub enum LeaseOutcome {
1790 Acquired(LeaderLease),
1792 Held(LeaderLease),
1794}
1795
1796enum AuthorityCreateOutcome {
1797 Created,
1798 ExistingIdentical,
1799 Contended(Box<LeaderAuthorityRecord>),
1800}
1801
1802#[derive(Clone, Copy)]
1803enum SameOwnerToken {
1804 #[cfg(test)]
1805 Preserve,
1806 Rotate,
1807 Exact(u64),
1808}
1809
1810#[derive(Debug)]
1812pub struct LeaderLeaseObservation {
1813 lease: LeaderLease,
1814 started: Instant,
1815}
1816
1817#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1820pub struct LeaderCandidacy {
1821 eligible: bool,
1822 generation: u64,
1823}
1824
1825impl LeaderCandidacy {
1826 pub(crate) const fn initial(eligible: bool) -> Self {
1827 Self {
1828 eligible,
1829 generation: 1,
1830 }
1831 }
1832
1833 pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1834 let generation = if self.eligible && !eligible {
1835 self.generation.checked_add(1)?
1836 } else {
1837 self.generation
1838 };
1839 Some(Self {
1840 eligible,
1841 generation,
1842 })
1843 }
1844
1845 pub(crate) const fn terminal() -> Self {
1846 Self {
1847 eligible: false,
1848 generation: u64::MAX,
1849 }
1850 }
1851
1852 #[must_use]
1854 pub const fn is_eligible(self) -> bool {
1855 self.eligible
1856 }
1857}
1858
1859#[derive(Debug, thiserror::Error)]
1861pub enum LeaseError {
1862 #[error("object store I/O: {0}")]
1864 Io(String),
1865 #[error("invalid leader lease: {0}")]
1867 Invalid(String),
1868 #[error("leader lease fenced: {0}")]
1870 Fenced(String),
1871 #[error("JSON: {0}")]
1873 Json(#[from] serde_json::Error),
1874}
1875
1876#[derive(Debug, thiserror::Error)]
1878pub enum ClusterCheckpointAuthorityError {
1879 #[error("cluster checkpoint authority is not installed")]
1881 NotConfigured,
1882 #[error("leader authority: {0}")]
1884 Authority(#[from] LeaseError),
1885 #[error("checkpoint decision: {0}")]
1887 Decision(#[from] DecisionError),
1888 #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1890 Fenced,
1891}
1892
1893#[derive(Debug, Clone, PartialEq, Eq)]
1895pub struct ClusterOutcomeRetentionBoundary {
1896 pub artifact_before_epoch: u64,
1898 pub terminal_before_epoch: u64,
1900 pub committed_anchor: Option<CheckpointOutcome>,
1902 pub terminal_anchor: Option<CheckpointOutcome>,
1904}
1905
1906impl ClusterOutcomeRetentionBoundary {
1907 fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1908 floor.map_or(
1909 Self {
1910 artifact_before_epoch: 0,
1911 terminal_before_epoch: 0,
1912 committed_anchor: None,
1913 terminal_anchor: None,
1914 },
1915 |floor| Self {
1916 artifact_before_epoch: floor.artifact_before_epoch,
1917 terminal_before_epoch: floor.authority_before_epoch,
1918 committed_anchor: floor.committed_anchor.clone(),
1919 terminal_anchor: floor.terminal_anchor.clone(),
1920 },
1921 )
1922 }
1923}
1924
1925#[derive(Debug, Clone, PartialEq, Eq)]
1927pub struct ClusterOutcomeInventory {
1928 pub outcomes: Vec<CheckpointOutcome>,
1930 pub retention_boundary: ClusterOutcomeRetentionBoundary,
1932}
1933
1934pub struct LeaderLeaseStore {
1936 store: Arc<dyn ObjectStore>,
1937 ttl_ms: i64,
1938 prune_running: Arc<AtomicBool>,
1939 outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
1940 outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
1941}
1942
1943#[derive(Clone, PartialEq, Eq)]
1944struct ClusterOutcomeAuditKey {
1945 terminal_head: Option<OutcomeLink>,
1946 commit_head: Option<OutcomeLink>,
1947 floor: Option<AuthorityOutcomeFloor>,
1948}
1949
1950struct ClusterOutcomeAuditCache {
1951 key: ClusterOutcomeAuditKey,
1952 authority_sequence: u64,
1953 snapshot: ClusterOutcomeAuditSnapshot,
1954}
1955
1956#[derive(Clone)]
1957struct ClusterOutcomeAuditSnapshot {
1958 outcomes: Arc<[CheckpointOutcome]>,
1959 terminal_links: Arc<[OutcomeLink]>,
1960 commit_links: Arc<[OutcomeLink]>,
1961}
1962
1963struct ClusterOutcomeAuditFlight {
1964 key: ClusterOutcomeAuditKey,
1965 gate: Weak<tokio::sync::Mutex<()>>,
1966}
1967
1968struct PruneLatchGuard(Arc<AtomicBool>);
1969
1970impl Drop for PruneLatchGuard {
1971 fn drop(&mut self) {
1972 self.0.store(false, Ordering::Release);
1973 }
1974}
1975
1976impl std::fmt::Debug for LeaderLeaseStore {
1977 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1978 formatter
1979 .debug_struct("LeaderLeaseStore")
1980 .field("ttl_ms", &self.ttl_ms)
1981 .finish_non_exhaustive()
1982 }
1983}
1984
1985impl LeaderLeaseStore {
1986 #[must_use]
1991 pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
1992 Self {
1993 store,
1994 ttl_ms,
1995 prune_running: Arc::new(AtomicBool::new(false)),
1996 outcome_audit_cache: Mutex::new(None),
1997 outcome_audit_flights: Mutex::new(Vec::new()),
1998 }
1999 }
2000
2001 pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
2010 probe_object_store_conditional_update(
2011 self.store.as_ref(),
2012 STORE_CONTRACT_PROBE_PREFIX,
2013 timeout,
2014 )
2015 .await
2016 .map_err(|error| match error {
2017 CheckpointStoreError::Invalid(message) => LeaseError::Invalid(message),
2018 error => LeaseError::Io(error.to_string()),
2019 })
2020 }
2021
2022 fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
2023 RecoveryFaultInventory {
2024 revision: record.recovery_fault_revision,
2025 faults: record
2026 .recovery_fault_slots
2027 .iter()
2028 .filter(|slot| slot.active)
2029 .map(AuthorityRecoveryFaultSlot::fault)
2030 .collect(),
2031 }
2032 }
2033
2034 pub(crate) async fn recovery_fault_inventory(
2035 &self,
2036 ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
2037 let record = self
2038 .load_record()
2039 .await?
2040 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2041 Ok(Self::recovery_fault_inventory_from(&record))
2042 }
2043
2044 #[cfg(test)]
2045 pub(crate) async fn record_recovery_fault(
2046 &self,
2047 publisher: RecoveryFaultPublisher,
2048 request_sequence: u64,
2049 ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2050 self.record_recovery_fault_with_disposition(
2051 publisher,
2052 request_sequence,
2053 RecoveryFaultDisposition::Recoverable,
2054 )
2055 .await
2056 }
2057
2058 pub(crate) async fn record_recovery_fault_with_disposition(
2059 &self,
2060 publisher: RecoveryFaultPublisher,
2061 request_sequence: u64,
2062 disposition: RecoveryFaultDisposition,
2063 ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2064 publisher.validate().map_err(LeaseError::Invalid)?;
2065 if request_sequence == 0 {
2066 return Err(LeaseError::Invalid(
2067 "recovery fault request sequence must be nonzero".into(),
2068 )
2069 .into());
2070 }
2071
2072 loop {
2073 let head = self
2074 .load_published_authority_head()
2075 .await?
2076 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2077 let current = &head.record;
2078 let node_id = publisher.participant.node_id;
2079 let slot_index = current
2080 .recovery_fault_slots
2081 .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
2082 let (insert_at, replace) = match slot_index {
2083 Ok(index) => {
2084 let slot = ¤t.recovery_fault_slots[index];
2085 if slot.matches_request(publisher, request_sequence, disposition) {
2086 return Ok(if slot.active {
2087 RecordRecoveryFaultResult::Active
2088 } else {
2089 RecordRecoveryFaultResult::AlreadyCleared
2090 });
2091 }
2092 if slot.disposition == RecoveryFaultDisposition::Terminal {
2095 return Ok(RecordRecoveryFaultResult::TerminalFenceActive);
2096 }
2097 if publisher.process_term < slot.publisher.process_term {
2098 return Ok(RecordRecoveryFaultResult::Superseded);
2099 }
2100 if publisher.process_term == slot.publisher.process_term
2101 && publisher.participant.boot_incarnation
2102 != slot.publisher.participant.boot_incarnation
2103 {
2104 return Err(LeaseError::Invalid(format!(
2105 "stable node {node_id} has two recovery fault publishers for process term {}",
2106 publisher.process_term
2107 ))
2108 .into());
2109 }
2110 if publisher.process_term == slot.publisher.process_term
2111 && request_sequence < slot.request_sequence
2112 {
2113 return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
2114 }
2115 (index, true)
2116 }
2117 Err(index) => (index, false),
2118 };
2119
2120 let sequence =
2121 current.lease.seq.checked_add(1).ok_or_else(|| {
2122 LeaseError::Invalid("leader authority sequence exhausted".into())
2123 })?;
2124 let mut lease = current.lease.clone();
2125 lease.seq = sequence;
2126 let mut candidate = current.preserve_with_lease(lease);
2127 let slot = AuthorityRecoveryFaultSlot {
2128 publisher,
2129 request_sequence,
2130 fault_sequence: sequence,
2134 disposition,
2135 active: true,
2136 };
2137 if replace {
2138 candidate.recovery_fault_slots[insert_at] = slot;
2139 } else {
2140 candidate.recovery_fault_slots.insert(insert_at, slot);
2141 }
2142 candidate.recovery_fault_revision = sequence;
2143 candidate.validate()?;
2144
2145 match self
2146 .create_authority_record(Some(&head), &candidate)
2147 .await?
2148 {
2149 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2150 return Ok(RecordRecoveryFaultResult::Active);
2151 }
2152 AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
2153 }
2154 }
2155 }
2156
2157 pub(crate) async fn authorize_recovery_release(
2158 &self,
2159 clearer: RecoveryFaultPublisher,
2160 terminal: &RecoveryAnnouncement,
2161 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2162 clearer.validate().map_err(LeaseError::Invalid)?;
2163 let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2164 let reporter = NodeId(clearer.participant.node_id);
2165 let expected = terminal.round.fault_sequence(reporter);
2166 let current = self
2167 .load_record()
2168 .await?
2169 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2170 let Some(release_head) = current.recovery_release_head.as_ref() else {
2171 return Ok(false);
2172 };
2173 if release_head.terminal != terminal_reference
2174 || current.recovery_fault_revision != release_head.sequence
2175 {
2176 return Ok(false);
2177 }
2178 if current
2179 .recovery_fault_slots
2180 .iter()
2181 .any(|slot| slot.active || slot.disposition == RecoveryFaultDisposition::Terminal)
2182 {
2183 return Ok(false);
2184 }
2185 let slot_index = current
2186 .recovery_fault_slots
2187 .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2188 let Some(expected_sequence) = expected else {
2189 return Ok(match slot_index {
2190 Ok(index) => !current.recovery_fault_slots[index].active,
2191 Err(_) => true,
2192 });
2193 };
2194 let Ok(index) = slot_index else {
2195 return Ok(false);
2196 };
2197 let slot = ¤t.recovery_fault_slots[index];
2198 Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2199 }
2200
2201 pub(crate) async fn stage_recovery_release_terminal(
2202 &self,
2203 terminal: &RecoveryAnnouncement,
2204 ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2205 let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2206 let path = recovery_release_terminal_path(&reference);
2207 let put_error = match self
2208 .store
2209 .put_opts(
2210 &path,
2211 PutPayload::from(encoded),
2212 PutOptions {
2213 mode: PutMode::Create,
2214 ..PutOptions::default()
2215 },
2216 )
2217 .await
2218 {
2219 Ok(_)
2220 | Err(
2221 object_store::Error::AlreadyExists { .. }
2222 | object_store::Error::Precondition { .. },
2223 ) => None,
2224 Err(error) => Some(error),
2225 };
2226 match self.load_recovery_release_terminal(&reference).await {
2227 Ok(stored) if stored == *terminal => Ok(reference),
2228 Ok(_) => Err(LeaseError::Invalid(format!(
2229 "recovery release terminal '{}' differs from the proposed content",
2230 reference.sha256
2231 ))
2232 .into()),
2233 Err(reconcile_error) => {
2234 if let Some(put_error) = put_error {
2235 Err(LeaseError::Io(format!(
2236 "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2237 ))
2238 .into())
2239 } else {
2240 Err(reconcile_error)
2241 }
2242 }
2243 }
2244 }
2245
2246 pub(crate) async fn load_recovery_release_terminal(
2247 &self,
2248 reference: &RecoveryReleaseTerminalRef,
2249 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2250 reference.validate()?;
2251 let result = match self
2252 .store
2253 .get(&recovery_release_terminal_path(reference))
2254 .await
2255 {
2256 Ok(result) => result,
2257 Err(object_store::Error::NotFound { .. }) => {
2258 return Err(LeaseError::Invalid(format!(
2259 "recovery release terminal '{}' is missing",
2260 reference.sha256
2261 ))
2262 .into());
2263 }
2264 Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2265 };
2266 if result.meta.size != reference.encoded_len {
2267 return Err(LeaseError::Invalid(format!(
2268 "recovery release terminal '{}' is {} bytes, expected {}",
2269 reference.sha256, result.meta.size, reference.encoded_len
2270 ))
2271 .into());
2272 }
2273 let bytes = result
2274 .bytes()
2275 .await
2276 .map_err(|error| LeaseError::Io(error.to_string()))?;
2277 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2278 return Err(LeaseError::Invalid(format!(
2279 "recovery release terminal '{}' payload length changed while reading",
2280 reference.sha256
2281 ))
2282 .into());
2283 }
2284 let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2285 LeaseError::Invalid(format!(
2286 "recovery release terminal '{}': {error}",
2287 reference.sha256
2288 ))
2289 })?;
2290 let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2291 if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2292 return Err(LeaseError::Invalid(format!(
2293 "recovery release terminal '{}' does not match its content-addressed reference",
2294 reference.sha256
2295 ))
2296 .into());
2297 }
2298 Ok(terminal)
2299 }
2300
2301 pub(crate) async fn record_recovery_release_commit(
2302 &self,
2303 proof: &LeaderProof,
2304 reference: RecoveryReleaseTerminalRef,
2305 ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2306 reference.validate()?;
2307 let terminal = self.load_recovery_release_terminal(&reference).await?;
2308 if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2309 return Err(ClusterCheckpointAuthorityError::Fenced);
2310 }
2311
2312 loop {
2313 let published = self
2314 .load_published_authority_head()
2315 .await?
2316 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2317 let current = &published.record;
2318 if let Some(winner) = current.recovery_release_head.as_ref() {
2319 if winner.terminal == reference
2320 || winner.terminal.generation() >= reference.generation()
2321 {
2322 self.load_recovery_release_terminal(&winner.terminal)
2323 .await?;
2324 return if winner.terminal == reference {
2325 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2326 } else {
2327 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2328 winner: winner.terminal.clone(),
2329 })
2330 };
2331 }
2332 }
2333 if !current.lease.matches_proof(proof) {
2334 return Err(ClusterCheckpointAuthorityError::Fenced);
2335 }
2336 let fault_inventory = Self::recovery_fault_inventory_from(current);
2337 if fault_inventory.revision != terminal.round.fault_revision()
2338 || fault_inventory.faults != terminal.round.faults
2339 || fault_inventory
2340 .faults
2341 .iter()
2342 .copied()
2343 .any(RecoveryFault::is_terminal)
2344 {
2345 return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2346 }
2347
2348 let base_sequence = current.lease.seq;
2349 let sequence = base_sequence
2350 .checked_add(1)
2351 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2352 let mut candidate = current.preserve_with_lease(LeaderLease {
2353 seq: sequence,
2354 renewal_sequence: current.lease.renewal_sequence,
2355 token: current.lease.token,
2356 owner: current.lease.owner.clone(),
2357 expires_at_ms: current.lease.expires_at_ms,
2358 catalog_manifest: current.lease.catalog_manifest.clone(),
2359 });
2360 candidate.recovery_fault_slots.retain(|slot| {
2363 slot.active
2364 && terminal
2365 .round
2366 .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2367 == Some(slot.publisher.participant.boot_incarnation)
2368 });
2369 for slot in &mut candidate.recovery_fault_slots {
2370 slot.active = false;
2371 }
2372 candidate.recovery_fault_revision = sequence;
2373 candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2374 terminal: reference.clone(),
2375 leader_proof: proof.clone(),
2376 });
2377 candidate.recovery_release_head = Some(RecoveryReleaseLink {
2378 sequence,
2379 terminal: reference.clone(),
2380 });
2381 candidate.validate()?;
2382
2383 match self
2384 .create_authority_record(Some(&published), &candidate)
2385 .await?
2386 {
2387 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2388 return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2389 }
2390 AuthorityCreateOutcome::Contended(winner_head) => {
2391 if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2392 if winner.terminal == reference
2393 || winner.terminal.generation() >= reference.generation()
2394 {
2395 self.load_recovery_release_terminal(&winner.terminal)
2396 .await?;
2397 return if winner.terminal == reference {
2398 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2399 } else {
2400 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2401 winner: winner.terminal.clone(),
2402 })
2403 };
2404 }
2405 }
2406 if !winner_head.lease.matches_proof(proof) {
2407 return Err(ClusterCheckpointAuthorityError::Fenced);
2408 }
2409 if winner_head.lease.seq <= base_sequence {
2410 return Err(LeaseError::Invalid(
2411 "recovery release authority contention did not advance the sequence"
2412 .into(),
2413 )
2414 .into());
2415 }
2416 tokio::task::yield_now().await;
2417 }
2418 }
2419 }
2420 }
2421
2422 async fn recovery_release_terminal_from(
2423 &self,
2424 head: &LeaderAuthorityRecord,
2425 link: &RecoveryReleaseLink,
2426 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2427 let admission = if link.sequence == head.lease.seq {
2428 head.clone()
2429 } else {
2430 read_authority_record(self.store.as_ref(), link.sequence)
2431 .await?
2432 .ok_or_else(|| {
2433 LeaseError::Invalid(
2434 "recovery release admission is missing from retained authority history"
2435 .into(),
2436 )
2437 })?
2438 };
2439 let commit = admission
2440 .recovery_release_commit
2441 .as_ref()
2442 .filter(|commit| {
2443 commit.terminal == link.terminal
2444 && admission.recovery_release_head.as_ref() == Some(link)
2445 })
2446 .ok_or_else(|| {
2447 LeaseError::Invalid(
2448 "recovery release head does not match its admitting authority record".into(),
2449 )
2450 })?;
2451 let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2452 if terminal.round.leader_proof != commit.leader_proof {
2453 return Err(LeaseError::Invalid(
2454 "recovery release terminal does not match its admitting leader proof".into(),
2455 )
2456 .into());
2457 }
2458 Ok(terminal)
2459 }
2460
2461 pub(crate) async fn recovery_admission_snapshot(
2462 &self,
2463 ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2464 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2465 let head = self
2466 .load_record()
2467 .await?
2468 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2469 let Some(link) = head.recovery_release_head.clone() else {
2470 return Ok(RecoveryAdmissionSnapshot {
2471 committed_release: None,
2472 fault_inventory: Self::recovery_fault_inventory_from(&head),
2473 authority_sequence: head.lease.seq,
2474 release_head: None,
2475 });
2476 };
2477 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2478 let rechecked = self
2479 .load_record()
2480 .await?
2481 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2482 if rechecked.lease.seq >= head.lease.seq
2483 && rechecked.recovery_release_head == head.recovery_release_head
2484 && rechecked.recovery_fault_revision == head.recovery_fault_revision
2485 && rechecked.recovery_fault_slots == head.recovery_fault_slots
2486 {
2487 return Ok(RecoveryAdmissionSnapshot {
2488 committed_release: Some(terminal?),
2489 fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2490 authority_sequence: rechecked.lease.seq,
2491 release_head: rechecked.recovery_release_head,
2492 });
2493 }
2494 tokio::task::yield_now().await;
2495 }
2496 Err(LeaseError::Io(format!(
2497 "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2498 ))
2499 .into())
2500 }
2501
2502 pub(crate) async fn recovery_admission_is_current(
2503 &self,
2504 snapshot: &RecoveryAdmissionSnapshot,
2505 leader_proof: &LeaderProof,
2506 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2507 let current = self
2508 .load_record()
2509 .await?
2510 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2511 let inventory = Self::recovery_fault_inventory_from(¤t);
2512 Ok(current.lease.matches_proof(leader_proof)
2513 && current.lease.seq >= snapshot.authority_sequence
2514 && current.recovery_release_head == snapshot.release_head
2515 && inventory == snapshot.fault_inventory
2516 && inventory.faults().is_empty())
2517 }
2518
2519 pub(crate) async fn latest_recovery_release_terminal(
2520 &self,
2521 ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2522 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2523 let Some(head) = self.load_record().await? else {
2524 return Ok(None);
2525 };
2526 let Some(link) = head.recovery_release_head.clone() else {
2527 return Ok(None);
2528 };
2529 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2530 let rechecked = self
2531 .load_record()
2532 .await?
2533 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2534 if rechecked.recovery_release_head.as_ref() == Some(&link) {
2535 return Ok(Some(terminal?));
2536 }
2537 tokio::task::yield_now().await;
2538 }
2539 Err(LeaseError::Io(format!(
2540 "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2541 ))
2542 .into())
2543 }
2544
2545 fn ttl(&self) -> Result<Duration, LeaseError> {
2546 let ttl = u64::try_from(self.ttl_ms)
2547 .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2548 if ttl == 0 {
2549 return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2550 }
2551 Ok(Duration::from_millis(ttl))
2552 }
2553
2554 fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2555 now_ms
2556 .checked_add(self.ttl_ms)
2557 .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2558 }
2559
2560 #[cfg(test)]
2561 async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2562 let prefix = OsPath::from(LEASE_PREFIX);
2563 let mut entries = self.store.list(Some(&prefix));
2564 let mut sequences = Vec::new();
2565 while let Some(entry) = entries.next().await {
2566 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2567 if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2568 return Err(LeaseError::Invalid(format!(
2569 "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2570 )));
2571 }
2572 sequences.push(lease_sequence_from_path(&entry.location)?);
2573 }
2574 sequences.sort_unstable();
2575 sequences.dedup();
2576 Ok(sequences)
2577 }
2578
2579 fn schedule_history_prune(&self) {
2580 if self
2581 .prune_running
2582 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2583 .is_err()
2584 {
2585 return;
2586 }
2587 let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2588 self.prune_running.store(false, Ordering::Release);
2589 return;
2590 };
2591 let store = Arc::clone(&self.store);
2592 let prune_running = Arc::clone(&self.prune_running);
2593 let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2594 runtime.spawn(async move {
2595 let _latch = PruneLatchGuard(prune_running);
2596 match tokio::time::timeout(
2597 LEADER_LEASE_PRUNE_TIMEOUT,
2598 Self::prune_history(&store, grace_ms),
2599 )
2600 .await
2601 {
2602 Ok(Ok(())) => {}
2603 Ok(Err(error)) => {
2604 tracing::warn!(%error, "leader lease history prune failed");
2605 }
2606 Err(_) => {
2607 tracing::warn!(
2608 timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2609 "leader lease history prune timed out"
2610 );
2611 }
2612 }
2613 });
2614 }
2615
2616 async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2617 let authority = Self::new(Arc::clone(store), 1);
2618 let Some(head) = authority.load_record().await? else {
2619 return Ok(());
2620 };
2621 let head_sequence = head.lease.seq;
2622 let mut retained = BTreeSet::from([head_sequence]);
2623 if let Some(previous) = head_sequence
2624 .checked_sub(1)
2625 .filter(|sequence| *sequence != 0)
2626 {
2627 retained.insert(previous);
2628 }
2629 let floor = head
2630 .outcome_floor
2631 .as_ref()
2632 .map_or(0, |floor| floor.authority_before_epoch);
2633 let artifact_floor = head
2634 .outcome_floor
2635 .as_ref()
2636 .map_or(0, |floor| floor.artifact_before_epoch);
2637 let mut link = head.outcome_head;
2638 let mut outcome_links = 0;
2639 let mut terminal_commit_links = BTreeSet::new();
2640 let mut expected_commit_head = head.commit_head;
2641 while let Some(current) = link {
2642 if current.epoch < floor {
2643 if head
2644 .outcome_floor
2645 .as_ref()
2646 .and_then(|floor| floor.terminal_anchor_link)
2647 != Some(current)
2648 {
2649 return Err(LeaseError::Invalid(
2650 "outcome floor does not anchor the retained authority chain".into(),
2651 ));
2652 }
2653 break;
2654 }
2655 if !consume_live_authority_link(&mut outcome_links) {
2656 return Err(LeaseError::Invalid(format!(
2657 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2658 )));
2659 }
2660 let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2661 .await?
2662 .filter(|record| {
2663 record.outcome_head == Some(current)
2664 && record
2665 .checkpoint_outcome
2666 .as_ref()
2667 .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2668 == Some((current.epoch, current.checkpoint_id))
2669 })
2670 .ok_or_else(|| {
2671 LeaseError::Invalid("retained outcome authority chain is broken".into())
2672 })?;
2673 if outcome_record.commit_head != expected_commit_head {
2674 return Err(LeaseError::Invalid(
2675 "retained outcome chain did not preserve the exact Commit head".into(),
2676 ));
2677 }
2678 retained.insert(current.sequence);
2679 if outcome_record
2680 .checkpoint_outcome
2681 .as_ref()
2682 .is_some_and(CheckpointOutcome::is_commit)
2683 {
2684 terminal_commit_links.insert(current);
2685 expected_commit_head = outcome_record.previous_commit;
2686 } else {
2687 expected_commit_head = outcome_record.commit_head;
2688 }
2689 link = outcome_record.previous_outcome;
2690 }
2691 let mut commit_link = head.commit_head;
2692 let mut commit_links = 0;
2693 let mut retained_commit_links = BTreeSet::new();
2694 while let Some(current) = commit_link {
2695 if current.epoch < artifact_floor {
2696 break;
2697 }
2698 if !consume_live_authority_link(&mut commit_links) {
2699 return Err(LeaseError::Invalid(format!(
2700 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2701 )));
2702 }
2703 let commit_record = read_authority_record(store.as_ref(), current.sequence)
2704 .await?
2705 .filter(|record| {
2706 record.commit_head == Some(current)
2707 && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2708 outcome.is_commit()
2709 && outcome.epoch == current.epoch
2710 && outcome.checkpoint_id == current.checkpoint_id
2711 })
2712 })
2713 .ok_or_else(|| {
2714 LeaseError::Invalid("retained Commit authority chain is broken".into())
2715 })?;
2716 retained.insert(current.sequence);
2717 retained_commit_links.insert(current);
2718 commit_link = commit_record.previous_commit;
2719 }
2720 match (
2721 commit_link,
2722 head.outcome_floor
2723 .as_ref()
2724 .and_then(|floor| floor.committed_anchor_link),
2725 head.outcome_floor
2726 .as_ref()
2727 .and_then(|floor| floor.committed_anchor.as_ref()),
2728 ) {
2729 (Some(link), Some(anchor_link), Some(anchor))
2730 if link == anchor_link
2731 && link.epoch == anchor.epoch
2732 && link.checkpoint_id == anchor.checkpoint_id => {}
2733 (None, None, None) => {}
2734 _ => {
2735 return Err(LeaseError::Invalid(
2736 "Commit chain does not meet the retained artifact-floor anchor".into(),
2737 ));
2738 }
2739 }
2740 let boundary_commit_head = head
2741 .outcome_floor
2742 .as_ref()
2743 .and_then(|floor| floor.terminal_anchor.as_ref())
2744 .and_then(|anchor| {
2745 retained_commit_links
2746 .iter()
2747 .rev()
2748 .find(|link| link.epoch <= anchor.epoch)
2749 .copied()
2750 .or_else(|| {
2751 head.outcome_floor
2752 .as_ref()
2753 .and_then(|floor| floor.committed_anchor_link)
2754 })
2755 });
2756 if expected_commit_head != boundary_commit_head {
2757 return Err(LeaseError::Invalid(
2758 "retained terminal chain lost Commit continuity at its durable floor".into(),
2759 ));
2760 }
2761 if !terminal_commit_links.is_subset(&retained_commit_links) {
2762 return Err(LeaseError::Invalid(
2763 "terminal Commit records are not linked from the retained Commit chain".into(),
2764 ));
2765 }
2766 if let Some((anchor, anchor_link)) = head.outcome_floor.as_ref().and_then(|floor| {
2767 floor
2768 .terminal_anchor
2769 .as_ref()
2770 .zip(floor.terminal_anchor_link)
2771 }) {
2772 if anchor.is_commit() {
2773 let linked = if anchor.epoch >= artifact_floor {
2774 retained_commit_links.contains(&anchor_link)
2775 } else {
2776 head.outcome_floor.as_ref().is_some_and(|floor| {
2777 floor.committed_anchor.as_ref() == Some(anchor)
2778 && floor.committed_anchor_link == Some(anchor_link)
2779 })
2780 };
2781 if !linked {
2782 return Err(LeaseError::Invalid(
2783 "terminal Commit anchor is not linked from the retained Commit chain"
2784 .into(),
2785 ));
2786 }
2787 }
2788 }
2789 let mut assignment_link = head.assignment_decision_head;
2790 let mut assignment_links = 0;
2791 while let Some(current) = assignment_link {
2792 if !consume_live_authority_link(&mut assignment_links) {
2793 return Err(LeaseError::Invalid(format!(
2794 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2795 )));
2796 }
2797 if head
2798 .assignment_decision_floor
2799 .as_ref()
2800 .is_some_and(|floor| current.target_version < floor.before_target_version)
2801 {
2802 if head
2803 .assignment_decision_floor
2804 .as_ref()
2805 .and_then(|floor| floor.terminal_anchor_link)
2806 != Some(current)
2807 {
2808 return Err(LeaseError::Invalid(
2809 "assignment-decision floor does not anchor the retained authority chain"
2810 .into(),
2811 ));
2812 }
2813 break;
2814 }
2815 let decision_record = read_authority_record(store.as_ref(), current.sequence)
2816 .await?
2817 .filter(|record| {
2818 record.assignment_decision_head == Some(current)
2819 && record
2820 .assignment_decision
2821 .as_ref()
2822 .map(AuthorityAssignmentDecision::target_version)
2823 == Some(current.target_version)
2824 })
2825 .ok_or_else(|| {
2826 LeaseError::Invalid(
2827 "retained assignment-decision authority chain is broken".into(),
2828 )
2829 })?;
2830 retained.insert(current.sequence);
2831 assignment_link = decision_record.previous_assignment_decision;
2832 }
2833 let retained_release = match head.recovery_release_head.as_ref() {
2834 Some(link) => {
2835 let admission = read_authority_record(store.as_ref(), link.sequence)
2836 .await?
2837 .filter(|record| {
2838 record.recovery_release_head.as_ref() == Some(link)
2839 && record
2840 .recovery_release_commit
2841 .as_ref()
2842 .is_some_and(|commit| commit.terminal == link.terminal)
2843 })
2844 .ok_or_else(|| {
2845 LeaseError::Invalid(
2846 "retained recovery release authority link is broken".into(),
2847 )
2848 })?;
2849 if admission.lease.seq != link.sequence {
2850 return Err(LeaseError::Invalid(
2851 "recovery release admission sequence does not match its retained link"
2852 .into(),
2853 ));
2854 }
2855 let terminal = authority
2856 .load_recovery_release_terminal(&link.terminal)
2857 .await
2858 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2859 if admission
2860 .recovery_release_commit
2861 .as_ref()
2862 .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2863 {
2864 return Err(LeaseError::Invalid(
2865 "retained recovery release terminal does not match its admission".into(),
2866 ));
2867 }
2868 retained.insert(link.sequence);
2869 Some(link.terminal.clone())
2870 }
2871 None => None,
2872 };
2873
2874 let mut history_exhausted = false;
2875 for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2876 let (candidates, exhausted) =
2877 Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2878 if candidates.is_empty() {
2879 history_exhausted = true;
2880 break;
2881 }
2882 let deletions =
2883 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2884 let mut results = store.delete_stream(Box::pin(deletions));
2885 while let Some(result) = results.next().await {
2886 if let Err(error) = result {
2887 if !matches!(error, object_store::Error::NotFound { .. }) {
2888 return Err(LeaseError::Io(error.to_string()));
2889 }
2890 }
2891 }
2892 if exhausted {
2893 history_exhausted = true;
2894 break;
2895 }
2896 tokio::task::yield_now().await;
2897 }
2898 if !history_exhausted {
2899 return Err(LeaseError::Io(
2900 "leader lease history still exceeds the bounded prune budget".into(),
2901 ));
2902 }
2903 Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await
2904 }
2905
2906 async fn prune_recovery_release_terminals(
2907 store: &Arc<dyn ObjectStore>,
2908 retained: Option<&RecoveryReleaseTerminalRef>,
2909 grace_ms: i64,
2910 ) -> Result<(), LeaseError> {
2911 let Some(retained) = retained else {
2912 return Ok(());
2913 };
2914 retained.validate()?;
2915 for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
2916 let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
2917 let mut listed = store.list(Some(&prefix));
2918 let now = now_millis();
2919 let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
2920 let mut exhausted = true;
2921 while let Some(entry) = listed.next().await {
2922 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2923 let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
2924 let canonical = OsPath::from(format!(
2925 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
2926 ));
2927 if canonical != entry.location {
2928 return Err(LeaseError::Invalid(format!(
2929 "noncanonical recovery release terminal path {}",
2930 entry.location
2931 )));
2932 }
2933 if entry.location == recovery_release_terminal_path(retained)
2934 || generation > retained.generation()
2935 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2936 {
2937 continue;
2938 }
2939 candidates.push(entry.location);
2940 if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
2941 exhausted = false;
2942 break;
2943 }
2944 }
2945 if candidates.is_empty() {
2946 return Ok(());
2947 }
2948 let deletions =
2949 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2950 let mut results = store.delete_stream(Box::pin(deletions));
2951 while let Some(result) = results.next().await {
2952 if let Err(error) = result {
2953 if !matches!(error, object_store::Error::NotFound { .. }) {
2954 return Err(LeaseError::Io(error.to_string()));
2955 }
2956 }
2957 }
2958 if exhausted {
2959 return Ok(());
2960 }
2961 tokio::task::yield_now().await;
2962 }
2963 Ok(())
2964 }
2965
2966 async fn prune_candidates(
2967 store: &Arc<dyn ObjectStore>,
2968 retained: &BTreeSet<u64>,
2969 snapshot_head_sequence: u64,
2970 grace_ms: i64,
2971 ) -> Result<(Vec<OsPath>, bool), LeaseError> {
2972 let prefix = OsPath::from(LEASE_PREFIX);
2973 let mut listed = store.list(Some(&prefix));
2974 let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
2975 let now = now_millis();
2976 let mut exhausted = true;
2977 while let Some(entry) = listed.next().await {
2978 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2979 let sequence = lease_sequence_from_path(&entry.location)?;
2980 if sequence >= snapshot_head_sequence
2981 || retained.contains(&sequence)
2982 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2983 {
2984 continue;
2985 }
2986 candidates.push(entry.location);
2987 if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
2988 exhausted = false;
2989 break;
2990 }
2991 }
2992 Ok((candidates, exhausted))
2993 }
2994
2995 pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
3000 Ok(self.load_record().await?.map(|record| record.lease))
3001 }
3002
3003 async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3004 Ok(self
3005 .load_published_authority_head()
3006 .await?
3007 .map(|head| head.record))
3008 }
3009
3010 async fn load_published_authority_head(
3011 &self,
3012 ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
3013 let pointer = if let Some(pointer) =
3014 read_authority_head_pointer(self.store.as_ref()).await?
3015 {
3016 pointer
3017 } else {
3018 let Some(discovered) = self.discover_authority_head().await? else {
3019 let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
3020 return Ok(None);
3021 };
3022 return self.load_authority_head_target(pointer).await.map(Some);
3023 };
3024 let discovered_sequence = discovered.lease.seq;
3025 let published_sequence = self
3026 .publish_authority_head(discovered_sequence, None)
3027 .await?;
3028 let pointer = self
3029 .reload_authority_head_pointer(published_sequence)
3030 .await?;
3031 if published_sequence > discovered_sequence
3032 || pointer.pointer.sequence > published_sequence
3033 {
3034 return self.load_authority_head_target(pointer).await.map(Some);
3035 }
3036 pointer
3037 };
3038
3039 let sequence = pointer.pointer.sequence;
3040 let successor_sequence = sequence.checked_add(1);
3041 let (record, successor) = if let Some(successor_sequence) = successor_sequence {
3042 tokio::try_join!(
3043 read_authority_record(self.store.as_ref(), sequence),
3044 read_authority_record(self.store.as_ref(), successor_sequence)
3045 )?
3046 } else {
3047 (
3048 read_authority_record(self.store.as_ref(), sequence).await?,
3049 None,
3050 )
3051 };
3052 let Some(record) = record else {
3053 let rechecked = read_authority_head_pointer(self.store.as_ref())
3054 .await?
3055 .ok_or_else(|| {
3056 LeaseError::Invalid(
3057 "leader authority head disappeared while reading its target".into(),
3058 )
3059 })?;
3060 if rechecked.pointer.sequence > sequence {
3061 return self
3062 .reload_published_authority_snapshot(rechecked.pointer.sequence)
3063 .await
3064 .map(Some);
3065 }
3066 if rechecked.pointer.sequence < sequence {
3067 return Err(LeaseError::Invalid(format!(
3068 "leader authority head regressed from sequence {sequence} to {}",
3069 rechecked.pointer.sequence
3070 )));
3071 }
3072 return Err(LeaseError::Invalid(format!(
3073 "leader authority head points ahead to missing sequence {sequence}"
3074 )));
3075 };
3076 let Some(successor_sequence) = successor_sequence else {
3077 return Ok(Some(PublishedAuthorityHead { record, pointer }));
3078 };
3079 if successor.is_none() {
3080 return Ok(Some(PublishedAuthorityHead { record, pointer }));
3081 }
3082
3083 if let Some(after_successor) = successor_sequence.checked_add(1) {
3084 if read_authority_record(self.store.as_ref(), after_successor)
3085 .await?
3086 .is_some()
3087 {
3088 let rechecked = read_authority_head_pointer(self.store.as_ref())
3089 .await?
3090 .ok_or_else(|| {
3091 LeaseError::Invalid(
3092 "leader authority head disappeared while checking pointer lag".into(),
3093 )
3094 })?;
3095 if rechecked.pointer.sequence == sequence {
3096 return Err(LeaseError::Invalid(format!(
3097 "leader authority head at sequence {sequence} lags by more than one record"
3098 )));
3099 }
3100 if rechecked.pointer.sequence < sequence {
3101 return Err(LeaseError::Invalid(format!(
3102 "leader authority head regressed from sequence {sequence} to {}",
3103 rechecked.pointer.sequence
3104 )));
3105 }
3106 return self
3107 .reload_published_authority_snapshot(rechecked.pointer.sequence)
3108 .await
3109 .map(Some);
3110 }
3111 }
3112
3113 let published_sequence = self
3114 .publish_authority_head(successor_sequence, Some(&pointer))
3115 .await?;
3116 self.reload_published_authority_snapshot(published_sequence)
3117 .await
3118 .map(Some)
3119 }
3120
3121 async fn reload_published_authority_snapshot(
3122 &self,
3123 minimum_sequence: u64,
3124 ) -> Result<PublishedAuthorityHead, LeaseError> {
3125 let pointer = self.reload_authority_head_pointer(minimum_sequence).await?;
3126 self.load_authority_head_target(pointer).await
3127 }
3128
3129 async fn reload_authority_head_pointer(
3130 &self,
3131 minimum_sequence: u64,
3132 ) -> Result<VersionedAuthorityHeadPointer, LeaseError> {
3133 let pointer = read_authority_head_pointer(self.store.as_ref())
3134 .await?
3135 .ok_or_else(|| {
3136 LeaseError::Invalid(
3137 "leader authority head disappeared while reloading its published snapshot"
3138 .into(),
3139 )
3140 })?;
3141 if pointer.pointer.sequence < minimum_sequence {
3142 return Err(LeaseError::Invalid(format!(
3143 "leader authority head regressed from sequence {minimum_sequence} to {}",
3144 pointer.pointer.sequence
3145 )));
3146 }
3147 Ok(pointer)
3148 }
3149
3150 async fn load_authority_head_target(
3151 &self,
3152 mut pointer: VersionedAuthorityHeadPointer,
3153 ) -> Result<PublishedAuthorityHead, LeaseError> {
3154 for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
3155 let sequence = pointer.pointer.sequence;
3156 if let Some(record) = read_authority_record(self.store.as_ref(), sequence).await? {
3157 return Ok(PublishedAuthorityHead { record, pointer });
3158 }
3159
3160 let rechecked = read_authority_head_pointer(self.store.as_ref())
3161 .await?
3162 .ok_or_else(|| {
3163 LeaseError::Invalid(
3164 "leader authority head disappeared while reading its target".into(),
3165 )
3166 })?;
3167 if rechecked.pointer.sequence < sequence {
3168 return Err(LeaseError::Invalid(format!(
3169 "leader authority head regressed from sequence {sequence} to {}",
3170 rechecked.pointer.sequence
3171 )));
3172 }
3173 if rechecked.pointer.sequence == sequence {
3174 return Err(LeaseError::Invalid(format!(
3175 "leader authority head points ahead to missing sequence {sequence}"
3176 )));
3177 }
3178 pointer = rechecked;
3179 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
3180 tokio::task::yield_now().await;
3181 }
3182 }
3183 Err(LeaseError::Io(format!(
3184 "leader authority head target changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
3185 )))
3186 }
3187
3188 async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3189 let prefix = OsPath::from(LEASE_PREFIX);
3190 let mut listed = self.store.list(Some(&prefix));
3191 let mut discovered = 0usize;
3192 let mut maximum = None;
3193 while let Some(entry) = listed.next().await {
3194 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3195 discovered = discovered.checked_add(1).ok_or_else(|| {
3196 LeaseError::Invalid("leader authority discovery count overflowed".into())
3197 })?;
3198 if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
3199 return Err(LeaseError::Invalid(format!(
3200 "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
3201 )));
3202 }
3203 let sequence = lease_sequence_from_path(&entry.location)?;
3204 maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
3205 }
3206 let Some(sequence) = maximum else {
3207 return Ok(None);
3208 };
3209 read_authority_record(self.store.as_ref(), sequence)
3210 .await?
3211 .map(Some)
3212 .ok_or_else(|| {
3213 LeaseError::Io(format!(
3214 "discovered leader authority sequence {sequence} vanished before publication"
3215 ))
3216 })
3217 }
3218
3219 async fn publish_authority_head(
3220 &self,
3221 sequence: u64,
3222 expected: Option<&VersionedAuthorityHeadPointer>,
3223 ) -> Result<u64, LeaseError> {
3224 if let Some(expected) = expected {
3225 if expected.pointer.sequence.checked_add(1) != Some(sequence) {
3226 return Err(LeaseError::Invalid(
3227 "leader authority head update is not the exact successor".into(),
3228 ));
3229 }
3230 if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3231 {
3232 return Err(LeaseError::Invalid(
3233 "leader authority store did not provide a native conditional update version"
3234 .into(),
3235 ));
3236 }
3237 }
3238
3239 let options = PutOptions {
3240 mode: expected.map_or(PutMode::Create, |expected| {
3241 PutMode::Update(expected.update_version.clone())
3242 }),
3243 ..PutOptions::default()
3244 };
3245 let result = self
3246 .store
3247 .put_opts(
3248 &authority_head_path(),
3249 PutPayload::from(encode_authority_head_pointer(sequence)?),
3250 options,
3251 )
3252 .await;
3253 if result.is_ok() {
3254 return Ok(sequence);
3255 }
3256 let write_error = result
3257 .expect_err("successful authority head writes return above")
3258 .to_string();
3259 let current = read_authority_head_pointer(self.store.as_ref())
3260 .await?
3261 .ok_or_else(|| {
3262 LeaseError::Io(format!(
3263 "leader authority head write failed ({write_error}) and no pointer was durable"
3264 ))
3265 })?;
3266 if let Some(expected) = expected {
3267 if current.pointer.sequence < expected.pointer.sequence {
3268 return Err(LeaseError::Invalid(format!(
3269 "leader authority head regressed from sequence {} to {}",
3270 expected.pointer.sequence, current.pointer.sequence
3271 )));
3272 }
3273 }
3274 if current.pointer.sequence >= sequence {
3275 return Ok(current.pointer.sequence);
3276 }
3277 Err(LeaseError::Io(format!(
3278 "leader authority head write failed ({write_error}) and remained at sequence {}",
3279 current.pointer.sequence
3280 )))
3281 }
3282
3283 async fn read_catalog_manifest_blob(
3284 &self,
3285 reference: &CatalogManifestRef,
3286 ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3287 reference.validate()?;
3288 let path = reference.object_path();
3289 let result = match self.store.get(&path).await {
3290 Ok(result) => result,
3291 Err(object_store::Error::NotFound { .. }) => {
3292 return Err(CatalogManifestError::Invalid(format!(
3293 "catalog manifest blob '{}' is missing",
3294 reference.sha256
3295 )));
3296 }
3297 Err(error) => {
3298 return Err(CatalogManifestError::Authority(LeaseError::Io(
3299 error.to_string(),
3300 )));
3301 }
3302 };
3303 if result.meta.size != reference.encoded_len {
3304 return Err(CatalogManifestError::Invalid(format!(
3305 "catalog manifest blob '{}' is {} bytes, expected {}",
3306 reference.sha256, result.meta.size, reference.encoded_len
3307 )));
3308 }
3309 let bytes = result
3310 .bytes()
3311 .await
3312 .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3313 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3314 return Err(CatalogManifestError::Invalid(format!(
3315 "catalog manifest blob '{}' payload length changed while reading",
3316 reference.sha256
3317 )));
3318 }
3319 let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3320 let (canonical, actual_reference) = manifest.encode_and_reference()?;
3321 if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3322 return Err(CatalogManifestError::Invalid(format!(
3323 "catalog manifest blob '{}' does not match its sealed reference",
3324 reference.sha256
3325 )));
3326 }
3327 Ok((manifest, bytes))
3328 }
3329
3330 pub(super) async fn load_catalog_manifest(
3331 &self,
3332 reference: &CatalogManifestRef,
3333 ) -> Result<CatalogManifest, CatalogManifestError> {
3334 self.read_catalog_manifest_blob(reference)
3335 .await
3336 .map(|(manifest, _)| manifest)
3337 }
3338
3339 async fn ensure_catalog_manifest_blob(
3340 &self,
3341 encoded: &[u8],
3342 reference: &CatalogManifestRef,
3343 ) -> Result<(), CatalogManifestError> {
3344 reference.validate()?;
3345 let options = PutOptions {
3346 mode: PutMode::Create,
3347 ..PutOptions::default()
3348 };
3349 let path = reference.object_path();
3350 let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3351 let put_error = match self.store.put_opts(&path, payload, options).await {
3352 Ok(_)
3353 | Err(
3354 object_store::Error::AlreadyExists { .. }
3355 | object_store::Error::Precondition { .. },
3356 ) => None,
3357 Err(error) => Some(error),
3358 };
3359
3360 match self.read_catalog_manifest_blob(reference).await {
3361 Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3362 Ok(_) => Err(CatalogManifestError::Invalid(format!(
3363 "catalog manifest blob '{}' differs from the proposed content",
3364 reference.sha256
3365 ))),
3366 Err(error) => {
3367 if let Some(put_error) = put_error {
3368 Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3369 "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3370 ))))
3371 } else {
3372 Err(error)
3373 }
3374 }
3375 }
3376 }
3377
3378 async fn create_authority_record(
3379 &self,
3380 expected: Option<&PublishedAuthorityHead>,
3381 candidate: &LeaderAuthorityRecord,
3382 ) -> Result<AuthorityCreateOutcome, LeaseError> {
3383 let encoded = encode_authority_record(candidate)?;
3384 let expected_pointer = match expected {
3385 None => {
3386 if candidate.lease.seq != 1 {
3387 return Err(LeaseError::Invalid(format!(
3388 "cannot append authority sequence {} to an empty published namespace",
3389 candidate.lease.seq
3390 )));
3391 }
3392 None
3393 }
3394 Some(head) if head.record.lease.seq == candidate.lease.seq => {
3395 return if head.record == *candidate {
3396 Ok(AuthorityCreateOutcome::ExistingIdentical)
3397 } else {
3398 Ok(AuthorityCreateOutcome::Contended(Box::new(
3399 head.record.clone(),
3400 )))
3401 };
3402 }
3403 Some(head) if head.record.lease.seq > candidate.lease.seq => {
3404 return Ok(AuthorityCreateOutcome::Contended(Box::new(
3405 head.record.clone(),
3406 )));
3407 }
3408 Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3409 Some(&head.pointer)
3410 }
3411 Some(head) => {
3412 return Err(LeaseError::Invalid(format!(
3413 "cannot append authority sequence {} after published sequence {}",
3414 candidate.lease.seq, head.record.lease.seq
3415 )));
3416 }
3417 };
3418 let options = PutOptions {
3419 mode: PutMode::Create,
3420 ..PutOptions::default()
3421 };
3422 let result = self
3423 .store
3424 .put_opts(
3425 &lease_path(candidate.lease.seq),
3426 PutPayload::from(encoded),
3427 options,
3428 )
3429 .await;
3430 let created = result.is_ok();
3431 if let Err(error) = result {
3432 let Some(at_sequence) =
3433 read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3434 else {
3435 return Err(LeaseError::Io(format!(
3436 "authority record {} write failed ({error}) and no record was durable",
3437 candidate.lease.seq
3438 )));
3439 };
3440 if at_sequence != *candidate {
3441 let winner = self.load_record().await?.ok_or_else(|| {
3442 LeaseError::Io("authority contender was not published or readable".into())
3443 })?;
3444 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3445 }
3446 }
3447
3448 let published_sequence = self
3449 .publish_authority_head(candidate.lease.seq, expected_pointer)
3450 .await?;
3451 if published_sequence > candidate.lease.seq {
3452 if created {
3453 self.schedule_history_prune();
3454 }
3455 let winner = self
3456 .load_record()
3457 .await?
3458 .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3459 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3460 }
3461 self.schedule_history_prune();
3462 if created {
3463 Ok(AuthorityCreateOutcome::Created)
3464 } else {
3465 Ok(AuthorityCreateOutcome::ExistingIdentical)
3466 }
3467 }
3468
3469 pub(super) async fn seal_catalog(
3478 &self,
3479 proof: &LeaderProof,
3480 manifest: &CatalogManifest,
3481 ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3482 if !proof.is_canonical() {
3483 return Err(CatalogManifestError::Fenced);
3484 }
3485 let (encoded, reference) = manifest.encode_and_reference()?;
3486
3487 let current = self
3488 .load_record()
3489 .await?
3490 .ok_or(CatalogManifestError::Fenced)?;
3491 if let Some(sealed) = ¤t.lease.catalog_manifest {
3492 let durable = self.load_catalog_manifest(sealed).await?;
3493 return if durable == *manifest {
3494 Ok(CatalogSealOutcome::ExistingIdentical)
3495 } else {
3496 Err(CatalogManifestError::Conflict)
3497 };
3498 }
3499 if !current.lease.matches_proof(proof) {
3500 return Err(CatalogManifestError::Fenced);
3501 }
3502 self.ensure_catalog_manifest_blob(&encoded, &reference)
3503 .await?;
3504
3505 loop {
3506 let published = self
3507 .load_published_authority_head()
3508 .await?
3509 .ok_or(CatalogManifestError::Fenced)?;
3510 let current = &published.record;
3511 if let Some(sealed) = ¤t.lease.catalog_manifest {
3512 let durable = self.load_catalog_manifest(sealed).await?;
3513 return if durable == *manifest {
3514 Ok(CatalogSealOutcome::ExistingIdentical)
3515 } else {
3516 Err(CatalogManifestError::Conflict)
3517 };
3518 }
3519 if !current.lease.matches_proof(proof) {
3520 return Err(CatalogManifestError::Fenced);
3521 }
3522
3523 let base_sequence = current.lease.seq;
3524 let candidate_lease = LeaderLease {
3525 seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3526 CatalogManifestError::Authority(LeaseError::Invalid(
3527 "lease sequence exhausted".into(),
3528 ))
3529 })?,
3530 renewal_sequence: current.lease.renewal_sequence,
3531 token: current.lease.token,
3532 owner: current.lease.owner.clone(),
3533 expires_at_ms: current.lease.expires_at_ms,
3534 catalog_manifest: Some(reference.clone()),
3535 };
3536 let candidate = current.preserve_with_lease(candidate_lease);
3537 match self
3538 .create_authority_record(Some(&published), &candidate)
3539 .await?
3540 {
3541 AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3542 AuthorityCreateOutcome::ExistingIdentical => {
3543 return Ok(CatalogSealOutcome::ExistingIdentical);
3544 }
3545 AuthorityCreateOutcome::Contended(winner) => {
3546 if let Some(sealed) = &winner.lease.catalog_manifest {
3547 let durable = self.load_catalog_manifest(sealed).await?;
3548 return if durable == *manifest {
3549 Ok(CatalogSealOutcome::ExistingIdentical)
3550 } else {
3551 Err(CatalogManifestError::Conflict)
3552 };
3553 }
3554 if !winner.lease.matches_proof(proof) {
3555 return Err(CatalogManifestError::Fenced);
3556 }
3557 if winner.lease.seq <= base_sequence {
3558 return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3559 "catalog seal contention did not advance the authority sequence".into(),
3560 )));
3561 }
3562 tokio::task::yield_now().await;
3563 }
3564 }
3565 }
3566 }
3567
3568 async fn audited_cluster_outcomes_from(
3569 &self,
3570 head: &LeaderAuthorityRecord,
3571 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3572 let floor = head.outcome_floor.as_ref();
3573 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3574 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3575 let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3576 let mut terminal_newest_first = Vec::new();
3577 let mut terminal_links = Vec::new();
3578 let mut link = head.outcome_head;
3579 let mut expected_commit_head = head.commit_head;
3580 let mut traversed = 0;
3581 while let Some(current) = link {
3582 if current.epoch < authority_before_epoch {
3583 break;
3584 }
3585 if !consume_live_authority_link(&mut traversed) {
3586 return Err(DecisionError::Conflict(format!(
3587 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3588 ))
3589 .into());
3590 }
3591 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3592 record.clone()
3593 } else {
3594 let record = read_authority_record(self.store.as_ref(), current.sequence)
3595 .await?
3596 .ok_or_else(|| {
3597 DecisionError::InventoryChanged(format!(
3598 "cluster outcome authority record {} disappeared during audit",
3599 current.sequence
3600 ))
3601 })?;
3602 terminal_records.insert(current.sequence, record.clone());
3603 record
3604 };
3605 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3606 DecisionError::Conflict(format!(
3607 "cluster outcome head epoch {} points to non-outcome authority record {}",
3608 current.epoch, current.sequence
3609 ))
3610 })?;
3611 if record.outcome_head != Some(current)
3612 || outcome.epoch != current.epoch
3613 || outcome.checkpoint_id != current.checkpoint_id
3614 {
3615 return Err(DecisionError::Conflict(format!(
3616 "cluster outcome link epoch {} sequence {} does not match its authority record",
3617 current.epoch, current.sequence
3618 ))
3619 .into());
3620 }
3621 if record.commit_head != expected_commit_head {
3622 return Err(DecisionError::Conflict(format!(
3623 "cluster outcome epoch {} did not preserve the exact Commit head",
3624 outcome.epoch
3625 ))
3626 .into());
3627 }
3628 expected_commit_head = if outcome.is_commit() {
3629 record.previous_commit
3630 } else {
3631 record.commit_head
3632 };
3633 terminal_newest_first.push(outcome);
3634 terminal_links.push(current);
3635 link = record.previous_outcome;
3636 }
3637
3638 if let Some(floor) = floor {
3639 match (
3640 link,
3641 floor.terminal_anchor_link,
3642 floor.terminal_anchor.as_ref(),
3643 ) {
3644 (Some(link), Some(anchor_link), Some(anchor))
3645 if link == anchor_link
3646 && link.epoch == anchor.epoch
3647 && link.checkpoint_id == anchor.checkpoint_id => {}
3648 (None, None, None) => {}
3649 _ => {
3650 return Err(DecisionError::Conflict(format!(
3651 "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3652 floor.authority_before_epoch
3653 ))
3654 .into());
3655 }
3656 }
3657 } else if link.is_some() {
3658 return Err(DecisionError::Conflict(
3659 "cluster outcome chain stopped without a durable retention floor".into(),
3660 )
3661 .into());
3662 }
3663
3664 let mut commit_newest_first = Vec::new();
3665 let mut commit_links = Vec::new();
3666 link = head.commit_head;
3667 traversed = 0;
3668 while let Some(current) = link {
3669 if current.epoch < artifact_before_epoch {
3670 break;
3671 }
3672 if !consume_live_authority_link(&mut traversed) {
3673 return Err(DecisionError::Conflict(format!(
3674 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3675 ))
3676 .into());
3677 }
3678 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3679 record.clone()
3680 } else {
3681 read_authority_record(self.store.as_ref(), current.sequence)
3682 .await?
3683 .ok_or_else(|| {
3684 DecisionError::InventoryChanged(format!(
3685 "cluster Commit authority record {} disappeared during audit",
3686 current.sequence
3687 ))
3688 })?
3689 };
3690 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3691 DecisionError::Conflict(format!(
3692 "cluster Commit head epoch {} points to non-outcome authority record {}",
3693 current.epoch, current.sequence
3694 ))
3695 })?;
3696 if !outcome.is_commit()
3697 || record.commit_head != Some(current)
3698 || outcome.epoch != current.epoch
3699 || outcome.checkpoint_id != current.checkpoint_id
3700 {
3701 return Err(DecisionError::Conflict(format!(
3702 "cluster Commit link epoch {} sequence {} does not match its authority record",
3703 current.epoch, current.sequence
3704 ))
3705 .into());
3706 }
3707 commit_newest_first.push(outcome);
3708 commit_links.push(current);
3709 link = record.previous_commit;
3710 }
3711
3712 if let Some(floor) = floor {
3713 match (
3714 link,
3715 floor.committed_anchor_link,
3716 floor.committed_anchor.as_ref(),
3717 ) {
3718 (Some(link), Some(anchor_link), Some(anchor))
3719 if link == anchor_link
3720 && link.epoch == anchor.epoch
3721 && link.checkpoint_id == anchor.checkpoint_id => {}
3722 (None, None, None) => {}
3723 _ => {
3724 return Err(DecisionError::Conflict(format!(
3725 "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3726 floor.artifact_before_epoch
3727 ))
3728 .into());
3729 }
3730 }
3731 } else if link.is_some() {
3732 return Err(DecisionError::Conflict(
3733 "cluster Commit chain stopped without a durable retention floor".into(),
3734 )
3735 .into());
3736 }
3737
3738 terminal_newest_first.reverse();
3739 terminal_links.reverse();
3740 commit_newest_first.reverse();
3741 commit_links.reverse();
3742 let boundary_commit_head = floor
3743 .and_then(|floor| floor.terminal_anchor.as_ref())
3744 .and_then(|anchor| {
3745 commit_links
3746 .iter()
3747 .rev()
3748 .find(|link| link.epoch <= anchor.epoch)
3749 .copied()
3750 .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3751 });
3752 if expected_commit_head != boundary_commit_head {
3753 return Err(DecisionError::Conflict(
3754 "cluster terminal chain does not preserve Commit continuity at its durable floor"
3755 .into(),
3756 )
3757 .into());
3758 }
3759 for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3760 if !outcome.is_commit() {
3761 continue;
3762 }
3763 let commit_link = commit_links
3764 .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3765 .ok()
3766 .map(|index| commit_links[index]);
3767 if commit_link != Some(*terminal_link) {
3768 return Err(DecisionError::Conflict(format!(
3769 "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3770 outcome.epoch
3771 ))
3772 .into());
3773 }
3774 }
3775 if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3776 floor
3777 .terminal_anchor
3778 .as_ref()
3779 .zip(floor.terminal_anchor_link)
3780 }) {
3781 if terminal_anchor.is_commit() {
3782 let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3783 commit_links
3784 .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3785 .ok()
3786 .map(|index| commit_links[index])
3787 } else {
3788 floor.and_then(|floor| {
3789 (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3790 .then_some(floor.committed_anchor_link)
3791 .flatten()
3792 })
3793 };
3794 if commit_link != Some(terminal_anchor_link) {
3795 return Err(DecisionError::Conflict(format!(
3796 "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3797 terminal_anchor.epoch
3798 ))
3799 .into());
3800 }
3801 }
3802 }
3803 let mut outcomes = Vec::with_capacity(
3804 terminal_newest_first
3805 .len()
3806 .saturating_add(commit_newest_first.len())
3807 .saturating_add(2),
3808 );
3809 outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3810 outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3811 outcomes.extend(terminal_newest_first);
3812 outcomes.extend(commit_newest_first);
3813 outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3814 let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3815 for outcome in outcomes {
3816 if let Some(previous) = merged.last() {
3817 if previous.epoch == outcome.epoch {
3818 if previous != &outcome {
3819 return Err(DecisionError::Conflict(format!(
3820 "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3821 outcome.epoch
3822 ))
3823 .into());
3824 }
3825 continue;
3826 }
3827 }
3828 merged.push(outcome);
3829 }
3830
3831 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3832 .load_or_create_deployment_id()
3833 .await?;
3834 for outcome in &merged {
3835 if outcome.deployment_id != expected_deployment {
3836 return Err(DecisionError::Conflict(format!(
3837 "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3838 outcome.epoch, outcome.deployment_id, expected_deployment
3839 ))
3840 .into());
3841 }
3842 }
3843 for pair in merged.windows(2) {
3844 let previous = &pair[0];
3845 let current = &pair[1];
3846 if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3847 return Err(DecisionError::Conflict(format!(
3848 "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3849 previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3850 ))
3851 .into());
3852 }
3853 }
3854 Ok(ClusterOutcomeAuditSnapshot {
3855 outcomes: Arc::from(merged),
3856 terminal_links: Arc::from(terminal_links),
3857 commit_links: Arc::from(commit_links),
3858 })
3859 }
3860
3861 fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3862 ClusterOutcomeAuditKey {
3863 terminal_head: head.outcome_head,
3864 commit_head: head.commit_head,
3865 floor: head.outcome_floor.clone(),
3866 }
3867 }
3868
3869 fn cached_cluster_outcome_audit(
3870 &self,
3871 key: &ClusterOutcomeAuditKey,
3872 authority_sequence: u64,
3873 ) -> Option<ClusterOutcomeAuditSnapshot> {
3874 let mut cache = self.outcome_audit_cache.lock();
3875 let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3876 cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3877 Some(cached.snapshot.clone())
3878 }
3879
3880 fn install_cluster_outcome_audit(
3881 &self,
3882 key: ClusterOutcomeAuditKey,
3883 authority_sequence: u64,
3884 snapshot: ClusterOutcomeAuditSnapshot,
3885 ) {
3886 let mut cache = self.outcome_audit_cache.lock();
3887 if cache.as_ref().is_none_or(|cached| {
3888 cached.authority_sequence < authority_sequence
3889 || (cached.authority_sequence == authority_sequence && cached.key == key)
3890 }) {
3891 *cache = Some(ClusterOutcomeAuditCache {
3892 key,
3893 authority_sequence,
3894 snapshot,
3895 });
3896 }
3897 }
3898
3899 fn cluster_outcome_audit_gate(
3900 &self,
3901 key: &ClusterOutcomeAuditKey,
3902 ) -> Arc<tokio::sync::Mutex<()>> {
3903 let mut flights = self.outcome_audit_flights.lock();
3904 flights.retain(|flight| flight.gate.strong_count() != 0);
3905 if let Some(gate) = flights
3906 .iter()
3907 .find(|flight| flight.key == *key)
3908 .and_then(|flight| flight.gate.upgrade())
3909 {
3910 return gate;
3911 }
3912 let gate = Arc::new(tokio::sync::Mutex::new(()));
3913 flights.push(ClusterOutcomeAuditFlight {
3914 key: key.clone(),
3915 gate: Arc::downgrade(&gate),
3916 });
3917 gate
3918 }
3919
3920 async fn cached_audited_cluster_outcomes_from(
3921 &self,
3922 head: &LeaderAuthorityRecord,
3923 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3924 let key = Self::cluster_outcome_audit_key(head);
3925 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3926 return Ok(snapshot);
3927 }
3928
3929 let gate = self.cluster_outcome_audit_gate(&key);
3930 let _guard = gate.lock().await;
3931 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3932 return Ok(snapshot);
3933 }
3934
3935 let snapshot = self.audited_cluster_outcomes_from(head).await?;
3936 self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
3937 Ok(snapshot)
3938 }
3939
3940 fn outcomes_retained_by_floor(
3941 floor: &AuthorityOutcomeFloor,
3942 snapshot: &ClusterOutcomeAuditSnapshot,
3943 ) -> ClusterOutcomeAuditSnapshot {
3944 let outcomes = &snapshot.outcomes;
3945 let mut retained = Vec::with_capacity(outcomes.len());
3946 if let Some(anchor) = floor.committed_anchor.as_ref() {
3947 retained.push(anchor.clone());
3948 }
3949 if let Some(anchor) = floor.terminal_anchor.as_ref() {
3950 if retained.last() != Some(anchor) {
3951 retained.push(anchor.clone());
3952 }
3953 }
3954 retained.extend(
3955 outcomes
3956 .iter()
3957 .filter(|outcome| {
3958 outcome.epoch >= floor.authority_before_epoch
3959 || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
3960 })
3961 .cloned(),
3962 );
3963 retained.sort_unstable_by_key(|outcome| outcome.epoch);
3964 retained.dedup();
3965 ClusterOutcomeAuditSnapshot {
3966 outcomes: Arc::from(retained),
3967 terminal_links: Arc::from(
3968 snapshot
3969 .terminal_links
3970 .iter()
3971 .filter(|link| link.epoch >= floor.authority_before_epoch)
3972 .copied()
3973 .collect::<Vec<_>>(),
3974 ),
3975 commit_links: Arc::from(
3976 snapshot
3977 .commit_links
3978 .iter()
3979 .filter(|link| link.epoch >= floor.artifact_before_epoch)
3980 .copied()
3981 .collect::<Vec<_>>(),
3982 ),
3983 }
3984 }
3985
3986 async fn build_cluster_outcome_floor(
3987 &self,
3988 current: &LeaderAuthorityRecord,
3989 snapshot: &ClusterOutcomeAuditSnapshot,
3990 artifact_before_epoch: u64,
3991 authority_before_epoch: u64,
3992 ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
3993 let terminal_anchor = snapshot
3994 .outcomes
3995 .iter()
3996 .rev()
3997 .find(|outcome| outcome.epoch < authority_before_epoch)
3998 .cloned();
3999 let terminal_anchor_link = match terminal_anchor.as_ref() {
4000 Some(anchor)
4001 if current
4002 .outcome_floor
4003 .as_ref()
4004 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
4005 {
4006 current
4007 .outcome_floor
4008 .as_ref()
4009 .and_then(|floor| floor.terminal_anchor_link)
4010 }
4011 Some(anchor) => {
4012 let index = snapshot
4013 .terminal_links
4014 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4015 .map_err(|_| {
4016 DecisionError::Conflict(format!(
4017 "cluster outcome epoch {} has no audited authority link",
4018 anchor.epoch
4019 ))
4020 })?;
4021 let link = snapshot.terminal_links[index];
4022 if link.checkpoint_id != anchor.checkpoint_id {
4023 return Err(DecisionError::Conflict(format!(
4024 "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
4025 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4026 ))
4027 .into());
4028 }
4029 Some(link)
4030 }
4031 None => None,
4032 };
4033 let committed_anchor = snapshot
4034 .outcomes
4035 .iter()
4036 .rev()
4037 .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
4038 .cloned();
4039 let committed_anchor_link = match committed_anchor.as_ref() {
4040 Some(anchor)
4041 if current
4042 .outcome_floor
4043 .as_ref()
4044 .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
4045 {
4046 current
4047 .outcome_floor
4048 .as_ref()
4049 .and_then(|floor| floor.committed_anchor_link)
4050 }
4051 Some(anchor) => {
4052 let index = snapshot
4053 .commit_links
4054 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4055 .map_err(|_| {
4056 DecisionError::Conflict(format!(
4057 "cluster Commit epoch {} has no audited Commit link",
4058 anchor.epoch
4059 ))
4060 })?;
4061 let link = snapshot.commit_links[index];
4062 if link.checkpoint_id != anchor.checkpoint_id {
4063 return Err(DecisionError::Conflict(format!(
4064 "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
4065 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4066 ))
4067 .into());
4068 }
4069 Some(link)
4070 }
4071 None => None,
4072 };
4073 let floor = AuthorityOutcomeFloor {
4074 deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
4075 .load_or_create_deployment_id()
4076 .await?,
4077 artifact_before_epoch,
4078 authority_before_epoch,
4079 terminal_anchor,
4080 terminal_anchor_link,
4081 committed_anchor,
4082 committed_anchor_link,
4083 };
4084 floor.validate()?;
4085 Ok(floor)
4086 }
4087
4088 async fn compact_cluster_outcome_history_before_append(
4089 &self,
4090 proof: &LeaderProof,
4091 current: &LeaderAuthorityRecord,
4092 snapshot: &ClusterOutcomeAuditSnapshot,
4093 ) -> Result<bool, ClusterCheckpointAuthorityError> {
4094 if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
4095 return Ok(false);
4096 }
4097 let authority_before_epoch = snapshot.terminal_links
4098 [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
4099 .epoch;
4100 let artifact_before_epoch = current
4101 .outcome_floor
4102 .as_ref()
4103 .map_or(0, |floor| floor.artifact_before_epoch);
4104 let floor = self
4105 .build_cluster_outcome_floor(
4106 current,
4107 snapshot,
4108 artifact_before_epoch,
4109 authority_before_epoch,
4110 )
4111 .await?;
4112 let published = self
4115 .load_published_authority_head()
4116 .await?
4117 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4118 let rechecked = &published.record;
4119 if !rechecked.lease.matches_proof(proof) {
4120 return Err(ClusterCheckpointAuthorityError::Fenced);
4121 }
4122 if rechecked.outcome_head != current.outcome_head
4123 || rechecked.commit_head != current.commit_head
4124 || rechecked.outcome_floor != current.outcome_floor
4125 {
4126 return Ok(true);
4127 }
4128
4129 let sequence = rechecked
4130 .lease
4131 .seq
4132 .checked_add(1)
4133 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4134 let mut lease = rechecked.lease.clone();
4135 lease.seq = sequence;
4136 let mut next = rechecked.preserve_with_lease(lease);
4137 next.outcome_floor = Some(floor.clone());
4138 next.validate()?;
4139 match self
4140 .create_authority_record(Some(&published), &next)
4141 .await?
4142 {
4143 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4144 self.install_cluster_outcome_audit(
4145 Self::cluster_outcome_audit_key(&next),
4146 next.lease.seq,
4147 Self::outcomes_retained_by_floor(&floor, snapshot),
4148 );
4149 Ok(true)
4150 }
4151 AuthorityCreateOutcome::Contended(winner) => {
4152 if !winner.lease.matches_proof(proof) {
4153 return Err(ClusterCheckpointAuthorityError::Fenced);
4154 }
4155 if winner.lease.seq <= current.lease.seq {
4156 return Err(LeaseError::Invalid(
4157 "outcome history compaction contention did not advance the authority sequence"
4158 .into(),
4159 )
4160 .into());
4161 }
4162 Ok(true)
4163 }
4164 }
4165 }
4166
4167 async fn audited_cluster_outcomes(
4168 &self,
4169 ) -> Result<
4170 (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
4171 ClusterCheckpointAuthorityError,
4172 > {
4173 const AUDIT_RETRIES: usize = 3;
4174 for attempt in 0..AUDIT_RETRIES {
4175 let Some(head) = self.load_record().await? else {
4176 *self.outcome_audit_cache.lock() = None;
4177 return Ok((None, Arc::from([])));
4178 };
4179 match self.cached_audited_cluster_outcomes_from(&head).await {
4180 Err(ClusterCheckpointAuthorityError::Decision(
4181 DecisionError::InventoryChanged(_),
4182 )) if attempt + 1 < AUDIT_RETRIES => {
4183 tokio::task::yield_now().await;
4184 }
4185 Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
4186 Err(error) => return Err(error),
4187 }
4188 }
4189 Err(DecisionError::InventoryChanged(
4190 "cluster outcome audit exhausted stability retries".into(),
4191 )
4192 .into())
4193 }
4194
4195 async fn audited_assignment_decisions_from(
4196 &self,
4197 head: &LeaderAuthorityRecord,
4198 ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4199 let mut newest_first = Vec::new();
4200 let floor = head.assignment_decision_floor.as_ref();
4201 let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
4202 let mut stopped_at_anchor = false;
4203 let mut link = head.assignment_decision_head;
4204 let mut traversed = 0;
4205 while let Some(current) = link {
4206 if !consume_live_authority_link(&mut traversed) {
4207 return Err(DecisionError::Conflict(format!(
4208 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
4209 ))
4210 .into());
4211 }
4212 if current.target_version < before_target_version {
4213 if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
4214 return Err(DecisionError::Conflict(format!(
4215 "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
4216 ))
4217 .into());
4218 }
4219 stopped_at_anchor = true;
4220 break;
4221 }
4222 let record = read_authority_record(self.store.as_ref(), current.sequence)
4223 .await?
4224 .ok_or_else(|| {
4225 DecisionError::InventoryChanged(format!(
4226 "assignment decision authority record {} disappeared during audit",
4227 current.sequence
4228 ))
4229 })?;
4230 let decision = record.assignment_decision.clone().ok_or_else(|| {
4231 DecisionError::Conflict(format!(
4232 "assignment decision head version {} points to non-decision authority record {}",
4233 current.target_version, current.sequence
4234 ))
4235 })?;
4236 if record.assignment_decision_head != Some(current)
4237 || decision.target_version() != current.target_version
4238 {
4239 return Err(DecisionError::Conflict(format!(
4240 "assignment decision link version {} sequence {} does not match its authority record",
4241 current.target_version, current.sequence
4242 ))
4243 .into());
4244 }
4245 newest_first.push(decision);
4246 link = record.previous_assignment_decision;
4247 }
4248 if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4249 return Err(DecisionError::Conflict(
4250 "assignment decision chain stopped without its durable retention anchor".into(),
4251 )
4252 .into());
4253 }
4254 newest_first.reverse();
4255 if let Some(pair) = newest_first
4256 .windows(2)
4257 .find(|pair| pair[0].target_version() >= pair[1].target_version())
4258 {
4259 return Err(DecisionError::Conflict(format!(
4260 "assignment decisions regress from version {} to {}",
4261 pair[0].target_version(),
4262 pair[1].target_version()
4263 ))
4264 .into());
4265 }
4266 Ok(newest_first)
4267 }
4268
4269 async fn exact_assignment_decision_link(
4270 &self,
4271 head: &LeaderAuthorityRecord,
4272 decision: &AuthorityAssignmentDecision,
4273 ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4274 if let Some(floor) = head.assignment_decision_floor.as_ref() {
4275 if floor.terminal_anchor.as_ref() == Some(decision) {
4276 return floor.terminal_anchor_link.ok_or_else(|| {
4277 DecisionError::Conflict(
4278 "assignment decision floor lost its terminal authority link".into(),
4279 )
4280 .into()
4281 });
4282 }
4283 }
4284 let before_target_version = head
4285 .assignment_decision_floor
4286 .as_ref()
4287 .map_or(0, |floor| floor.before_target_version);
4288 let mut link = head.assignment_decision_head;
4289 let mut traversed = 0;
4290 while let Some(current) = link {
4291 if !consume_live_authority_link(&mut traversed) {
4292 return Err(DecisionError::Conflict(format!(
4293 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4294 ))
4295 .into());
4296 }
4297 if current.target_version < before_target_version {
4298 break;
4299 }
4300 let record = read_authority_record(self.store.as_ref(), current.sequence)
4301 .await?
4302 .ok_or_else(|| {
4303 DecisionError::InventoryChanged(format!(
4304 "assignment decision authority record {} disappeared during exact lookup",
4305 current.sequence
4306 ))
4307 })?;
4308 if record.assignment_decision.as_ref() == Some(decision) {
4309 return Ok(current);
4310 }
4311 link = record.previous_assignment_decision;
4312 }
4313 Err(DecisionError::Conflict(format!(
4314 "assignment decision version {} is not linked from the durable authority head",
4315 decision.target_version()
4316 ))
4317 .into())
4318 }
4319
4320 async fn reject_consumed_checkpoint_assignment(
4321 &self,
4322 current: &LeaderAuthorityRecord,
4323 assignment_fence: &CheckpointAssignmentFence,
4324 ) -> Result<(), ClusterCheckpointAuthorityError> {
4325 let decisions = self.audited_assignment_decisions_from(current).await?;
4326 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4327 if assignment_fence.assignment_version < floor.before_target_version {
4331 return Err(DecisionError::Conflict(format!(
4332 "checkpoint assignment version {} is below durable assignment-decision floor {}",
4333 assignment_fence.assignment_version, floor.before_target_version
4334 ))
4335 .into());
4336 }
4337 }
4338 let Some(newest) = decisions.last() else {
4339 return Ok(());
4340 };
4341 if newest.target_version() > assignment_fence.assignment_version {
4342 return Err(DecisionError::Conflict(format!(
4343 "checkpoint assignment version {} was consumed by assignment decision version {}",
4344 assignment_fence.assignment_version,
4345 newest.target_version()
4346 ))
4347 .into());
4348 }
4349 if newest.target_version() == assignment_fence.assignment_version
4350 && newest.materialized_target() != *assignment_fence
4351 {
4352 return Err(DecisionError::Conflict(format!(
4353 "checkpoint assignment version {} does not match its materialized assignment decision",
4354 assignment_fence.assignment_version
4355 ))
4356 .into());
4357 }
4358 Ok(())
4359 }
4360
4361 pub async fn begin_cluster_checkpoint_artifacts(
4370 &self,
4371 proof: &LeaderProof,
4372 inventory: CheckpointArtifactInventory,
4373 ) -> Result<CheckpointArtifactInventory, ClusterCheckpointAuthorityError> {
4374 if !proof.is_canonical() {
4375 return Err(ClusterCheckpointAuthorityError::Fenced);
4376 }
4377 inventory.validate().map_err(DecisionError::Conflict)?;
4378 let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
4379 DecisionError::Conflict(
4380 "cluster checkpoint artifact inventory requires an assignment fence".into(),
4381 )
4382 })?;
4383 if assignment_fence.participant_incarnation(proof.owner.node_id)
4384 != Some(proof.owner.boot_id)
4385 {
4386 return Err(DecisionError::Conflict(
4387 "checkpoint artifact admitting leader is outside the assignment fence".into(),
4388 )
4389 .into());
4390 }
4391 let initial = self
4392 .load_record()
4393 .await?
4394 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4395 if !initial.lease.matches_proof(proof) {
4396 return Err(ClusterCheckpointAuthorityError::Fenced);
4397 }
4398 let deployment_id = CheckpointDecisionStore::new(Arc::clone(&self.store))
4399 .load_or_create_deployment_id()
4400 .await?;
4401 if inventory.deployment_id != deployment_id {
4402 return Err(DecisionError::Conflict(format!(
4403 "checkpoint artifact inventory deployment {} does not match authority deployment {deployment_id}",
4404 inventory.deployment_id
4405 ))
4406 .into());
4407 }
4408
4409 loop {
4410 let published = self
4411 .load_published_authority_head()
4412 .await?
4413 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4414 let current = &published.record;
4415 if !current.lease.matches_proof(proof) {
4416 return Err(ClusterCheckpointAuthorityError::Fenced);
4417 }
4418 self.reject_consumed_checkpoint_assignment(current, assignment_fence)
4419 .await?;
4420 if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
4421 if active == &inventory
4422 && current.active_checkpoint_artifact_leader_proof.as_ref() == Some(proof)
4423 && current.outcome_head.is_none_or(|head| {
4424 head.epoch != inventory.attempt.epoch
4425 || head.checkpoint_id != inventory.attempt.checkpoint_id
4426 })
4427 {
4428 return Ok(active.clone());
4429 }
4430 return Err(DecisionError::Conflict(format!(
4431 "checkpoint {} still has unresolved or inherited cluster artifacts",
4432 active.attempt.checkpoint_id
4433 ))
4434 .into());
4435 }
4436 if let Some(head) = current.outcome_head {
4437 let terminal =
4438 crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
4439 if inventory.attempt.relation_to(terminal)
4440 != crate::checkpoint::CheckpointAttemptRelation::Newer
4441 {
4442 return Err(DecisionError::Conflict(format!(
4443 "checkpoint {} does not advance terminal checkpoint {}",
4444 inventory.attempt.checkpoint_id, head.checkpoint_id
4445 ))
4446 .into());
4447 }
4448 }
4449
4450 let sequence =
4451 current.lease.seq.checked_add(1).ok_or_else(|| {
4452 LeaseError::Invalid("leader authority sequence exhausted".into())
4453 })?;
4454 let mut lease = current.lease.clone();
4455 lease.seq = sequence;
4456 let mut next = current.preserve_with_lease(lease);
4457 next.active_checkpoint_artifacts = Some(inventory.clone());
4458 next.active_checkpoint_artifact_leader_proof = Some(proof.clone());
4459 next.validate()?;
4460 match self
4461 .create_authority_record(Some(&published), &next)
4462 .await?
4463 {
4464 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4465 return Ok(inventory);
4466 }
4467 AuthorityCreateOutcome::Contended(winner) => {
4468 if !winner.lease.matches_proof(proof) {
4469 return Err(ClusterCheckpointAuthorityError::Fenced);
4470 }
4471 tokio::task::yield_now().await;
4472 }
4473 }
4474 }
4475 }
4476
4477 pub async fn finish_cluster_checkpoint_artifact_cleanup(
4483 &self,
4484 proof: &LeaderProof,
4485 expected: &CheckpointArtifactInventory,
4486 ) -> Result<(), ClusterCheckpointAuthorityError> {
4487 if !proof.is_canonical() {
4488 return Err(ClusterCheckpointAuthorityError::Fenced);
4489 }
4490 expected.validate().map_err(DecisionError::Conflict)?;
4491 if expected.assignment_fence.is_none() {
4492 return Err(DecisionError::Conflict(
4493 "cluster checkpoint artifact inventory requires an assignment fence".into(),
4494 )
4495 .into());
4496 }
4497
4498 loop {
4499 let published = self
4500 .load_published_authority_head()
4501 .await?
4502 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4503 let current = &published.record;
4504 if !current.lease.matches_proof(proof) {
4505 return Err(ClusterCheckpointAuthorityError::Fenced);
4506 }
4507 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4508 let matching_abort = snapshot.outcomes.iter().any(|outcome| {
4509 matches!(outcome.verdict, CheckpointVerdict::Abort)
4510 && outcome.deployment_id == expected.deployment_id
4511 && outcome.epoch == expected.attempt.epoch
4512 && outcome.checkpoint_id == expected.attempt.checkpoint_id
4513 && outcome.assignment_fence.as_ref() == expected.assignment_fence.as_ref()
4514 });
4515 match current.active_checkpoint_artifacts.as_ref() {
4516 None if matching_abort => return Ok(()),
4517 None => {
4518 return Err(DecisionError::Conflict(format!(
4519 "checkpoint {} has no retained cluster artifact inventory",
4520 expected.attempt.checkpoint_id
4521 ))
4522 .into());
4523 }
4524 Some(active) if active != expected => {
4525 return Err(DecisionError::Conflict(format!(
4526 "checkpoint {} still has unresolved cluster artifacts",
4527 active.attempt.checkpoint_id
4528 ))
4529 .into());
4530 }
4531 Some(_) if !matching_abort => {
4532 return Err(DecisionError::Conflict(format!(
4533 "checkpoint {} has no matching durable Abort",
4534 expected.attempt.checkpoint_id
4535 ))
4536 .into());
4537 }
4538 Some(_) => {}
4539 }
4540
4541 let sequence =
4542 current.lease.seq.checked_add(1).ok_or_else(|| {
4543 LeaseError::Invalid("leader authority sequence exhausted".into())
4544 })?;
4545 let mut lease = current.lease.clone();
4546 lease.seq = sequence;
4547 let mut next = current.preserve_with_lease(lease);
4548 next.active_checkpoint_artifacts = None;
4549 next.active_checkpoint_artifact_leader_proof = None;
4550 next.validate()?;
4551 match self
4552 .create_authority_record(Some(&published), &next)
4553 .await?
4554 {
4555 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4556 return Ok(());
4557 }
4558 AuthorityCreateOutcome::Contended(winner) => {
4559 if !winner.lease.matches_proof(proof) {
4560 return Err(ClusterCheckpointAuthorityError::Fenced);
4561 }
4562 tokio::task::yield_now().await;
4563 }
4564 }
4565 }
4566 }
4567
4568 pub async fn record_cluster_outcome(
4577 &self,
4578 proof: &LeaderProof,
4579 epoch: u64,
4580 checkpoint_id: u64,
4581 assignment_fence: CheckpointAssignmentFence,
4582 verdict: CheckpointVerdict,
4583 committed_checkpoint: Option<CommittedCheckpointRef>,
4584 ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4585 if !proof.is_canonical() {
4586 return Err(ClusterCheckpointAuthorityError::Fenced);
4587 }
4588 let attempt = crate::checkpoint::CheckpointAttempt::new(epoch, checkpoint_id);
4589 if !attempt.is_canonical() {
4590 return Err(DecisionError::Conflict(
4591 "cluster checkpoint outcomes require one nonzero canonical checkpoint ID".into(),
4592 )
4593 .into());
4594 }
4595 let initial = self
4596 .load_record()
4597 .await?
4598 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4599 if !initial.lease.matches_proof(proof) {
4600 return Err(ClusterCheckpointAuthorityError::Fenced);
4601 }
4602 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
4603 let (candidate, committed_index) = decisions
4604 .canonical_outcome_with_index(
4605 epoch,
4606 checkpoint_id,
4607 CheckpointScope::Cluster,
4608 Some(assignment_fence),
4609 Some(proof.clone()),
4610 verdict,
4611 committed_checkpoint,
4612 )
4613 .await?;
4614
4615 loop {
4616 let published = self
4617 .load_published_authority_head()
4618 .await?
4619 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4620 let current = &published.record;
4621 if !current.lease.matches_proof(proof) {
4622 return Err(ClusterCheckpointAuthorityError::Fenced);
4623 }
4624 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4625 let outcomes = &snapshot.outcomes;
4626 if let Some(winner) = outcomes
4627 .iter()
4628 .find(|outcome| outcome.epoch == candidate.epoch)
4629 {
4630 return if winner == &candidate {
4631 Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4632 } else {
4633 Ok(RecordOutcomeResult::Conflict {
4634 winner: winner.clone(),
4635 })
4636 };
4637 }
4638 let active = match current.active_checkpoint_artifacts.as_ref() {
4639 Some(active)
4640 if active.deployment_id == candidate.deployment_id
4641 && active.attempt.epoch == candidate.epoch
4642 && active.attempt.checkpoint_id == candidate.checkpoint_id
4643 && active.assignment_fence.as_ref()
4644 == candidate.assignment_fence.as_ref() =>
4645 {
4646 Some(active)
4647 }
4648 Some(_) => {
4649 return Err(DecisionError::Conflict(format!(
4650 "cluster checkpoint {} does not match the active artifact inventory",
4651 candidate.checkpoint_id
4652 ))
4653 .into());
4654 }
4655 None if candidate.is_commit() => {
4656 return Err(DecisionError::Conflict(format!(
4657 "cluster Commit checkpoint {} has no admitted artifact inventory",
4658 candidate.checkpoint_id
4659 ))
4660 .into());
4661 }
4662 None => None,
4663 };
4664 if candidate.is_commit()
4665 && current.active_checkpoint_artifact_leader_proof.as_ref() != Some(proof)
4666 {
4667 return Err(DecisionError::Conflict(format!(
4668 "takeover leader cannot Commit checkpoint {} admitted by an older leader term",
4669 candidate.checkpoint_id
4670 ))
4671 .into());
4672 }
4673 if let Some(last) = outcomes.last() {
4674 if candidate.checkpoint_id <= last.checkpoint_id {
4675 return Err(DecisionError::Conflict(format!(
4676 "cluster checkpoint {} does not advance durable checkpoint {}",
4677 candidate.checkpoint_id, last.checkpoint_id
4678 ))
4679 .into());
4680 }
4681 }
4682 if candidate.is_commit()
4683 && current
4684 .assignment_handoff_pin
4685 .as_ref()
4686 .is_some_and(|pin| candidate.assignment_fence.as_ref() != Some(&pin.target))
4687 {
4688 return Err(DecisionError::Conflict(
4689 "cluster Commit does not bind the active assignment handoff target".into(),
4690 )
4691 .into());
4692 }
4693 let (commit_index, expected_predecessor) = if candidate.is_commit() {
4694 let index = committed_index.as_ref().ok_or_else(|| {
4695 DecisionError::Conflict(
4696 "canonical cluster Commit is missing its committed checkpoint index".into(),
4697 )
4698 })?;
4699 let expected_predecessor = outcomes
4700 .iter()
4701 .rev()
4702 .find(|outcome| outcome.is_commit())
4703 .and_then(|outcome| outcome.committed_checkpoint.clone());
4704 if index.predecessor != expected_predecessor {
4705 return Err(DecisionError::Conflict(format!(
4706 "cluster Commit checkpoint {} does not extend the authoritative Commit head",
4707 candidate.checkpoint_id
4708 ))
4709 .into());
4710 }
4711 if active.is_none_or(|active| index.pipeline_identity != active.pipeline_identity) {
4712 return Err(DecisionError::Conflict(format!(
4713 "cluster Commit checkpoint {} does not match its admitted pipeline identity",
4714 candidate.checkpoint_id
4715 ))
4716 .into());
4717 }
4718 (Some(index), expected_predecessor)
4719 } else {
4720 (None, None)
4721 };
4722 if candidate.is_commit() && snapshot.commit_links.len() >= MAX_LIVE_AUTHORITY_LINKS {
4723 return Err(DecisionError::Conflict(format!(
4724 "live Commit retention reached the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound; advance the artifact-retention horizon before admitting another Commit"
4725 ))
4726 .into());
4727 }
4728 if let Some(floor) = current.outcome_floor.as_ref() {
4729 if candidate.deployment_id != floor.deployment_id
4730 || candidate.epoch < floor.authority_before_epoch
4731 {
4732 return Err(DecisionError::Conflict(format!(
4733 "cluster outcome epoch {} is below or outside authority floor {}",
4734 candidate.epoch, floor.authority_before_epoch
4735 ))
4736 .into());
4737 }
4738 }
4739 if Box::pin(
4740 self.compact_cluster_outcome_history_before_append(proof, current, &snapshot),
4741 )
4742 .await?
4743 {
4744 tokio::task::yield_now().await;
4745 continue;
4746 }
4747 if let (Some(index), Some(predecessor_ref)) =
4748 (commit_index, expected_predecessor.as_ref())
4749 {
4750 let predecessor = decisions.load_committed_checkpoint(predecessor_ref).await?;
4751 index
4752 .validate_predecessor_index(&predecessor)
4753 .map_err(DecisionError::Conflict)?;
4754 }
4755
4756 let base_sequence = current.lease.seq;
4757 let sequence = base_sequence
4758 .checked_add(1)
4759 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4760 let mut next = current.preserve_with_lease(LeaderLease {
4761 seq: sequence,
4762 renewal_sequence: current.lease.renewal_sequence,
4763 token: current.lease.token,
4764 owner: current.lease.owner.clone(),
4765 expires_at_ms: current.lease.expires_at_ms,
4766 catalog_manifest: current.lease.catalog_manifest.clone(),
4767 });
4768 next.checkpoint_outcome = Some(candidate.clone());
4769 next.previous_outcome = current.outcome_head;
4770 let new_link = OutcomeLink {
4771 sequence,
4772 epoch: candidate.epoch,
4773 checkpoint_id: candidate.checkpoint_id,
4774 };
4775 next.outcome_head = Some(new_link);
4776 if candidate.is_commit() {
4777 next.previous_commit = current.commit_head;
4778 next.commit_head = Some(new_link);
4779 next.active_checkpoint_artifacts = None;
4780 next.active_checkpoint_artifact_leader_proof = None;
4781 if next
4782 .assignment_handoff_pin
4783 .as_ref()
4784 .is_some_and(|pin| candidate.assignment_fence.as_ref() == Some(&pin.target))
4785 {
4786 next.assignment_handoff_pin = None;
4787 }
4788 }
4789 next.validate()?;
4790 match self
4791 .create_authority_record(Some(&published), &next)
4792 .await?
4793 {
4794 AuthorityCreateOutcome::Created => {
4795 let mut appended = outcomes.to_vec();
4796 appended.push(candidate.clone());
4797 let mut terminal_links = snapshot.terminal_links.to_vec();
4798 terminal_links.push(new_link);
4799 let mut commit_links = snapshot.commit_links.to_vec();
4800 if candidate.is_commit() {
4801 commit_links.push(new_link);
4802 }
4803 self.install_cluster_outcome_audit(
4804 Self::cluster_outcome_audit_key(&next),
4805 next.lease.seq,
4806 ClusterOutcomeAuditSnapshot {
4807 outcomes: Arc::from(appended),
4808 terminal_links: Arc::from(terminal_links),
4809 commit_links: Arc::from(commit_links),
4810 },
4811 );
4812 return Ok(RecordOutcomeResult::Created(candidate));
4813 }
4814 AuthorityCreateOutcome::ExistingIdentical => {
4815 return Ok(RecordOutcomeResult::Unchanged(candidate));
4816 }
4817 AuthorityCreateOutcome::Contended(winner_head) => {
4818 let winners = self
4819 .cached_audited_cluster_outcomes_from(&winner_head)
4820 .await?;
4821 if let Some(winner) = winners
4822 .outcomes
4823 .iter()
4824 .find(|outcome| outcome.epoch == candidate.epoch)
4825 {
4826 return if winner == &candidate {
4827 Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4828 } else {
4829 Ok(RecordOutcomeResult::Conflict {
4830 winner: winner.clone(),
4831 })
4832 };
4833 }
4834 if !winner_head.lease.matches_proof(proof) {
4835 return Err(ClusterCheckpointAuthorityError::Fenced);
4836 }
4837 if winner_head.lease.seq <= base_sequence {
4838 return Err(LeaseError::Invalid(
4839 "cluster outcome contention did not advance the authority sequence"
4840 .into(),
4841 )
4842 .into());
4843 }
4844 tokio::task::yield_now().await;
4845 }
4846 }
4847 }
4848 }
4849
4850 async fn validate_assignment_recovery_snapshot(
4851 &self,
4852 decision: &AssignmentRecoveryDecision,
4853 ) -> Result<(), ClusterCheckpointAuthorityError> {
4854 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4855 let predecessor = snapshots
4856 .load()
4857 .await
4858 .map_err(|error| {
4859 assignment_snapshot_error("load assignment recovery predecessor", error)
4860 })?
4861 .ok_or_else(|| {
4862 LeaseError::Invalid(
4863 "assignment recovery requires a durable predecessor snapshot".into(),
4864 )
4865 })?;
4866 let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4867 LeaseError::Invalid(format!(
4868 "assignment recovery predecessor has no valid fence: {error}"
4869 ))
4870 })?;
4871 if predecessor.draining || predecessor_fence != decision.predecessor {
4872 return Err(LeaseError::Invalid(
4873 "assignment recovery predecessor is not the exact committed assignment head".into(),
4874 )
4875 .into());
4876 }
4877 let proposal = snapshots
4878 .load_recovery_proposal(&decision.proposal)
4879 .await
4880 .map_err(|error| {
4881 assignment_snapshot_error("load assignment recovery proposal", error)
4882 })?;
4883 let proposal_fence = proposal.assignment_fence().map_err(|error| {
4884 LeaseError::Invalid(format!(
4885 "assignment recovery proposal has no valid fence: {error}"
4886 ))
4887 })?;
4888 if proposal_fence != decision.target {
4889 return Err(LeaseError::Invalid(
4890 "assignment recovery proposal does not match its authorized target fence".into(),
4891 )
4892 .into());
4893 }
4894 Ok(())
4895 }
4896
4897 async fn current_assignment_checkpoint(
4898 &self,
4899 current: &LeaderAuthorityRecord,
4900 predecessor: &CheckpointAssignmentFence,
4901 purpose: &str,
4902 ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
4903 let Some(commit_head) = current.commit_head else {
4904 return Ok(None);
4905 };
4906 let outcome = self
4907 .cluster_outcome_from_snapshot(current, commit_head.epoch)
4908 .await?
4909 .ok_or_else(|| {
4910 DecisionError::Conflict(format!(
4911 "{purpose} checkpoint {} is not live",
4912 commit_head.checkpoint_id
4913 ))
4914 })?;
4915 if !outcome.is_commit()
4916 || outcome.checkpoint_id != commit_head.checkpoint_id
4917 || outcome.assignment_fence.as_ref() != Some(predecessor)
4918 {
4919 return Err(DecisionError::Conflict(format!(
4920 "{purpose} checkpoint {} does not bind its predecessor fence",
4921 commit_head.checkpoint_id
4922 ))
4923 .into());
4924 }
4925 let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
4926 DecisionError::Conflict(format!(
4927 "{purpose} checkpoint {} has no committed index reference",
4928 commit_head.checkpoint_id
4929 ))
4930 })?;
4931 let index = CheckpointDecisionStore::new(Arc::clone(&self.store))
4932 .validate_committed_checkpoint_for_outcome(&outcome)
4933 .await?;
4934 if index.assignment_fence.as_ref() != Some(predecessor) {
4935 return Err(DecisionError::Conflict(format!(
4936 "{purpose} checkpoint index {} does not bind its predecessor fence",
4937 commit_head.checkpoint_id
4938 ))
4939 .into());
4940 }
4941 Ok(Some(reference.clone()))
4942 }
4943
4944 async fn validate_current_assignment_checkpoint(
4945 &self,
4946 current: &LeaderAuthorityRecord,
4947 reference: &CommittedCheckpointRef,
4948 predecessor: &CheckpointAssignmentFence,
4949 purpose: &str,
4950 ) -> Result<(), ClusterCheckpointAuthorityError> {
4951 let current_reference = self
4952 .current_assignment_checkpoint(current, predecessor, purpose)
4953 .await?
4954 .ok_or_else(|| {
4955 DecisionError::Conflict(format!("{purpose} has no authoritative checkpoint Commit"))
4956 })?;
4957 if current_reference != *reference {
4958 return Err(DecisionError::Conflict(format!(
4959 "{purpose} checkpoint {} is not the current Commit head {}",
4960 reference.checkpoint_id, current_reference.checkpoint_id
4961 ))
4962 .into());
4963 }
4964 Ok(())
4965 }
4966
4967 fn aborted_handoff_target(
4968 transition: &AssignmentDrainTransition,
4969 ) -> Result<CheckpointAssignmentFence, ClusterCheckpointAuthorityError> {
4970 let mut target = transition.predecessor.clone();
4971 target.assignment_version = transition.target.assignment_version;
4972 if !target.is_canonical() {
4973 return Err(LeaseError::Invalid(
4974 "assignment drain abort produced an invalid handoff target".into(),
4975 )
4976 .into());
4977 }
4978 Ok(target)
4979 }
4980
4981 async fn next_assignment_handoff_pin(
4982 &self,
4983 current: &LeaderAuthorityRecord,
4984 decision: &AuthorityAssignmentDecision,
4985 ) -> Result<Option<AssignmentHandoffPin>, ClusterCheckpointAuthorityError> {
4986 match decision {
4987 AuthorityAssignmentDecision::Drain(drain) => match drain.verdict {
4988 AssignmentDrainVerdict::Commit => {
4989 if current.assignment_handoff_pin.is_some() {
4990 return Err(DecisionError::Conflict(
4991 "assignment decision cannot overtake an unresolved state handoff"
4992 .into(),
4993 )
4994 .into());
4995 }
4996 let reference = drain.handoff_checkpoint.as_ref().ok_or_else(|| {
4997 LeaseError::Invalid(
4998 "committed assignment drain has no handoff checkpoint".into(),
4999 )
5000 })?;
5001 self.validate_current_assignment_checkpoint(
5002 current,
5003 reference,
5004 &drain.transition.predecessor,
5005 "assignment drain handoff",
5006 )
5007 .await?;
5008 Ok(Some(AssignmentHandoffPin {
5009 target: drain.transition.target.clone(),
5010 checkpoint: reference.clone(),
5011 }))
5012 }
5013 AssignmentDrainVerdict::Abort => match current.assignment_handoff_pin.as_ref() {
5014 None => {
5015 let checkpoint = self
5016 .current_assignment_checkpoint(
5017 current,
5018 &drain.transition.predecessor,
5019 "assignment drain abort",
5020 )
5021 .await?;
5022 match checkpoint {
5023 Some(checkpoint) => Ok(Some(AssignmentHandoffPin {
5024 target: Self::aborted_handoff_target(&drain.transition)?,
5025 checkpoint,
5026 })),
5027 None => Ok(None),
5028 }
5029 }
5030 Some(pin) if pin.target == drain.transition.predecessor => {
5031 Ok(Some(AssignmentHandoffPin {
5032 target: Self::aborted_handoff_target(&drain.transition)?,
5033 checkpoint: pin.checkpoint.clone(),
5034 }))
5035 }
5036 Some(_) => Err(DecisionError::Conflict(
5037 "assignment decision cannot overtake an unresolved state handoff".into(),
5038 )
5039 .into()),
5040 },
5041 },
5042 AuthorityAssignmentDecision::Recovery(recovery) => {
5043 if let Some(pin) = current.assignment_handoff_pin.as_ref() {
5044 if pin.target != recovery.predecessor
5045 || pin.checkpoint != recovery.recovery_checkpoint
5046 {
5047 return Err(DecisionError::Conflict(
5048 "assignment recovery does not continue the exact unresolved state handoff"
5049 .into(),
5050 )
5051 .into());
5052 }
5053 } else {
5054 self.validate_current_assignment_checkpoint(
5055 current,
5056 &recovery.recovery_checkpoint,
5057 &recovery.predecessor,
5058 "assignment recovery",
5059 )
5060 .await?;
5061 }
5062 Ok(Some(AssignmentHandoffPin {
5063 target: recovery.target.clone(),
5064 checkpoint: recovery.recovery_checkpoint.clone(),
5065 }))
5066 }
5067 }
5068 }
5069
5070 async fn record_assignment_decision(
5071 &self,
5072 proof: &LeaderProof,
5073 decision: AuthorityAssignmentDecision,
5074 ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
5075 decision.validate()?;
5076 if decision.leader_proof() != proof || !proof.is_canonical() {
5077 return Err(ClusterCheckpointAuthorityError::Fenced);
5078 }
5079
5080 loop {
5081 let published = self
5082 .load_published_authority_head()
5083 .await?
5084 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5085 let current = &published.record;
5086 if !current.lease.matches_proof(proof) {
5087 return Err(ClusterCheckpointAuthorityError::Fenced);
5088 }
5089 if let Some(floor) = current.assignment_decision_floor.as_ref() {
5090 if decision.target_version() < floor.before_target_version {
5091 return Err(DecisionError::Conflict(format!(
5092 "assignment decision version {} is below durable retention floor {}",
5093 decision.target_version(),
5094 floor.before_target_version
5095 ))
5096 .into());
5097 }
5098 }
5099 let decisions = self.audited_assignment_decisions_from(current).await?;
5100 if let Some(winner) = decisions
5101 .iter()
5102 .find(|winner| winner.target_version() == decision.target_version())
5103 {
5104 return if winner == &decision {
5105 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5106 winner.clone(),
5107 ))
5108 } else {
5109 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5110 winner: winner.clone(),
5111 })
5112 };
5113 }
5114 if matches!(&decision, AuthorityAssignmentDecision::Drain(_)) {
5115 let predecessor = decision.predecessor();
5116 if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
5117 let active_fence = active.assignment_fence.as_ref().ok_or_else(|| {
5118 DecisionError::Conflict(
5119 "active checkpoint artifact inventory lost its assignment fence".into(),
5120 )
5121 })?;
5122 if active_fence.assignment_version == predecessor.assignment_version {
5123 let detail = if active_fence == predecessor {
5124 "matches"
5125 } else {
5126 "conflicts with"
5127 };
5128 return Err(DecisionError::Conflict(format!(
5129 "assignment drain decision cannot overtake checkpoint {} whose assignment fence {detail} predecessor version {}",
5130 active.attempt.checkpoint_id,
5131 predecessor.assignment_version
5132 ))
5133 .into());
5134 }
5135 }
5136 }
5137 if let Some(last) = decisions.last() {
5138 if decision.target_version() <= last.target_version() {
5139 return Err(DecisionError::Conflict(format!(
5140 "assignment decision version {} does not advance durable version {}",
5141 decision.target_version(),
5142 last.target_version()
5143 ))
5144 .into());
5145 }
5146 }
5147 if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
5148 self.validate_assignment_recovery_snapshot(recovery).await?;
5149 }
5150 let assignment_handoff_pin =
5151 self.next_assignment_handoff_pin(current, &decision).await?;
5152
5153 let base_sequence = current.lease.seq;
5154 let sequence = base_sequence
5155 .checked_add(1)
5156 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5157 let mut candidate = current.preserve_with_lease(LeaderLease {
5158 seq: sequence,
5159 renewal_sequence: current.lease.renewal_sequence,
5160 token: current.lease.token,
5161 owner: current.lease.owner.clone(),
5162 expires_at_ms: current.lease.expires_at_ms,
5163 catalog_manifest: current.lease.catalog_manifest.clone(),
5164 });
5165 candidate.assignment_decision = Some(decision.clone());
5166 candidate.previous_assignment_decision = current.assignment_decision_head;
5167 candidate.assignment_decision_head = Some(AssignmentDecisionLink {
5168 sequence,
5169 target_version: decision.target_version(),
5170 });
5171 candidate.assignment_handoff_pin = assignment_handoff_pin;
5172 candidate.validate()?;
5173
5174 match self
5175 .create_authority_record(Some(&published), &candidate)
5176 .await?
5177 {
5178 AuthorityCreateOutcome::Created => {
5179 return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
5180 }
5181 AuthorityCreateOutcome::ExistingIdentical => {
5182 return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
5183 }
5184 AuthorityCreateOutcome::Contended(winner_head) => {
5185 let winners = self.audited_assignment_decisions_from(&winner_head).await?;
5186 if let Some(winner) = winners
5187 .iter()
5188 .find(|winner| winner.target_version() == decision.target_version())
5189 {
5190 return if winner == &decision {
5191 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5192 winner.clone(),
5193 ))
5194 } else {
5195 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5196 winner: winner.clone(),
5197 })
5198 };
5199 }
5200 if !winner_head.lease.matches_proof(proof) {
5201 return Err(ClusterCheckpointAuthorityError::Fenced);
5202 }
5203 if winner_head.lease.seq > base_sequence {
5204 tokio::task::yield_now().await;
5205 continue;
5206 }
5207 return Err(LeaseError::Invalid(
5208 "assignment authority contention did not advance the sequence".into(),
5209 )
5210 .into());
5211 }
5212 }
5213 }
5214 }
5215
5216 pub async fn record_assignment_drain_decision(
5224 &self,
5225 proof: &LeaderProof,
5226 decision: AssignmentDrainDecision,
5227 ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5228 match Box::pin(
5229 self.record_assignment_decision(proof, AuthorityAssignmentDecision::Drain(decision)),
5230 )
5231 .await?
5232 {
5233 RecordAuthorityAssignmentDecisionResult::Created(
5234 AuthorityAssignmentDecision::Drain(decision),
5235 ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
5236 RecordAuthorityAssignmentDecisionResult::Unchanged(
5237 AuthorityAssignmentDecision::Drain(decision),
5238 ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
5239 RecordAuthorityAssignmentDecisionResult::Conflict {
5240 winner: AuthorityAssignmentDecision::Drain(winner),
5241 } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
5242 RecordAuthorityAssignmentDecisionResult::Conflict {
5243 winner: AuthorityAssignmentDecision::Recovery(winner),
5244 } => Err(DecisionError::Conflict(format!(
5245 "assignment recovery decision already settled target version {}",
5246 winner.target_version()
5247 ))
5248 .into()),
5249 RecordAuthorityAssignmentDecisionResult::Created(_)
5250 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5251 "assignment decision recorder returned the wrong decision kind".into(),
5252 )
5253 .into()),
5254 }
5255 }
5256
5257 pub(crate) async fn record_assignment_recovery_decision(
5262 &self,
5263 proof: &LeaderProof,
5264 decision: AssignmentRecoveryDecision,
5265 ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
5266 match Box::pin(
5267 self.record_assignment_decision(proof, AuthorityAssignmentDecision::Recovery(decision)),
5268 )
5269 .await?
5270 {
5271 RecordAuthorityAssignmentDecisionResult::Created(
5272 AuthorityAssignmentDecision::Recovery(decision),
5273 ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
5274 RecordAuthorityAssignmentDecisionResult::Unchanged(
5275 AuthorityAssignmentDecision::Recovery(decision),
5276 ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
5277 RecordAuthorityAssignmentDecisionResult::Conflict {
5278 winner: AuthorityAssignmentDecision::Recovery(winner),
5279 } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
5280 RecordAuthorityAssignmentDecisionResult::Conflict {
5281 winner: AuthorityAssignmentDecision::Drain(winner),
5282 } => Err(DecisionError::Conflict(format!(
5283 "assignment drain decision already settled target version {}",
5284 winner.target_version()
5285 ))
5286 .into()),
5287 RecordAuthorityAssignmentDecisionResult::Created(_)
5288 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5289 "assignment decision recorder returned the wrong decision kind".into(),
5290 )
5291 .into()),
5292 }
5293 }
5294
5295 async fn assignment_decision(
5296 &self,
5297 target_version: u64,
5298 ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
5299 if target_version == 0 {
5300 return Err(LeaseError::Invalid(
5301 "assignment decision target version must be nonzero".into(),
5302 )
5303 .into());
5304 }
5305 let Some(head) = self.load_record().await? else {
5306 return Ok(None);
5307 };
5308 if let Some(floor) = head.assignment_decision_floor.as_ref() {
5309 if target_version < floor.before_target_version {
5310 return Err(DecisionError::Conflict(format!(
5311 "assignment decision version {target_version} is below durable retention floor {}",
5312 floor.before_target_version
5313 ))
5314 .into());
5315 }
5316 }
5317 Ok(self
5318 .audited_assignment_decisions_from(&head)
5319 .await?
5320 .into_iter()
5321 .find(|decision| decision.target_version() == target_version))
5322 }
5323
5324 pub async fn assignment_drain_decision(
5329 &self,
5330 target_version: u64,
5331 ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
5332 match self.assignment_decision(target_version).await? {
5333 Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
5334 Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
5335 format!("target version {target_version} was settled by assignment recovery"),
5336 )
5337 .into()),
5338 None => Ok(None),
5339 }
5340 }
5341
5342 pub async fn assignment_recovery_decision(
5347 &self,
5348 target_version: u64,
5349 ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
5350 match self.assignment_decision(target_version).await? {
5351 Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
5352 Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
5353 "target version {target_version} was settled by assignment drain"
5354 ))
5355 .into()),
5356 None => Ok(None),
5357 }
5358 }
5359
5360 pub async fn assignment_handoff_checkpoint(
5365 &self,
5366 target: &CheckpointAssignmentFence,
5367 ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
5368 if !target.is_canonical() {
5369 return Err(LeaseError::Invalid(
5370 "assignment handoff checkpoint lookup requires a canonical target fence".into(),
5371 )
5372 .into());
5373 }
5374 Ok(self.load_record().await?.and_then(|head| {
5375 head.assignment_handoff_pin
5376 .filter(|pin| pin.target == *target)
5377 .map(|pin| pin.checkpoint)
5378 }))
5379 }
5380
5381 pub async fn materialize_assignment_recovery(
5389 &self,
5390 target_version: u64,
5391 ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
5392 let decision = self
5393 .assignment_recovery_decision(target_version)
5394 .await?
5395 .ok_or_else(|| {
5396 DecisionError::Conflict(format!(
5397 "assignment recovery target {target_version} has no authority decision"
5398 ))
5399 })?;
5400 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5401 let proposal = snapshots
5402 .load_recovery_proposal(&decision.proposal)
5403 .await
5404 .map_err(|error| {
5405 assignment_snapshot_error("load authorized assignment recovery proposal", error)
5406 })?;
5407 if proposal.assignment_fence().map_err(|error| {
5408 LeaseError::Invalid(format!(
5409 "authorized assignment recovery proposal has no valid fence: {error}"
5410 ))
5411 })? != decision.target
5412 {
5413 return Err(LeaseError::Invalid(
5414 "authorized assignment recovery proposal does not match its target".into(),
5415 )
5416 .into());
5417 }
5418 snapshots
5419 .materialize_recovery(&decision.proposal)
5420 .await
5421 .map_err(|error| {
5422 assignment_snapshot_error("materialize authorized assignment recovery", error)
5423 .into()
5424 })
5425 }
5426
5427 pub async fn prune_assignment_drain_decisions_before(
5439 &self,
5440 proof: &LeaderProof,
5441 before_target_version: u64,
5442 ) -> Result<u64, ClusterCheckpointAuthorityError> {
5443 if before_target_version == 0 {
5444 return Ok(self
5445 .load_record()
5446 .await?
5447 .and_then(|head| head.assignment_decision_floor)
5448 .map_or(0, |floor| floor.before_target_version));
5449 }
5450 if !proof.is_canonical() {
5451 return Err(ClusterCheckpointAuthorityError::Fenced);
5452 }
5453 loop {
5454 let published = self
5455 .load_published_authority_head()
5456 .await?
5457 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5458 let current = &published.record;
5459 if !current.lease.matches_proof(proof) {
5460 return Err(ClusterCheckpointAuthorityError::Fenced);
5461 }
5462 if let Some(floor) = current.assignment_decision_floor.as_ref() {
5463 if floor.before_target_version >= before_target_version {
5464 self.schedule_history_prune();
5465 return Ok(floor.before_target_version);
5466 }
5467 }
5468
5469 let decisions = self.audited_assignment_decisions_from(current).await?;
5470 let terminal_anchor = decisions
5471 .iter()
5472 .rev()
5473 .find(|decision| decision.target_version() < before_target_version)
5474 .cloned()
5475 .or_else(|| {
5476 current
5477 .assignment_decision_floor
5478 .as_ref()
5479 .and_then(|floor| floor.terminal_anchor.clone())
5480 });
5481 let terminal_anchor_link = match terminal_anchor.as_ref() {
5482 Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
5483 None => None,
5484 };
5485 let floor = AuthorityAssignmentDecisionFloor {
5486 before_target_version,
5487 terminal_anchor,
5488 terminal_anchor_link,
5489 };
5490 floor.validate()?;
5491
5492 let base_sequence = current.lease.seq;
5493 let sequence = base_sequence
5494 .checked_add(1)
5495 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5496 let mut next = current.preserve_with_lease(LeaderLease {
5497 seq: sequence,
5498 renewal_sequence: current.lease.renewal_sequence,
5499 token: current.lease.token,
5500 owner: current.lease.owner.clone(),
5501 expires_at_ms: current.lease.expires_at_ms,
5502 catalog_manifest: current.lease.catalog_manifest.clone(),
5503 });
5504 next.assignment_decision_floor = Some(floor);
5505 next.validate()?;
5506
5507 match self
5508 .create_authority_record(Some(&published), &next)
5509 .await?
5510 {
5511 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5512 return Ok(before_target_version);
5513 }
5514 AuthorityCreateOutcome::Contended(winner) => {
5515 if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
5516 if winner_floor.before_target_version >= before_target_version {
5517 self.schedule_history_prune();
5518 return Ok(winner_floor.before_target_version);
5519 }
5520 }
5521 if !winner.lease.matches_proof(proof) {
5522 return Err(ClusterCheckpointAuthorityError::Fenced);
5523 }
5524 if winner.lease.seq > base_sequence {
5525 tokio::task::yield_now().await;
5526 continue;
5527 }
5528 return Err(LeaseError::Invalid(
5529 "assignment decision floor contention did not advance the sequence".into(),
5530 )
5531 .into());
5532 }
5533 }
5534 }
5535 }
5536
5537 async fn cluster_outcome_from_snapshot(
5538 &self,
5539 head: &LeaderAuthorityRecord,
5540 epoch: u64,
5541 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5542 let floor = head.outcome_floor.as_ref();
5543 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5544 if epoch < artifact_before_epoch {
5545 return Ok(None);
5546 }
5547 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
5548 let cache_key = Self::cluster_outcome_audit_key(head);
5549 if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
5550 return Ok(snapshot
5551 .outcomes
5552 .binary_search_by_key(&epoch, |outcome| outcome.epoch)
5553 .ok()
5554 .map(|index| snapshot.outcomes[index].clone())
5555 .filter(|outcome| {
5556 outcome.epoch >= authority_before_epoch
5557 || outcome.is_commit()
5558 || floor
5559 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
5560 }));
5561 }
5562 if let Some(anchor) = floor
5563 .and_then(|floor| floor.terminal_anchor.as_ref())
5564 .filter(|anchor| anchor.epoch == epoch)
5565 {
5566 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5567 .load_or_create_deployment_id()
5568 .await?;
5569 if anchor.deployment_id != expected_deployment {
5570 return Err(DecisionError::Conflict(format!(
5571 "cluster terminal anchor epoch {} does not belong to current deployment {}",
5572 anchor.epoch, expected_deployment
5573 ))
5574 .into());
5575 }
5576 return Ok(Some(anchor.clone()));
5577 }
5578
5579 let commit_only = epoch < authority_before_epoch;
5580 let before_epoch = if commit_only {
5581 artifact_before_epoch
5582 } else {
5583 authority_before_epoch
5584 };
5585 let mut current = if commit_only {
5586 head.commit_head
5587 } else {
5588 head.outcome_head
5589 };
5590 let expected_anchor = if commit_only {
5591 floor.and_then(|floor| floor.committed_anchor_link)
5592 } else {
5593 floor.and_then(|floor| floor.terminal_anchor_link)
5594 };
5595 let expected_anchor_outcome = if commit_only {
5596 floor.and_then(|floor| floor.committed_anchor.as_ref())
5597 } else {
5598 floor.and_then(|floor| floor.terminal_anchor.as_ref())
5599 };
5600 let mut traversed = 0;
5601 while let Some(link) = current {
5602 if link.epoch < before_epoch {
5603 match (expected_anchor, expected_anchor_outcome) {
5604 (Some(anchor_link), Some(anchor))
5605 if link == anchor_link
5606 && link.epoch == anchor.epoch
5607 && link.checkpoint_id == anchor.checkpoint_id =>
5608 {
5609 return Ok(None)
5610 }
5611 (None, None) => return Ok(None),
5612 _ => {
5613 return Err(DecisionError::Conflict(format!(
5614 "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
5615 if commit_only { "Commit" } else { "terminal outcome" }
5616 ))
5617 .into());
5618 }
5619 }
5620 }
5621 if link.epoch < epoch {
5622 return Ok(None);
5623 }
5624 if !consume_live_authority_link(&mut traversed) {
5625 return Err(DecisionError::Conflict(format!(
5626 "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
5627 if commit_only { "Commit" } else { "outcome" }
5628 ))
5629 .into());
5630 }
5631
5632 let admission;
5633 let record = if link.sequence == head.lease.seq {
5634 head
5635 } else {
5636 admission = read_authority_record(self.store.as_ref(), link.sequence)
5637 .await?
5638 .ok_or_else(|| {
5639 DecisionError::InventoryChanged(format!(
5640 "cluster {} authority record {} disappeared during exact lookup",
5641 if commit_only { "Commit" } else { "outcome" },
5642 link.sequence
5643 ))
5644 })?;
5645 &admission
5646 };
5647 let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
5648 DecisionError::Conflict(format!(
5649 "cluster {} head epoch {} points to non-outcome authority record {}",
5650 if commit_only { "Commit" } else { "outcome" },
5651 link.epoch,
5652 link.sequence
5653 ))
5654 })?;
5655 let record_head = if commit_only {
5656 record.commit_head
5657 } else {
5658 record.outcome_head
5659 };
5660 if record_head != Some(link)
5661 || (commit_only && !outcome.is_commit())
5662 || outcome.epoch != link.epoch
5663 || outcome.checkpoint_id != link.checkpoint_id
5664 {
5665 return Err(DecisionError::Conflict(format!(
5666 "cluster {} link epoch {} sequence {} does not match its authority record",
5667 if commit_only { "Commit" } else { "outcome" },
5668 link.epoch,
5669 link.sequence
5670 ))
5671 .into());
5672 }
5673 if outcome.epoch == epoch {
5674 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5675 .load_or_create_deployment_id()
5676 .await?;
5677 if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
5678 || outcome.deployment_id != expected_deployment
5679 {
5680 return Err(DecisionError::Conflict(format!(
5681 "cluster outcome epoch {} does not belong to current deployment {}",
5682 outcome.epoch, expected_deployment
5683 ))
5684 .into());
5685 }
5686 return Ok(Some(outcome.clone()));
5687 }
5688 current = if commit_only {
5689 record.previous_commit
5690 } else {
5691 record.previous_outcome
5692 };
5693 }
5694 if expected_anchor.is_some() {
5695 return Err(DecisionError::Conflict(format!(
5696 "cluster {} chain stopped without its durable floor anchor",
5697 if commit_only { "Commit" } else { "outcome" }
5698 ))
5699 .into());
5700 }
5701 Ok(None)
5702 }
5703
5704 async fn stable_cluster_outcome(
5705 &self,
5706 epoch: u64,
5707 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5708 const LOOKUP_RETRIES: usize = 3;
5709 for attempt in 0..LOOKUP_RETRIES {
5710 let Some(head) = self.load_record().await? else {
5711 return Ok(None);
5712 };
5713 match self.cluster_outcome_from_snapshot(&head, epoch).await {
5714 Err(ClusterCheckpointAuthorityError::Decision(
5715 DecisionError::InventoryChanged(_),
5716 )) if attempt + 1 < LOOKUP_RETRIES => {
5717 tokio::task::yield_now().await;
5718 }
5719 result => return result,
5720 }
5721 }
5722 Err(DecisionError::InventoryChanged(
5723 "cluster outcome exact lookup exhausted stability retries".into(),
5724 )
5725 .into())
5726 }
5727
5728 pub async fn cluster_outcome(
5734 &self,
5735 epoch: u64,
5736 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5737 self.stable_cluster_outcome(epoch).await
5738 }
5739
5740 pub async fn create_committed_checkpoint(
5746 &self,
5747 index: &CommittedCheckpointIndex,
5748 ) -> Result<CommittedCheckpointRef, ClusterCheckpointAuthorityError> {
5749 CheckpointDecisionStore::new(Arc::clone(&self.store))
5750 .create_committed_checkpoint(index)
5751 .await
5752 .map_err(Into::into)
5753 }
5754
5755 pub async fn load_committed_checkpoint(
5761 &self,
5762 reference: &CommittedCheckpointRef,
5763 ) -> Result<CommittedCheckpointIndex, ClusterCheckpointAuthorityError> {
5764 CheckpointDecisionStore::new(Arc::clone(&self.store))
5765 .load_committed_checkpoint(reference)
5766 .await
5767 .map_err(Into::into)
5768 }
5769
5770 pub async fn cluster_outcome_with_committed_checkpoint(
5777 &self,
5778 epoch: u64,
5779 ) -> Result<
5780 Option<(CheckpointOutcome, Option<CommittedCheckpointIndex>)>,
5781 ClusterCheckpointAuthorityError,
5782 > {
5783 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5784 let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5785 return Ok(None);
5786 };
5787 let index = if outcome.is_commit() {
5788 let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
5789 DecisionError::Conflict(format!(
5790 "cluster Commit epoch {} checkpoint {} has no committed checkpoint index",
5791 outcome.epoch, outcome.checkpoint_id
5792 ))
5793 })?;
5794 let index = decisions.load_committed_checkpoint(reference).await?;
5795 if index.epoch != outcome.epoch
5796 || index.checkpoint_id != outcome.checkpoint_id
5797 || index.scope != outcome.scope
5798 || index.assignment_fence.as_ref() != outcome.assignment_fence.as_ref()
5799 || index.deployment_id != outcome.deployment_id
5800 {
5801 return Err(DecisionError::Conflict(format!(
5802 "cluster Commit epoch {} checkpoint {} does not match committed checkpoint '{}'",
5803 outcome.epoch, outcome.checkpoint_id, reference.sha256
5804 ))
5805 .into());
5806 }
5807 Some(index)
5808 } else {
5809 None
5810 };
5811 Ok(Some((outcome, index)))
5812 }
5813
5814 pub async fn cluster_outcome_inventory(
5820 &self,
5821 ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5822 let (head, outcomes) = self.audited_cluster_outcomes().await?;
5823 Ok(Self::cluster_outcome_inventory_from_audit(
5824 head.as_ref(),
5825 &outcomes,
5826 ))
5827 }
5828
5829 fn cluster_outcome_inventory_from_audit(
5830 head: Option<&LeaderAuthorityRecord>,
5831 outcomes: &[CheckpointOutcome],
5832 ) -> ClusterOutcomeInventory {
5833 let floor = head.and_then(|head| head.outcome_floor.as_ref());
5834 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5835 ClusterOutcomeInventory {
5836 outcomes: outcomes
5837 .iter()
5838 .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5839 .cloned()
5840 .collect(),
5841 retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5842 }
5843 }
5844
5845 pub async fn cluster_outcomes(
5851 &self,
5852 ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5853 Ok(self.cluster_outcome_inventory().await?.outcomes)
5854 }
5855
5856 pub async fn highest_cluster_committed_outcome(
5862 &self,
5863 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5864 Ok(self
5865 .cluster_outcomes()
5866 .await?
5867 .into_iter()
5868 .rev()
5869 .find(CheckpointOutcome::is_commit))
5870 }
5871
5872 pub async fn highest_cluster_terminal_outcome(
5878 &self,
5879 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5880 Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5881 }
5882
5883 fn cleanup_participant_ids(
5884 index: &CommittedCheckpointIndex,
5885 ) -> Result<Vec<u64>, ClusterCheckpointAuthorityError> {
5886 index.validate().map_err(DecisionError::Conflict)?;
5887 let participant_ids = index
5888 .participants
5889 .iter()
5890 .map(|participant| participant.participant_id)
5891 .collect::<Vec<_>>();
5892 if participant_ids.is_empty()
5893 || participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
5894 || participant_ids[0] == 0
5895 || !participant_ids.windows(2).all(|pair| pair[0] < pair[1])
5896 {
5897 return Err(DecisionError::Conflict(
5898 "committed checkpoint participants are not canonical".into(),
5899 )
5900 .into());
5901 }
5902 Ok(participant_ids)
5903 }
5904
5905 fn cleanup_stop_before(head: &LeaderAuthorityRecord) -> Option<CommittedCheckpointRef> {
5906 head.outcome_floor
5907 .as_ref()
5908 .filter(|floor| floor.artifact_before_epoch != 0)
5909 .and_then(|floor| floor.committed_anchor.as_ref())
5910 .and_then(|outcome| outcome.committed_checkpoint.clone())
5911 }
5912
5913 pub async fn cluster_artifact_cleanup(
5921 &self,
5922 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
5923 Ok(self
5924 .load_record()
5925 .await?
5926 .and_then(|head| head.artifact_cleanup))
5927 }
5928
5929 pub async fn begin_cluster_artifact_cleanup<V, Fut>(
5939 &self,
5940 proof: &LeaderProof,
5941 protected: CommittedCheckpointRef,
5942 validate_artifacts: V,
5943 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError>
5944 where
5945 V: Fn(CheckpointOutcome) -> Fut + Send + Sync + 'static,
5946 Fut: std::future::Future<Output = Result<(), String>> + Send + 'static,
5947 {
5948 if !proof.is_canonical() {
5949 return Err(ClusterCheckpointAuthorityError::Fenced);
5950 }
5951 protected.validate().map_err(DecisionError::Conflict)?;
5952 loop {
5953 let published = self
5954 .load_published_authority_head()
5955 .await?
5956 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5957 let current = &published.record;
5958 if !current.lease.matches_proof(proof) {
5959 return Err(ClusterCheckpointAuthorityError::Fenced);
5960 }
5961 if let Some(active) = current.artifact_cleanup.as_ref() {
5962 if active.protected == protected {
5963 return Ok(Some(active.clone()));
5964 }
5965 return Err(DecisionError::Conflict(format!(
5966 "cluster artifact cleanup for protected checkpoint {} is still active",
5967 active.protected.checkpoint_id
5968 ))
5969 .into());
5970 }
5971 if current.assignment_handoff_pin.as_ref().is_some_and(|pin| {
5972 protected.epoch > pin.checkpoint.epoch
5973 && protected.checkpoint_id > pin.checkpoint.checkpoint_id
5974 }) {
5975 return Ok(None);
5976 }
5977 if current
5978 .outcome_floor
5979 .as_ref()
5980 .is_some_and(|floor| floor.artifact_before_epoch >= protected.epoch)
5981 {
5982 return Ok(None);
5983 }
5984
5985 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
5986 let protected_outcome = snapshot
5987 .outcomes
5988 .iter()
5989 .find(|outcome| {
5990 outcome.is_commit() && outcome.committed_checkpoint.as_ref() == Some(&protected)
5991 })
5992 .cloned()
5993 .ok_or_else(|| {
5994 DecisionError::Conflict(format!(
5995 "protected checkpoint {} is not a live cluster Commit",
5996 protected.checkpoint_id
5997 ))
5998 })?;
5999 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6000 let protected_index = decisions
6001 .validate_committed_checkpoint_for_outcome(&protected_outcome)
6002 .await?;
6003 validate_artifacts(protected_outcome.clone())
6004 .await
6005 .map_err(|error| {
6006 DecisionError::Conflict(format!(
6007 "protected cluster checkpoint {} failed durable artifact preflight: {error}",
6008 protected.checkpoint_id
6009 ))
6010 })?;
6011 let Some(expired) = protected_index.predecessor.clone() else {
6012 return Ok(None);
6013 };
6014 let expired_index = decisions.load_committed_checkpoint(&expired).await?;
6015 protected_index
6016 .validate_predecessor_index(&expired_index)
6017 .map_err(DecisionError::Conflict)?;
6018
6019 let authority_before_epoch = current
6020 .outcome_floor
6021 .as_ref()
6022 .map_or(protected.epoch, |floor| {
6023 floor.authority_before_epoch.max(protected.epoch)
6024 });
6025 let floor = self
6026 .build_cluster_outcome_floor(
6027 current,
6028 &snapshot,
6029 protected.epoch,
6030 authority_before_epoch,
6031 )
6032 .await?;
6033 if floor
6034 .committed_anchor
6035 .as_ref()
6036 .and_then(|outcome| outcome.committed_checkpoint.as_ref())
6037 != Some(&expired)
6038 {
6039 return Err(DecisionError::Conflict(
6040 "protected checkpoint predecessor is not the artifact-floor Commit anchor"
6041 .into(),
6042 )
6043 .into());
6044 }
6045 let cursor = ClusterArtifactCleanupCursor {
6046 protected: protected.clone(),
6047 current: expired,
6048 next: expired_index.predecessor.clone(),
6049 stop_before: Self::cleanup_stop_before(current),
6050 participant_ids: Self::cleanup_participant_ids(&expired_index)?,
6051 phase: ClusterArtifactCleanupPhase::DeleteData,
6052 };
6053 cursor.validate()?;
6054
6055 let rechecked = self
6058 .load_published_authority_head()
6059 .await?
6060 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6061 if !rechecked.record.lease.matches_proof(proof) {
6062 return Err(ClusterCheckpointAuthorityError::Fenced);
6063 }
6064 if rechecked.record.outcome_head != current.outcome_head
6065 || rechecked.record.commit_head != current.commit_head
6066 || rechecked.record.outcome_floor != current.outcome_floor
6067 || rechecked.record.artifact_cleanup != current.artifact_cleanup
6068 {
6069 tokio::task::yield_now().await;
6070 continue;
6071 }
6072 let sequence =
6073 rechecked.record.lease.seq.checked_add(1).ok_or_else(|| {
6074 LeaseError::Invalid("leader authority sequence exhausted".into())
6075 })?;
6076 let mut lease = rechecked.record.lease.clone();
6077 lease.seq = sequence;
6078 let mut next = rechecked.record.preserve_with_lease(lease);
6079 next.outcome_floor = Some(floor.clone());
6080 next.artifact_cleanup = Some(cursor.clone());
6081 next.validate()?;
6082 match self
6083 .create_authority_record(Some(&rechecked), &next)
6084 .await?
6085 {
6086 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6087 self.install_cluster_outcome_audit(
6088 Self::cluster_outcome_audit_key(&next),
6089 next.lease.seq,
6090 Self::outcomes_retained_by_floor(&floor, &snapshot),
6091 );
6092 return Ok(Some(cursor));
6093 }
6094 AuthorityCreateOutcome::Contended(winner) => {
6095 if !winner.lease.matches_proof(proof) {
6096 return Err(ClusterCheckpointAuthorityError::Fenced);
6097 }
6098 tokio::task::yield_now().await;
6099 }
6100 }
6101 }
6102 }
6103
6104 pub async fn mark_cluster_artifact_data_deleted(
6109 &self,
6110 proof: &LeaderProof,
6111 expected: &ClusterArtifactCleanupCursor,
6112 ) -> Result<ClusterArtifactCleanupCursor, ClusterCheckpointAuthorityError> {
6113 if !proof.is_canonical() {
6114 return Err(ClusterCheckpointAuthorityError::Fenced);
6115 }
6116 expected.validate()?;
6117 if expected.phase != ClusterArtifactCleanupPhase::DeleteData {
6118 return Err(DecisionError::Conflict(
6119 "cluster artifact data transition requires a DeleteData cursor".into(),
6120 )
6121 .into());
6122 }
6123 let mut replacement = expected.clone();
6124 replacement.phase = ClusterArtifactCleanupPhase::DeleteMetadata;
6125 loop {
6126 let published = self
6127 .load_published_authority_head()
6128 .await?
6129 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6130 let current = &published.record;
6131 if !current.lease.matches_proof(proof) {
6132 return Err(ClusterCheckpointAuthorityError::Fenced);
6133 }
6134 if current.artifact_cleanup.as_ref() == Some(&replacement) {
6135 return Ok(replacement);
6136 }
6137 if current.artifact_cleanup.as_ref() != Some(expected) {
6138 return Err(DecisionError::Conflict(
6139 "cluster artifact cleanup cursor changed before data completion".into(),
6140 )
6141 .into());
6142 }
6143 let sequence =
6144 current.lease.seq.checked_add(1).ok_or_else(|| {
6145 LeaseError::Invalid("leader authority sequence exhausted".into())
6146 })?;
6147 let mut lease = current.lease.clone();
6148 lease.seq = sequence;
6149 let mut next = current.preserve_with_lease(lease);
6150 next.artifact_cleanup = Some(replacement.clone());
6151 next.validate()?;
6152 match self
6153 .create_authority_record(Some(&published), &next)
6154 .await?
6155 {
6156 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6157 return Ok(replacement);
6158 }
6159 AuthorityCreateOutcome::Contended(winner) => {
6160 if !winner.lease.matches_proof(proof) {
6161 return Err(ClusterCheckpointAuthorityError::Fenced);
6162 }
6163 tokio::task::yield_now().await;
6164 }
6165 }
6166 }
6167 }
6168
6169 pub async fn mark_cluster_artifact_metadata_deleted(
6179 &self,
6180 proof: &LeaderProof,
6181 expected: &ClusterArtifactCleanupCursor,
6182 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
6183 if !proof.is_canonical() {
6184 return Err(ClusterCheckpointAuthorityError::Fenced);
6185 }
6186 expected.validate()?;
6187 if expected.phase != ClusterArtifactCleanupPhase::DeleteMetadata {
6188 return Err(DecisionError::Conflict(
6189 "cluster artifact metadata transition requires a DeleteMetadata cursor".into(),
6190 )
6191 .into());
6192 }
6193 loop {
6194 let published = self
6195 .load_published_authority_head()
6196 .await?
6197 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6198 let current = &published.record;
6199 if !current.lease.matches_proof(proof) {
6200 return Err(ClusterCheckpointAuthorityError::Fenced);
6201 }
6202 if current.artifact_cleanup.as_ref() != Some(expected) {
6203 let completed = expected.next.is_none()
6204 || expected.next.as_ref() == expected.stop_before.as_ref();
6205 if completed && current.artifact_cleanup.is_none() {
6206 return Ok(None);
6207 }
6208 if let (Some(next_reference), Some(actual)) =
6209 (expected.next.as_ref(), current.artifact_cleanup.as_ref())
6210 {
6211 if actual.phase == ClusterArtifactCleanupPhase::DeleteData
6212 && actual.protected == expected.protected
6213 && &actual.current == next_reference
6214 && actual.stop_before == expected.stop_before
6215 {
6216 return Ok(Some(actual.clone()));
6217 }
6218 }
6219 return Err(DecisionError::Conflict(
6220 "cluster artifact cleanup cursor changed before metadata completion".into(),
6221 )
6222 .into());
6223 }
6224
6225 let replacement = match expected.next.as_ref() {
6226 None => None,
6227 Some(next_reference) if Some(next_reference) == expected.stop_before.as_ref() => {
6228 None
6229 }
6230 Some(next_reference) => {
6231 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6232 let index = decisions.load_committed_checkpoint(next_reference).await?;
6233 if index.scope != CheckpointScope::Cluster {
6234 return Err(DecisionError::Conflict(
6235 "cluster artifact cleanup next checkpoint is not cluster-scoped".into(),
6236 )
6237 .into());
6238 }
6239 let replacement = ClusterArtifactCleanupCursor {
6240 protected: expected.protected.clone(),
6241 current: next_reference.clone(),
6242 next: index.predecessor.clone(),
6243 stop_before: expected.stop_before.clone(),
6244 participant_ids: Self::cleanup_participant_ids(&index)?,
6245 phase: ClusterArtifactCleanupPhase::DeleteData,
6246 };
6247 replacement.validate()?;
6248 Some(replacement)
6249 }
6250 };
6251
6252 let sequence =
6253 current.lease.seq.checked_add(1).ok_or_else(|| {
6254 LeaseError::Invalid("leader authority sequence exhausted".into())
6255 })?;
6256 let mut lease = current.lease.clone();
6257 lease.seq = sequence;
6258 let mut next = current.preserve_with_lease(lease);
6259 next.artifact_cleanup = replacement.clone();
6260 next.validate()?;
6261 match self
6262 .create_authority_record(Some(&published), &next)
6263 .await?
6264 {
6265 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6266 return Ok(replacement);
6267 }
6268 AuthorityCreateOutcome::Contended(winner) => {
6269 if !winner.lease.matches_proof(proof) {
6270 return Err(ClusterCheckpointAuthorityError::Fenced);
6271 }
6272 tokio::task::yield_now().await;
6273 }
6274 }
6275 }
6276 }
6277
6278 pub async fn cluster_outcome_retention_boundary(
6284 &self,
6285 ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
6286 let head = self.load_record().await?;
6287 Ok(ClusterOutcomeRetentionBoundary::from_floor(
6288 head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
6289 ))
6290 }
6291
6292 pub async fn begin_new_term(
6299 &self,
6300 owner: &LeaderLeaseOwner,
6301 now_ms: i64,
6302 ) -> Result<LeaseOutcome, LeaseError> {
6303 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6304 owner,
6305 now_ms,
6306 SameOwnerToken::Rotate,
6307 ))
6308 .await
6309 }
6310
6311 pub async fn renew_exact(
6317 &self,
6318 owner: &LeaderLeaseOwner,
6319 token: u64,
6320 now_ms: i64,
6321 ) -> Result<LeaseOutcome, LeaseError> {
6322 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6323 owner,
6324 now_ms,
6325 SameOwnerToken::Exact(token),
6326 ))
6327 .await
6328 }
6329
6330 #[cfg(test)]
6331 pub(crate) async fn acquire_or_renew_current_term_for_test(
6332 &self,
6333 owner: &LeaderLeaseOwner,
6334 now_ms: i64,
6335 ) -> Result<LeaseOutcome, LeaseError> {
6336 self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
6337 .await
6338 }
6339
6340 async fn acquire_or_renew_current_term_for_test_inner(
6341 &self,
6342 owner: &LeaderLeaseOwner,
6343 now_ms: i64,
6344 same_owner_token: SameOwnerToken,
6345 ) -> Result<LeaseOutcome, LeaseError> {
6346 owner.validate()?;
6347 self.ttl()?;
6348 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6349 loop {
6350 let published = self.load_published_authority_head().await?;
6351 let candidate = match published.as_ref() {
6352 None => {
6353 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6354 return Err(LeaseError::Fenced(
6355 "exact leader renewal lost the durable authority head".into(),
6356 ));
6357 }
6358 LeaderAuthorityRecord::initial(LeaderLease {
6359 seq: 1,
6360 renewal_sequence: 1,
6361 token: 1,
6362 owner: owner.clone(),
6363 expires_at_ms,
6364 catalog_manifest: None,
6365 })
6366 }
6367 Some(head) if head.record.lease.owner == *owner => {
6368 let record = &head.record;
6369 let token = match same_owner_token {
6370 #[cfg(test)]
6371 SameOwnerToken::Preserve => record.lease.token,
6372 SameOwnerToken::Rotate => {
6373 record.lease.token.checked_add(1).ok_or_else(|| {
6374 LeaseError::Invalid("leader fencing token exhausted".into())
6375 })?
6376 }
6377 SameOwnerToken::Exact(expected) if expected == record.lease.token => {
6378 expected
6379 }
6380 SameOwnerToken::Exact(_) => {
6381 return Err(LeaseError::Fenced(
6382 "exact leader renewal was superseded by a newer local term".into(),
6383 ));
6384 }
6385 };
6386 let lease = LeaderLease {
6387 seq: record.lease.seq.checked_add(1).ok_or_else(|| {
6388 LeaseError::Invalid("lease sequence exhausted".into())
6389 })?,
6390 renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
6391 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6392 )?,
6393 token,
6394 owner: owner.clone(),
6395 expires_at_ms,
6396 catalog_manifest: record.lease.catalog_manifest.clone(),
6397 };
6398 record.preserve_with_lease(lease)
6399 }
6400 Some(head) => {
6401 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6402 return Err(LeaseError::Fenced(
6403 "exact leader renewal was superseded by a rival owner".into(),
6404 ));
6405 }
6406 return Ok(LeaseOutcome::Held(head.record.lease.clone()));
6407 }
6408 };
6409 match self
6410 .create_authority_record(published.as_ref(), &candidate)
6411 .await?
6412 {
6413 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6414 return Ok(LeaseOutcome::Acquired(candidate.lease));
6415 }
6416 AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
6417 if matches!(
6418 same_owner_token,
6419 SameOwnerToken::Exact(expected) if winner.lease.token != expected
6420 ) {
6421 return Err(LeaseError::Fenced(
6422 "exact leader renewal lost a same-owner CAS race".into(),
6423 ));
6424 }
6425 tokio::task::yield_now().await;
6426 }
6427 AuthorityCreateOutcome::Contended(winner) => {
6428 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6429 return Err(LeaseError::Fenced(
6430 "exact leader renewal lost a rival-owner CAS race".into(),
6431 ));
6432 }
6433 return Ok(LeaseOutcome::Held(winner.lease));
6434 }
6435 }
6436 }
6437 }
6438
6439 pub fn observe_rival(
6444 &self,
6445 owner: &LeaderLeaseOwner,
6446 lease: &LeaderLease,
6447 ) -> Result<LeaderLeaseObservation, LeaseError> {
6448 owner.validate()?;
6449 lease.validate()?;
6450 self.ttl()?;
6451 if lease.owner == *owner {
6452 return Err(LeaseError::Invalid(
6453 "leader takeover observation must belong to a rival".into(),
6454 ));
6455 }
6456 Ok(LeaderLeaseObservation {
6457 lease: lease.clone(),
6458 started: Instant::now(),
6459 })
6460 }
6461
6462 pub async fn try_takeover(
6468 &self,
6469 owner: &LeaderLeaseOwner,
6470 observation: &LeaderLeaseObservation,
6471 now_ms: i64,
6472 ) -> Result<LeaseOutcome, LeaseError> {
6473 owner.validate()?;
6474 observation.lease.validate()?;
6475 let ttl = self.ttl()?;
6476 if observation.lease.owner == *owner {
6477 return Err(LeaseError::Invalid(
6478 "leader takeover observation must belong to a rival".into(),
6479 ));
6480 }
6481 if observation.started.elapsed() < ttl {
6482 return Ok(LeaseOutcome::Held(observation.lease.clone()));
6483 }
6484 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6485 loop {
6486 let published = self
6487 .load_published_authority_head()
6488 .await?
6489 .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
6490 let current = &published.record;
6491 if !current.lease.has_same_liveness_identity(&observation.lease) {
6492 return Ok(LeaseOutcome::Held(current.lease.clone()));
6493 }
6494 let candidate_lease = LeaderLease {
6495 seq: current
6496 .lease
6497 .seq
6498 .checked_add(1)
6499 .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
6500 renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
6501 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6502 )?,
6503 token: current
6504 .lease
6505 .token
6506 .checked_add(1)
6507 .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
6508 owner: owner.clone(),
6509 expires_at_ms,
6510 catalog_manifest: current.lease.catalog_manifest.clone(),
6511 };
6512 let candidate = current.preserve_with_lease(candidate_lease);
6513 match self
6514 .create_authority_record(Some(&published), &candidate)
6515 .await?
6516 {
6517 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6518 return Ok(LeaseOutcome::Acquired(candidate.lease));
6519 }
6520 AuthorityCreateOutcome::Contended(winner)
6521 if winner.lease.has_same_liveness_identity(&observation.lease) =>
6522 {
6523 tokio::task::yield_now().await;
6524 }
6525 AuthorityCreateOutcome::Contended(winner) => {
6526 return Ok(LeaseOutcome::Held(winner.lease));
6527 }
6528 }
6529 }
6530 }
6531}
6532
6533#[derive(Debug, Clone, Copy)]
6535pub struct LeaderLeaseConfig {
6536 pub ttl: Duration,
6538 pub renew_interval: Duration,
6540}
6541
6542impl Default for LeaderLeaseConfig {
6543 fn default() -> Self {
6544 Self {
6545 ttl: Duration::from_secs(15),
6549 renew_interval: Duration::from_secs(5),
6550 }
6551 }
6552}
6553
6554#[must_use]
6556pub fn lease_grants_leadership(
6557 lease: &Option<LeaderLease>,
6558 owner: &LeaderLeaseOwner,
6559 deadline: &LeaseDeadline,
6560) -> bool {
6561 deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
6562}
6563
6564#[must_use]
6566pub fn lease_grants_proof(
6567 lease: &Option<LeaderLease>,
6568 owner: &LeaderLeaseOwner,
6569 deadline: &LeaseDeadline,
6570 proof: &LeaderProof,
6571) -> bool {
6572 deadline.is_live()
6573 && proof.owner == owner.proof_owner()
6574 && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
6575}
6576
6577pub struct LeaderLeaseManager {
6579 store: Arc<LeaderLeaseStore>,
6580 owner: LeaderLeaseOwner,
6581 config: LeaderLeaseConfig,
6582 lease_tx: watch::Sender<Option<LeaderLease>>,
6583 deadline_tx: watch::Sender<Arc<LeaseDeadline>>,
6584}
6585
6586#[cfg(feature = "cluster")]
6587enum LeaseOperationEvent {
6588 Shutdown,
6589 Candidacy(Result<(), watch::error::RecvError>),
6590 Deadline,
6591 Completed {
6592 result: Result<LeaseOutcome, LeaseError>,
6593 valid_until: tokio::time::Instant,
6594 },
6595}
6596
6597#[cfg(feature = "cluster")]
6598struct LeaderLeaseExitGuard {
6599 shutdown: tokio_util::sync::CancellationToken,
6600 unexpected_exit: tokio_util::sync::CancellationToken,
6601 lease_tx: watch::Sender<Option<LeaderLease>>,
6602 deadline_tx: watch::Sender<Arc<LeaseDeadline>>,
6603}
6604
6605#[cfg(feature = "cluster")]
6606impl Drop for LeaderLeaseExitGuard {
6607 fn drop(&mut self) {
6608 self.deadline_tx.borrow().fence();
6609 self.lease_tx.send_replace(None);
6610 if !self.shutdown.is_cancelled() {
6611 tracing::error!(
6612 "leader lease manager exited without intentional shutdown; fencing process authority"
6613 );
6614 self.unexpected_exit.cancel();
6615 }
6616 }
6617}
6618
6619#[cfg(feature = "cluster")]
6620async fn wait_for_deadline(deadline: Option<tokio::time::Instant>) {
6621 if let Some(deadline) = deadline {
6622 tokio::time::sleep_until(deadline).await;
6623 } else {
6624 std::future::pending::<()>().await;
6625 }
6626}
6627
6628impl std::fmt::Debug for LeaderLeaseManager {
6629 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
6630 formatter
6631 .debug_struct("LeaderLeaseManager")
6632 .field("owner", &self.owner)
6633 .field("config", &self.config)
6634 .finish_non_exhaustive()
6635 }
6636}
6637
6638impl LeaderLeaseManager {
6639 pub fn new(
6644 store: Arc<LeaderLeaseStore>,
6645 process_lease: &ProcessLease,
6646 config: LeaderLeaseConfig,
6647 ) -> Result<Self, LeaseError> {
6648 let owner = LeaderLeaseOwner::from_process_lease(process_lease)?;
6649 let ttl_ms = i64::try_from(config.ttl.as_millis())
6650 .map_err(|_| LeaseError::Invalid("lease TTL exceeds diagnostic range".into()))?;
6651 let exact_ttl = u64::try_from(ttl_ms)
6652 .ok()
6653 .map(Duration::from_millis)
6654 .is_some_and(|ttl| ttl == config.ttl);
6655 if ttl_ms <= 0
6656 || !exact_ttl
6657 || config.renew_interval.is_zero()
6658 || config.renew_interval >= config.ttl
6659 || ttl_ms != store.ttl_ms
6660 {
6661 return Err(LeaseError::Invalid(
6662 "manager requires a renewal interval below the store's matching TTL".into(),
6663 ));
6664 }
6665 let (lease_tx, _lease_rx) = watch::channel(None);
6666 let (deadline_tx, _deadline_rx) = watch::channel(Arc::new(LeaseDeadline::uninitialized()));
6667 Ok(Self {
6668 store,
6669 owner,
6670 config,
6671 lease_tx,
6672 deadline_tx,
6673 })
6674 }
6675
6676 #[must_use]
6678 pub fn owner(&self) -> &LeaderLeaseOwner {
6679 &self.owner
6680 }
6681
6682 #[must_use]
6684 pub fn lease_watch(&self) -> watch::Receiver<Option<LeaderLease>> {
6685 self.lease_tx.subscribe()
6686 }
6687
6688 #[must_use]
6694 pub fn deadline(&self) -> Arc<LeaseDeadline> {
6695 self.deadline_tx.borrow().clone()
6696 }
6697
6698 #[must_use]
6703 pub fn deadline_watch(&self) -> watch::Receiver<Arc<LeaseDeadline>> {
6704 self.deadline_tx.subscribe()
6705 }
6706
6707 #[cfg(feature = "cluster")]
6708 fn withdraw(&self) -> bool {
6709 let deadline = self.deadline_tx.borrow().clone();
6710 let had_grant = self.lease_tx.borrow().is_some() || deadline.is_live();
6711 deadline.fence();
6715 self.lease_tx.send_replace(None);
6716 self.deadline_tx
6717 .send_replace(Arc::new(LeaseDeadline::uninitialized()));
6718 had_grant
6719 }
6720
6721 #[cfg(feature = "cluster")]
6722 fn fence(&self) {
6723 self.deadline_tx.borrow().fence();
6724 self.lease_tx.send_replace(None);
6725 }
6726
6727 #[cfg(feature = "cluster")]
6728 fn withdraw_for_new_term(
6729 &self,
6730 ticker: &mut tokio::time::Interval,
6731 valid_until: &mut Option<tokio::time::Instant>,
6732 observation: &mut Option<LeaderLeaseObservation>,
6733 held_token: &mut Option<u64>,
6734 rotate_not_before: &mut Option<tokio::time::Instant>,
6735 ) {
6736 let had_grant = self.withdraw();
6740 *valid_until = None;
6741 *observation = None;
6742 *held_token = None;
6743 let now = tokio::time::Instant::now();
6744 if had_grant {
6745 let retry_at = now
6746 .checked_add(self.config.ttl)
6747 .unwrap_or_else(tokio::time::Instant::now);
6748 if rotate_not_before.is_none_or(|current| retry_at > current) {
6752 *rotate_not_before = Some(retry_at);
6753 }
6754 }
6755 if let Some(retry_at) = *rotate_not_before {
6756 if retry_at > now {
6757 ticker.reset_at(retry_at);
6758 } else {
6759 *rotate_not_before = None;
6760 ticker.reset_immediately();
6761 }
6762 } else {
6763 ticker.reset_immediately();
6764 }
6765 }
6766
6767 #[cfg(feature = "cluster")]
6768 async fn attempt_lease(
6769 &self,
6770 shutdown: &tokio_util::sync::CancellationToken,
6771 candidate: &mut watch::Receiver<LeaderCandidacy>,
6772 valid_until: Option<tokio::time::Instant>,
6773 observation: Option<&LeaderLeaseObservation>,
6774 held_token: Option<u64>,
6775 ) -> LeaseOperationEvent {
6776 let Some(attempt_valid_until) = tokio::time::Instant::now().checked_add(self.config.ttl)
6777 else {
6778 return LeaseOperationEvent::Deadline;
6779 };
6780 let operation_deadline = valid_until.map_or(attempt_valid_until, |current| {
6781 current.min(attempt_valid_until)
6782 });
6783 let operation = async {
6784 if let Some(observation) = observation {
6785 Box::pin(
6786 self.store
6787 .try_takeover(&self.owner, observation, now_millis()),
6788 )
6789 .await
6790 } else if let Some(token) = held_token {
6791 Box::pin(self.store.renew_exact(&self.owner, token, now_millis())).await
6792 } else {
6793 Box::pin(self.store.begin_new_term(&self.owner, now_millis())).await
6794 }
6795 };
6796 tokio::select! {
6797 biased;
6798 () = shutdown.cancelled() => LeaseOperationEvent::Shutdown,
6799 changed = candidate.changed() => LeaseOperationEvent::Candidacy(changed),
6800 () = wait_for_deadline(Some(operation_deadline)) => LeaseOperationEvent::Deadline,
6801 result = operation => LeaseOperationEvent::Completed {
6802 result,
6803 valid_until: attempt_valid_until,
6804 },
6805 }
6806 }
6807
6808 #[cfg(feature = "cluster")]
6809 async fn wait_for_candidacy_change(
6810 &self,
6811 shutdown: &tokio_util::sync::CancellationToken,
6812 candidate: &mut watch::Receiver<LeaderCandidacy>,
6813 ) -> bool {
6814 tokio::select! {
6815 biased;
6816 () = shutdown.cancelled() => {
6817 self.fence();
6818 false
6819 }
6820 changed = candidate.changed() => {
6821 if changed.is_err() {
6822 self.fence();
6823 return false;
6824 }
6825 true
6826 }
6827 }
6828 }
6829
6830 #[cfg(feature = "cluster")]
6831 async fn run(
6832 self,
6833 shutdown: tokio_util::sync::CancellationToken,
6834 mut candidate: watch::Receiver<LeaderCandidacy>,
6835 ) {
6836 let mut ticker = tokio::time::interval(self.config.renew_interval);
6837 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6838 let mut valid_until: Option<tokio::time::Instant> = None;
6839 let mut observation: Option<LeaderLeaseObservation> = None;
6840 let mut held_token = None;
6843 let mut rotate_not_before = None;
6844 let mut candidacy_generation = candidate.borrow().generation;
6845
6846 loop {
6847 let candidacy = *candidate.borrow_and_update();
6848 if candidacy.generation != candidacy_generation {
6849 self.withdraw_for_new_term(
6850 &mut ticker,
6851 &mut valid_until,
6852 &mut observation,
6853 &mut held_token,
6854 &mut rotate_not_before,
6855 );
6856 candidacy_generation = candidacy.generation;
6857 }
6858 if !candidacy.eligible {
6859 self.withdraw_for_new_term(
6860 &mut ticker,
6861 &mut valid_until,
6862 &mut observation,
6863 &mut held_token,
6864 &mut rotate_not_before,
6865 );
6866 if !self
6867 .wait_for_candidacy_change(&shutdown, &mut candidate)
6868 .await
6869 {
6870 return;
6871 }
6872 continue;
6873 }
6874
6875 tokio::select! {
6876 biased;
6877 () = shutdown.cancelled() => {
6878 self.fence();
6879 return;
6880 }
6881 changed = candidate.changed() => {
6882 if changed.is_err() {
6883 self.fence();
6884 return;
6885 }
6886 continue;
6887 }
6888 () = wait_for_deadline(valid_until) => {
6889 tracing::warn!(
6890 owner = ?self.owner,
6891 "leader lease local deadline expired; withdrawing and rotating the fencing token"
6892 );
6893 self.withdraw_for_new_term(
6894 &mut ticker,
6895 &mut valid_until,
6896 &mut observation,
6897 &mut held_token,
6898 &mut rotate_not_before,
6899 );
6900 continue;
6901 }
6902 _ = ticker.tick() => {}
6903 }
6904
6905 let (result, attempt_valid_until) = match Box::pin(self.attempt_lease(
6906 &shutdown,
6907 &mut candidate,
6908 valid_until,
6909 observation.as_ref(),
6910 held_token,
6911 ))
6912 .await
6913 {
6914 LeaseOperationEvent::Shutdown => {
6915 self.fence();
6916 return;
6917 }
6918 LeaseOperationEvent::Deadline => {
6919 tracing::warn!(
6920 owner = ?self.owner,
6921 "leader lease operation exceeded its local deadline; withdrawing and rotating the fencing token"
6922 );
6923 self.withdraw_for_new_term(
6924 &mut ticker,
6925 &mut valid_until,
6926 &mut observation,
6927 &mut held_token,
6928 &mut rotate_not_before,
6929 );
6930 continue;
6931 }
6932 LeaseOperationEvent::Candidacy(changed) => {
6933 if changed.is_err() {
6934 self.fence();
6935 return;
6936 }
6937 continue;
6938 }
6939 LeaseOperationEvent::Completed {
6940 result,
6941 valid_until: attempt_valid_until,
6942 } => {
6943 let response_at = tokio::time::Instant::now();
6944 if response_at >= attempt_valid_until
6945 || valid_until.is_some_and(|current| response_at >= current)
6946 {
6947 tracing::warn!(
6948 owner = ?self.owner,
6949 "leader lease response arrived after its local deadline; withdrawing and rotating the fencing token"
6950 );
6951 self.withdraw_for_new_term(
6952 &mut ticker,
6953 &mut valid_until,
6954 &mut observation,
6955 &mut held_token,
6956 &mut rotate_not_before,
6957 );
6958 continue;
6959 }
6960 (result, attempt_valid_until)
6961 }
6962 };
6963
6964 match result {
6965 Ok(LeaseOutcome::Acquired(lease)) if lease.owner == self.owner => {
6966 let publication_at = tokio::time::Instant::now();
6967 if publication_at >= attempt_valid_until
6968 || valid_until.is_some_and(|current| publication_at >= current)
6969 {
6970 tracing::warn!(
6971 owner = ?self.owner,
6972 "leader lease publication crossed its local deadline; withdrawing and rotating the fencing token"
6973 );
6974 self.withdraw_for_new_term(
6975 &mut ticker,
6976 &mut valid_until,
6977 &mut observation,
6978 &mut held_token,
6979 &mut rotate_not_before,
6980 );
6981 continue;
6982 }
6983 observation = None;
6984 valid_until = Some(attempt_valid_until);
6985 held_token = Some(lease.token);
6986 rotate_not_before = None;
6987 let deadline = self.deadline_tx.borrow().clone();
6988 deadline.extend_until(attempt_valid_until.into_std());
6989 if !deadline.is_live() {
6990 tracing::warn!(
6991 owner = ?self.owner,
6992 "leader lease deadline expired during local publication; withdrawing and rotating the fencing token"
6993 );
6994 self.withdraw_for_new_term(
6995 &mut ticker,
6996 &mut valid_until,
6997 &mut observation,
6998 &mut held_token,
6999 &mut rotate_not_before,
7000 );
7001 continue;
7002 }
7003 self.lease_tx.send_replace(Some(lease));
7004 }
7005 Ok(LeaseOutcome::Acquired(_)) => {
7006 self.fence();
7007 return;
7008 }
7009 Ok(LeaseOutcome::Held(rival)) => {
7010 self.withdraw();
7011 valid_until = None;
7012 held_token = None;
7013 let unchanged = observation
7014 .as_ref()
7015 .is_some_and(|observed| observed.lease.has_same_liveness_identity(&rival));
7016 if !unchanged {
7017 match self.store.observe_rival(&self.owner, &rival) {
7018 Ok(new_observation) => observation = Some(new_observation),
7019 Err(error) => {
7020 tracing::warn!(%error, "leader lease observation rejected");
7021 observation = None;
7022 }
7023 }
7024 }
7025 }
7026 Err(LeaseError::Fenced(error)) => {
7027 tracing::warn!(%error, "leader lease renewal was fenced");
7028 self.fence();
7029 return;
7030 }
7031 Err(error) => {
7032 tracing::warn!(%error, "leader lease operation failed");
7033 }
7034 }
7035 }
7036 }
7037
7038 #[cfg(feature = "cluster")]
7043 #[must_use]
7044 pub fn spawn(
7045 self,
7046 shutdown: tokio_util::sync::CancellationToken,
7047 candidate: watch::Receiver<LeaderCandidacy>,
7048 ) -> tokio::task::JoinHandle<()> {
7049 tokio::spawn(self.run(shutdown, candidate))
7050 }
7051
7052 #[cfg(feature = "cluster")]
7057 #[must_use]
7058 pub fn spawn_supervised(
7059 self,
7060 shutdown: tokio_util::sync::CancellationToken,
7061 candidate: watch::Receiver<LeaderCandidacy>,
7062 unexpected_exit: tokio_util::sync::CancellationToken,
7063 ) -> tokio::task::JoinHandle<()> {
7064 tokio::spawn(async move {
7065 let run_shutdown = shutdown.clone();
7066 let _exit_guard = LeaderLeaseExitGuard {
7067 shutdown,
7068 unexpected_exit,
7069 lease_tx: self.lease_tx.clone(),
7070 deadline_tx: self.deadline_tx.clone(),
7071 };
7072 self.run(run_shutdown, candidate).await;
7073 })
7074 }
7075}
7076
7077#[cfg(test)]
7078mod tests;