Skip to main content

laminar_core/cluster/control/leader_lease/
mod.rs

1//! Durable, append-only leader fencing.
2
3mod artifact_admission;
4mod assignment_drain;
5mod attempt_status;
6mod subscription_replay;
7mod topology;
8mod topology_activation;
9mod topology_admission;
10mod topology_checkpoint;
11mod topology_commit;
12mod topology_committed_restore;
13mod topology_cut;
14mod topology_migration_root;
15mod topology_preparation;
16mod topology_recovery;
17mod topology_recovery_round;
18mod topology_restore;
19mod topology_retention;
20mod topology_subscription;
21mod topology_target_preparation;
22
23mod checkpoint_publication;
24mod history_retention;
25mod manager;
26mod release;
27
28pub use attempt_status::{ClusterAttemptStatus, ClusterCommitHead};
29pub use manager::LeaderLeaseManager;
30pub use subscription_replay::{
31    SubscriptionReplayPin, SubscriptionReplayPinAcquire, SUBSCRIPTION_REPLAY_PIN_RENEW_INTERVAL,
32};
33
34use std::collections::{BTreeMap, BTreeSet};
35use std::sync::atomic::{AtomicBool, Ordering};
36use std::sync::{Arc, Weak};
37use std::time::{Duration, Instant};
38
39use bytes::Bytes;
40use object_store::path::Path as OsPath;
41use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
42use parking_lot::Mutex;
43use serde::{Deserialize, Serialize};
44use sha2::{Digest as _, Sha256};
45use tokio::sync::watch;
46use tokio_stream::StreamExt;
47use uuid::Uuid;
48
49use crate::checkpoint::{
50    probe_object_store_conditional_update, AssignmentDrainId, AssignmentDrainTransition,
51    CheckpointAssignmentFence, CheckpointStoreError, CommittedCheckpointIndex,
52    CommittedCheckpointRef, LeaderProof, LeaderProofOwner, MAX_CHECKPOINT_PARTICIPANTS,
53};
54use crate::checkpoint_decision::{
55    CheckpointArtifactInventory, CheckpointDecisionStore, CheckpointOutcome, CheckpointScope,
56    CheckpointVerdict, DecisionError, RecordOutcomeResult,
57};
58use crate::cluster::discovery::NodeId;
59
60use super::catalog_manifest::{
61    CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
62};
63use super::controller::{
64    RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
65    RecoveryFaultDisposition, RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
66    MAX_RECOVERY_ANNOUNCEMENT_BYTES,
67};
68use super::lease_deadline::LeaseDeadline;
69use super::process_lease::{ProcessLease, ProcessLeaseFence};
70use super::snapshot::{
71    AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
72};
73
74const LEASE_PREFIX: &str = "control/leader-lease/";
75const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
76const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
77const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
78const AUTHORITY_RECORD_VERSION: u32 = 12;
79const TOPOLOGY_AUTHORITY_RECORD_VERSION: u32 = 13;
80const TOPOLOGY_ADMISSION_RECORD_VERSION: u32 = 14;
81const TOPOLOGY_CUT_RECORD_VERSION: u32 = 15;
82const TOPOLOGY_PREPARATION_RECORD_VERSION: u32 = 16;
83const TOPOLOGY_MIGRATION_ROOT_RECORD_VERSION: u32 = 17;
84const TOPOLOGY_SOURCE_ROOT_RECORD_VERSION: u32 = 18;
85const TOPOLOGY_TARGET_PREPARATION_RECORD_VERSION: u32 = 19;
86const TOPOLOGY_COMMIT_RECORD_VERSION: u32 = 20;
87const TOPOLOGY_INSTALLATION_RECORD_VERSION: u32 = 21;
88const TOPOLOGY_RECOVERY_RECORD_VERSION: u32 = 22;
89const TOPOLOGY_SUBMISSION_RECORD_VERSION: u32 = 23;
90const AUTHORITY_HEAD_VERSION: u32 = 1;
91const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
92const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
93const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
94    + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
95    + 2;
96const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
97const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
98const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
99const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
100const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
101const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
102const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
103const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
104const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
105const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
106const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
107const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
108#[cfg(test)]
109const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
110
111fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
112    match error {
113        SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
114        error => LeaseError::Invalid(format!("{context}: {error}")),
115    }
116}
117
118fn lease_path(sequence: u64) -> OsPath {
119    OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
120}
121
122fn authority_head_path() -> OsPath {
123    OsPath::from(AUTHORITY_HEAD_PATH)
124}
125
126fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
127    OsPath::from(format!(
128        "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
129        reference.release.generation(),
130        reference.sha256
131    ))
132}
133
134fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
135    let raw = path
136        .as_ref()
137        .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
138        .and_then(|value| value.strip_prefix("generation="))
139        .ok_or_else(|| {
140            LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
141        })?;
142    let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
143        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
144    })?;
145    let digest = digest.strip_suffix(".json").ok_or_else(|| {
146        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
147    })?;
148    if generation.len() != 20
149        || !generation.bytes().all(|byte| byte.is_ascii_digit())
150        || digest.len() != 64
151        || !digest
152            .bytes()
153            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
154    {
155        return Err(LeaseError::Invalid(format!(
156            "invalid recovery release terminal path {path}"
157        )));
158    }
159    let generation = generation.parse::<u64>().map_err(|error| {
160        LeaseError::Invalid(format!(
161            "invalid recovery release terminal generation in {path}: {error}"
162        ))
163    })?;
164    if generation == 0 {
165        return Err(LeaseError::Invalid(format!(
166            "invalid recovery release terminal path {path}"
167        )));
168    }
169    Ok((generation, digest.to_owned()))
170}
171
172fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
173    let raw = path
174        .as_ref()
175        .strip_prefix(LEASE_PREFIX)
176        .and_then(|file| file.strip_prefix('v'))
177        .and_then(|file| file.strip_suffix(".json"))
178        .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
179    if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
180        return Err(LeaseError::Invalid(format!(
181            "invalid leader authority sequence in {path}"
182        )));
183    }
184    let sequence = raw.parse::<u64>().map_err(|error| {
185        LeaseError::Invalid(format!(
186            "invalid leader authority sequence in {path}: {error}"
187        ))
188    })?;
189    if sequence == 0 || lease_path(sequence) != *path {
190        return Err(LeaseError::Invalid(format!(
191            "noncanonical leader authority path {path}"
192        )));
193    }
194    Ok(sequence)
195}
196
197fn consume_live_authority_link(traversed: &mut usize) -> bool {
198    if *traversed == MAX_LIVE_AUTHORITY_LINKS {
199        return false;
200    }
201    *traversed += 1;
202    true
203}
204
205fn now_millis() -> i64 {
206    std::time::SystemTime::now()
207        .duration_since(std::time::UNIX_EPOCH)
208        .ok()
209        .and_then(|duration| i64::try_from(duration.as_millis()).ok())
210        .unwrap_or(i64::MAX)
211}
212
213async fn read_authority_record(
214    store: &dyn ObjectStore,
215    sequence: u64,
216) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
217    let result = match store.get(&lease_path(sequence)).await {
218        Ok(result) => result,
219        Err(object_store::Error::NotFound { .. }) => return Ok(None),
220        Err(error) => return Err(LeaseError::Io(error.to_string())),
221    };
222    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
223        return Err(LeaseError::Invalid(format!(
224            "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
225            result.meta.size
226        )));
227    }
228    let bytes = result
229        .bytes()
230        .await
231        .map_err(|error| LeaseError::Io(error.to_string()))?;
232    let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
233    record.validate()?;
234    if record.lease.seq != sequence {
235        return Err(LeaseError::Invalid(
236            "authority record sequence does not match its object name".into(),
237        ));
238    }
239    let canonical = serde_json::to_vec(&record)?;
240    if canonical.as_slice() != bytes.as_ref() {
241        return Err(LeaseError::Invalid(format!(
242            "leader authority record {sequence} does not use its canonical body"
243        )));
244    }
245    Ok(Some(record))
246}
247
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249#[serde(deny_unknown_fields)]
250struct AuthorityHeadPointer {
251    version: u32,
252    sequence: u64,
253    nonce: Uuid,
254}
255
256impl AuthorityHeadPointer {
257    fn new(sequence: u64) -> Result<Self, LeaseError> {
258        let pointer = Self {
259            version: AUTHORITY_HEAD_VERSION,
260            sequence,
261            nonce: Uuid::new_v4(),
262        };
263        pointer.validate()?;
264        Ok(pointer)
265    }
266
267    fn validate(self) -> Result<(), LeaseError> {
268        if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
269            return Err(LeaseError::Invalid(
270                "leader authority head has an unsupported version, zero sequence, or nil nonce"
271                    .into(),
272            ));
273        }
274        Ok(())
275    }
276}
277
278#[derive(Clone)]
279struct VersionedAuthorityHeadPointer {
280    pointer: AuthorityHeadPointer,
281    update_version: UpdateVersion,
282}
283
284struct PublishedAuthorityHead {
285    record: LeaderAuthorityRecord,
286    pointer: VersionedAuthorityHeadPointer,
287}
288
289async fn read_authority_head_pointer(
290    store: &dyn ObjectStore,
291) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
292    let result = match store.get(&authority_head_path()).await {
293        Ok(result) => result,
294        Err(object_store::Error::NotFound { .. }) => return Ok(None),
295        Err(error) => return Err(LeaseError::Io(error.to_string())),
296    };
297    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
298        return Err(LeaseError::Invalid(format!(
299            "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
300            result.meta.size
301        )));
302    }
303    let update_version = UpdateVersion {
304        e_tag: result.meta.e_tag.clone(),
305        version: result.meta.version.clone(),
306    };
307    let bytes = result
308        .bytes()
309        .await
310        .map_err(|error| LeaseError::Io(error.to_string()))?;
311    let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
312    pointer.validate()?;
313    let canonical = serde_json::to_vec(&pointer)?;
314    if canonical.as_slice() != bytes.as_ref() {
315        return Err(LeaseError::Invalid(
316            "leader authority head does not use its canonical body".into(),
317        ));
318    }
319    Ok(Some(VersionedAuthorityHeadPointer {
320        pointer,
321        update_version,
322    }))
323}
324
325fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
326    let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
327    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
328    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
329        return Err(LeaseError::Invalid(format!(
330            "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
331            encoded.len()
332        )));
333    }
334    Ok(Bytes::from(encoded))
335}
336
337fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
338    record.validate()?;
339    let encoded = serde_json::to_vec(record)?;
340    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
341    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
342        return Err(LeaseError::Invalid(format!(
343            "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
344            encoded.len()
345        )));
346    }
347    if record.recovery_fault_slots.iter().any(|slot| slot.active) {
348        ensure_recovery_fault_authority_headroom(encoded_len)?;
349    }
350    Ok(Bytes::from(encoded))
351}
352
353fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
354    let limit = MAX_AUTHORITY_RECORD_BYTES
355        .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
356        .expect("recovery fault headroom must fit the authority record");
357    if encoded_len > limit {
358        return Err(LeaseError::Invalid(format!(
359            "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
360        )));
361    }
362    Ok(())
363}
364
365/// Exact process incarnation eligible to hold the leader lease.
366#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
367pub struct LeaderLeaseOwner {
368    /// Stable cluster node identity.
369    pub node: NodeId,
370    /// Boot-unique process identity.
371    pub boot: Uuid,
372    /// Durable stable-node process term.
373    pub process_term: u64,
374}
375
376impl LeaderLeaseOwner {
377    fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
378        process
379            .validate(process.node)
380            .map_err(|error| LeaseError::Invalid(error.to_string()))?;
381        Ok(Self {
382            node: process.node,
383            boot: process.owner,
384            process_term: process.term,
385        })
386    }
387
388    fn validate(&self) -> Result<(), LeaseError> {
389        if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
390            return Err(LeaseError::Invalid(
391                "leader owner node, boot identity, and process term must be nonzero".into(),
392            ));
393        }
394        Ok(())
395    }
396
397    fn proof_owner(&self) -> LeaderProofOwner {
398        LeaderProofOwner {
399            node_id: self.node.0,
400            boot_id: self.boot,
401            process_term: self.process_term,
402        }
403    }
404}
405
406/// Durable leader lease record.
407#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
408pub struct LeaderLease {
409    /// Append-only compare-and-set sequence.
410    pub seq: u64,
411    /// Monotonic liveness sequence, advanced only by an acquisition or renewal.
412    pub renewal_sequence: u64,
413    /// Fencing token, stable across uninterrupted renewals and advanced for each authority term.
414    pub token: u64,
415    /// Exact process incarnation holding the lease.
416    pub owner: LeaderLeaseOwner,
417    /// Owner-written wall-clock expiry for diagnostics only.
418    pub expires_at_ms: i64,
419    /// Immutable catalog content reference, advanced only by an atomic topology Commit after seal.
420    pub catalog_manifest: Option<CatalogManifestRef>,
421}
422
423#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
424#[serde(deny_unknown_fields)]
425struct OutcomeLink {
426    sequence: u64,
427    epoch: u64,
428    checkpoint_id: u64,
429}
430
431impl OutcomeLink {
432    fn validate(self) -> Result<(), LeaseError> {
433        let attempt = crate::checkpoint::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
434        if self.sequence == 0 || !attempt.is_canonical() {
435            return Err(LeaseError::Invalid(
436                "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
437                    .into(),
438            ));
439        }
440        Ok(())
441    }
442}
443
444#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
445#[serde(deny_unknown_fields)]
446struct AssignmentDecisionLink {
447    sequence: u64,
448    target_version: u64,
449}
450
451#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
452#[serde(deny_unknown_fields)]
453struct AuthorityRecoveryFaultSlot {
454    publisher: RecoveryFaultPublisher,
455    request_sequence: u64,
456    fault_sequence: u64,
457    #[serde(
458        default,
459        skip_serializing_if = "RecoveryFaultDisposition::is_recoverable"
460    )]
461    disposition: RecoveryFaultDisposition,
462    active: bool,
463}
464
465impl AuthorityRecoveryFaultSlot {
466    fn validate(&self) -> Result<(), LeaseError> {
467        self.publisher.validate().map_err(LeaseError::Invalid)?;
468        if self.request_sequence == 0 || self.fault_sequence == 0 {
469            return Err(LeaseError::Invalid(
470                "recovery fault request and authority sequence must be nonzero".into(),
471            ));
472        }
473        if self.disposition == RecoveryFaultDisposition::Terminal && !self.active {
474            return Err(LeaseError::Invalid(
475                "terminal recovery fault authority cannot be tombstoned".into(),
476            ));
477        }
478        Ok(())
479    }
480
481    fn fault(&self) -> RecoveryFault {
482        RecoveryFault {
483            reporter: NodeId(self.publisher.participant.node_id),
484            sequence: self.fault_sequence,
485            disposition: self.disposition,
486        }
487    }
488
489    fn matches_request(
490        &self,
491        publisher: RecoveryFaultPublisher,
492        request_sequence: u64,
493        disposition: RecoveryFaultDisposition,
494    ) -> bool {
495        self.publisher == publisher
496            && self.request_sequence == request_sequence
497            && self.disposition == disposition
498    }
499}
500
501#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
502#[serde(deny_unknown_fields)]
503pub(crate) struct RecoveryReleaseTerminalRef {
504    release: RecoveryReleaseId,
505    sha256: String,
506    encoded_len: u64,
507}
508
509impl RecoveryReleaseTerminalRef {
510    fn validate(&self) -> Result<(), LeaseError> {
511        if !self.release.is_canonical()
512            || self.sha256.len() != 64
513            || !self
514                .sha256
515                .bytes()
516                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
517            || self.encoded_len == 0
518            || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
519        {
520            return Err(LeaseError::Invalid(
521                "recovery release terminal reference is not canonical".into(),
522            ));
523        }
524        Ok(())
525    }
526
527    pub(crate) fn generation(&self) -> u64 {
528        self.release.generation()
529    }
530}
531
532#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
533#[serde(deny_unknown_fields)]
534struct AuthorityRecoveryReleaseCommit {
535    terminal: RecoveryReleaseTerminalRef,
536    leader_proof: LeaderProof,
537}
538
539impl AuthorityRecoveryReleaseCommit {
540    fn validate(&self) -> Result<(), LeaseError> {
541        self.terminal.validate()?;
542        if !self.leader_proof.is_canonical() {
543            return Err(LeaseError::Invalid(
544                "recovery release commit has no canonical leader proof".into(),
545            ));
546        }
547        Ok(())
548    }
549}
550
551#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
552#[serde(deny_unknown_fields)]
553pub(crate) struct RecoveryReleaseLink {
554    sequence: u64,
555    terminal: RecoveryReleaseTerminalRef,
556}
557
558impl RecoveryReleaseLink {
559    fn validate(&self) -> Result<(), LeaseError> {
560        if self.sequence == 0 {
561            return Err(LeaseError::Invalid(
562                "recovery release authority link has a zero sequence".into(),
563            ));
564        }
565        self.terminal.validate()
566    }
567}
568
569#[derive(Debug, Clone, PartialEq, Eq)]
570pub(crate) enum RecordRecoveryReleaseCommitResult {
571    Created(RecoveryReleaseTerminalRef),
572    Unchanged(RecoveryReleaseTerminalRef),
573    Conflict { winner: RecoveryReleaseTerminalRef },
574    FaultsChanged,
575}
576
577#[derive(Debug, Clone, Copy, PartialEq, Eq)]
578pub(crate) enum RecordRecoveryFaultResult {
579    Active,
580    AlreadyCleared,
581    CoveredByNewerRequest,
582    TerminalFenceActive,
583    Superseded,
584}
585
586fn recovery_release_id_for_terminal(
587    terminal: &RecoveryAnnouncement,
588) -> Result<RecoveryReleaseId, LeaseError> {
589    terminal.validate().map_err(LeaseError::Invalid)?;
590    let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
591        return Err(LeaseError::Invalid(
592            "recovery release terminal must carry ReleaseCommitted".into(),
593        ));
594    };
595    RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
596        round: terminal.round.clone(),
597        phase: RecoverPhase::Release { epoch },
598    })
599    .map_err(LeaseError::Invalid)
600}
601
602fn encode_recovery_release_terminal(
603    terminal: &RecoveryAnnouncement,
604) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
605    let release = recovery_release_id_for_terminal(terminal)?;
606    let encoded = serde_json::to_vec(terminal)?;
607    let encoded_len = u64::try_from(encoded.len())
608        .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
609    if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
610        return Err(LeaseError::Invalid(format!(
611            "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
612        )));
613    }
614    let reference = RecoveryReleaseTerminalRef {
615        release,
616        sha256: format!("{:x}", Sha256::digest(&encoded)),
617        encoded_len,
618    };
619    reference.validate()?;
620    Ok((Bytes::from(encoded), reference))
621}
622
623/// Immutable settlement of one exact assignment-drain transition.
624#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
625#[serde(deny_unknown_fields)]
626pub struct AssignmentDrainDecision {
627    /// Exact predecessor-to-target transition being settled.
628    pub transition: AssignmentDrainTransition,
629    /// Durable leader term that decided the transition.
630    pub leader_proof: LeaderProof,
631    /// Whether to install the certified target or restore the predecessor map.
632    pub verdict: AssignmentDrainVerdict,
633    /// Exact predecessor-fenced checkpoint used to hand moved vnodes to the target assignment.
634    /// Present if and only if the drain commits.
635    pub handoff_checkpoint: Option<CommittedCheckpointRef>,
636}
637
638impl AssignmentDrainDecision {
639    /// Commit an exact canonical transition against its state handoff cut.
640    ///
641    /// # Errors
642    /// Rejects malformed authority or checkpoint references.
643    pub fn commit(
644        transition: &AssignmentDrainTransition,
645        leader_proof: LeaderProof,
646        handoff_checkpoint: CommittedCheckpointRef,
647    ) -> Result<Self, String> {
648        handoff_checkpoint.validate()?;
649        if !transition.is_canonical() || leader_proof != transition.leader {
650            return Err("assignment drain decision requires a canonical transition".into());
651        }
652        Ok(Self {
653            transition: transition.clone(),
654            leader_proof,
655            verdict: AssignmentDrainVerdict::Commit,
656            handoff_checkpoint: Some(handoff_checkpoint),
657        })
658    }
659
660    /// Abort an exact canonical transition under the deciding leader term.
661    ///
662    /// # Errors
663    /// Rejects malformed transition or leader authority.
664    pub fn abort(
665        transition: &AssignmentDrainTransition,
666        leader_proof: LeaderProof,
667    ) -> Result<Self, String> {
668        if !transition.is_canonical() || !leader_proof.is_canonical() {
669            return Err("assignment drain decision requires a canonical transition".into());
670        }
671        Ok(Self {
672            transition: transition.clone(),
673            leader_proof,
674            verdict: AssignmentDrainVerdict::Abort,
675            handoff_checkpoint: None,
676        })
677    }
678
679    /// Compact identity used by source-drain receipts and authority lookups.
680    #[must_use]
681    pub fn round(&self) -> AssignmentDrainId {
682        self.transition.id()
683    }
684
685    /// Target assignment version settled by this decision.
686    #[must_use]
687    pub fn target_version(&self) -> u64 {
688        self.transition.target.assignment_version
689    }
690
691    fn validate(&self) -> Result<(), LeaseError> {
692        if !self.transition.is_canonical()
693            || !self.leader_proof.is_canonical()
694            || (self.verdict == AssignmentDrainVerdict::Commit
695                && self.leader_proof != self.transition.leader)
696        {
697            return Err(LeaseError::Invalid(
698                "assignment drain decision is not canonical".into(),
699            ));
700        }
701        match (self.verdict, self.handoff_checkpoint.as_ref()) {
702            (AssignmentDrainVerdict::Commit, Some(reference)) => {
703                reference.validate().map_err(LeaseError::Invalid)?;
704            }
705            (AssignmentDrainVerdict::Abort, None) => {}
706            _ => {
707                return Err(LeaseError::Invalid(
708                    "assignment drain decision has an invalid handoff checkpoint".into(),
709                ));
710            }
711        }
712        Ok(())
713    }
714}
715
716/// Terminal result for an assignment-drain transition.
717#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
718#[serde(rename_all = "snake_case")]
719pub enum AssignmentDrainVerdict {
720    /// Install the transition's certified target assignment.
721    Commit,
722    /// Restore the predecessor owner map at the target version.
723    Abort,
724}
725
726/// Result of admitting a drain decision through the shared authority sequence.
727#[derive(Debug, Clone, PartialEq, Eq)]
728pub enum RecordAssignmentDrainDecisionResult {
729    /// This call created the immutable decision.
730    Created(AssignmentDrainDecision),
731    /// The same decision was already durable.
732    Unchanged(AssignmentDrainDecision),
733    /// Another terminal decision already won for this transition version.
734    Conflict {
735        /// Immutable winner.
736        winner: AssignmentDrainDecision,
737    },
738}
739
740/// Immutable authorization to install one staged failure-recovery assignment.
741#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
742#[serde(deny_unknown_fields)]
743pub struct AssignmentRecoveryDecision {
744    /// Exact assignment being replaced.
745    pub predecessor: CheckpointAssignmentFence,
746    /// Exact successor assignment authorized for installation.
747    pub target: CheckpointAssignmentFence,
748    /// Content-addressed staged snapshot whose assignment must equal `target`.
749    pub proposal: AssignmentSnapshotRef,
750    /// Exact process-lease takeover proofs for every predecessor process absent from `target`.
751    pub removed_process_fences: Vec<ProcessLeaseFence>,
752    /// Exact committed state cut installed by this recovery.
753    pub recovery_checkpoint: CommittedCheckpointRef,
754    /// Durable leader term that authorized this recovery.
755    pub leader_proof: LeaderProof,
756}
757
758impl AssignmentRecoveryDecision {
759    /// Construct one exact failure-recovery authorization.
760    ///
761    /// # Errors
762    /// Rejects invalid fences, a non-successor proposal, or an incomplete process-removal proof.
763    pub fn new(
764        predecessor: CheckpointAssignmentFence,
765        target: CheckpointAssignmentFence,
766        proposal: AssignmentSnapshotRef,
767        removed_process_fences: Vec<ProcessLeaseFence>,
768        recovery_checkpoint: CommittedCheckpointRef,
769        leader_proof: LeaderProof,
770    ) -> Result<Self, String> {
771        let decision = Self {
772            predecessor,
773            target,
774            proposal,
775            removed_process_fences,
776            recovery_checkpoint,
777            leader_proof,
778        };
779        decision.validate().map_err(|error| error.to_string())?;
780        Ok(decision)
781    }
782
783    /// Target assignment version settled by this decision.
784    #[must_use]
785    pub fn target_version(&self) -> u64 {
786        self.target.assignment_version
787    }
788
789    fn validate(&self) -> Result<(), LeaseError> {
790        if !self.predecessor.is_canonical()
791            || !self.target.is_canonical()
792            || !self.leader_proof.is_canonical()
793            || self.predecessor.vnode_count != self.target.vnode_count
794            || self.predecessor.assignment_version.checked_add(1)
795                != Some(self.target.assignment_version)
796            || self.proposal.validate().is_err()
797            || self.proposal.version != self.target.assignment_version
798            || self.recovery_checkpoint.validate().is_err()
799        {
800            return Err(LeaseError::Invalid(
801                "assignment recovery decision is not a canonical successor".into(),
802            ));
803        }
804
805        let removed: Vec<_> = self
806            .predecessor
807            .participants
808            .iter()
809            .filter(|participant| {
810                self.target.participant_incarnation(participant.node_id)
811                    != Some(participant.boot_incarnation)
812            })
813            .collect();
814        if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
815            return Err(LeaseError::Invalid(
816                "assignment recovery decision requires the exact removed predecessor process set"
817                    .into(),
818            ));
819        }
820        for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
821            if !fence.is_canonical()
822                || fence.predecessor.node.0 != participant.node_id
823                || fence.predecessor.owner != participant.boot_incarnation
824            {
825                return Err(LeaseError::Invalid(
826                    "assignment recovery process fences are incomplete, unsorted, or unrelated"
827                        .into(),
828                ));
829            }
830        }
831        Ok(())
832    }
833}
834
835/// Result of admitting a recovery decision through the shared authority sequence.
836#[derive(Debug, Clone, PartialEq, Eq)]
837pub enum RecordAssignmentRecoveryDecisionResult {
838    /// This call created the immutable decision.
839    Created(AssignmentRecoveryDecision),
840    /// The same decision was already durable.
841    Unchanged(AssignmentRecoveryDecision),
842    /// Another recovery decision already won for this target version.
843    Conflict {
844        /// Immutable winner.
845        winner: AssignmentRecoveryDecision,
846    },
847}
848
849#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
850#[serde(
851    tag = "kind",
852    content = "decision",
853    rename_all = "snake_case",
854    deny_unknown_fields
855)]
856enum AuthorityAssignmentDecision {
857    Drain(AssignmentDrainDecision),
858    Recovery(AssignmentRecoveryDecision),
859}
860
861impl AuthorityAssignmentDecision {
862    fn target_version(&self) -> u64 {
863        match self {
864            Self::Drain(decision) => decision.target_version(),
865            Self::Recovery(decision) => decision.target_version(),
866        }
867    }
868
869    fn predecessor(&self) -> &CheckpointAssignmentFence {
870        match self {
871            Self::Drain(decision) => &decision.transition.predecessor,
872            Self::Recovery(decision) => &decision.predecessor,
873        }
874    }
875
876    fn materialized_target(&self) -> CheckpointAssignmentFence {
877        match self {
878            Self::Drain(decision) => match decision.verdict {
879                AssignmentDrainVerdict::Commit => decision.transition.target.clone(),
880                AssignmentDrainVerdict::Abort => {
881                    let mut target = decision.transition.predecessor.clone();
882                    target.assignment_version = decision.transition.target.assignment_version;
883                    target
884                }
885            },
886            Self::Recovery(decision) => decision.target.clone(),
887        }
888    }
889
890    fn leader_proof(&self) -> &LeaderProof {
891        match self {
892            Self::Drain(decision) => &decision.leader_proof,
893            Self::Recovery(decision) => &decision.leader_proof,
894        }
895    }
896
897    fn validate(&self) -> Result<(), LeaseError> {
898        match self {
899            Self::Drain(decision) => decision.validate(),
900            Self::Recovery(decision) => decision.validate(),
901        }
902    }
903}
904
905enum RecordAuthorityAssignmentDecisionResult {
906    Created(AuthorityAssignmentDecision),
907    Unchanged(AuthorityAssignmentDecision),
908    Conflict { winner: AuthorityAssignmentDecision },
909}
910
911/// Destructive phase authorized by the cluster leader authority.
912#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
913#[serde(rename_all = "snake_case")]
914pub enum ClusterArtifactCleanupPhase {
915    /// Delete state objects that are not referenced by the protected recovery cuts.
916    DeleteData,
917    /// Delete the exact participant manifests and committed-checkpoint index.
918    DeleteMetadata,
919}
920
921/// Exact, crash-resumable cluster checkpoint artifact cleanup position.
922#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
923#[serde(deny_unknown_fields)]
924pub struct ClusterArtifactCleanupCursor {
925    /// Oldest live committed checkpoint protected by this retention segment.
926    pub protected: CommittedCheckpointRef,
927    /// Exact expired committed checkpoint being reclaimed.
928    pub current: CommittedCheckpointRef,
929    /// Exact older checkpoint to process after `current`, if any.
930    pub next: Option<CommittedCheckpointRef>,
931    /// Previously reclaimed checkpoint at which this segment must stop.
932    pub stop_before: Option<CommittedCheckpointRef>,
933    /// Exact participants whose manifests belong to `current`.
934    pub participant_ids: Vec<u64>,
935    /// Destructive operation durably authorized for `current`.
936    pub phase: ClusterArtifactCleanupPhase,
937}
938
939impl ClusterArtifactCleanupCursor {
940    fn validate(&self) -> Result<(), LeaseError> {
941        for (name, reference) in [
942            ("protected", Some(&self.protected)),
943            ("current", Some(&self.current)),
944            ("next", self.next.as_ref()),
945            ("stop-before", self.stop_before.as_ref()),
946        ] {
947            if let Some(reference) = reference {
948                reference.validate().map_err(|error| {
949                    LeaseError::Invalid(format!(
950                        "cluster artifact cleanup {name} reference: {error}"
951                    ))
952                })?;
953            }
954        }
955        if self.current.epoch >= self.protected.epoch
956            || self.current.checkpoint_id >= self.protected.checkpoint_id
957        {
958            return Err(LeaseError::Invalid(
959                "cluster artifact cleanup current checkpoint is not older than its protected cut"
960                    .into(),
961            ));
962        }
963        if let Some(next) = self.next.as_ref() {
964            if next.epoch >= self.current.epoch || next.checkpoint_id >= self.current.checkpoint_id
965            {
966                return Err(LeaseError::Invalid(
967                    "cluster artifact cleanup next checkpoint is not older than its current checkpoint"
968                        .into(),
969                ));
970            }
971        }
972        if let Some(stop) = self.stop_before.as_ref() {
973            if stop.epoch >= self.current.epoch || stop.checkpoint_id >= self.current.checkpoint_id
974            {
975                return Err(LeaseError::Invalid(
976                    "cluster artifact cleanup stop boundary is not older than its current checkpoint"
977                        .into(),
978                ));
979            }
980            let next = self.next.as_ref().ok_or_else(|| {
981                LeaseError::Invalid(
982                    "cluster artifact cleanup lost the path to its stop boundary".into(),
983                )
984            })?;
985            if next.epoch < stop.epoch || next.checkpoint_id < stop.checkpoint_id {
986                return Err(LeaseError::Invalid(
987                    "cluster artifact cleanup next checkpoint crosses its stop boundary".into(),
988                ));
989            }
990        }
991        if self.participant_ids.is_empty()
992            || self.participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
993            || self.participant_ids[0] == 0
994            || !self
995                .participant_ids
996                .windows(2)
997                .all(|pair| pair[0] < pair[1])
998        {
999            return Err(LeaseError::Invalid(
1000                "cluster artifact cleanup participants are not canonical".into(),
1001            ));
1002        }
1003        Ok(())
1004    }
1005}
1006
1007#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1008#[serde(deny_unknown_fields)]
1009struct AuthorityOutcomeFloor {
1010    deployment_id: String,
1011    artifact_before_epoch: u64,
1012    authority_before_epoch: u64,
1013    terminal_anchor: Option<CheckpointOutcome>,
1014    terminal_anchor_link: Option<OutcomeLink>,
1015    committed_anchor: Option<CheckpointOutcome>,
1016    committed_anchor_link: Option<OutcomeLink>,
1017}
1018
1019#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1020#[serde(deny_unknown_fields)]
1021struct AuthorityAssignmentDecisionFloor {
1022    before_target_version: u64,
1023    terminal_anchor: Option<AuthorityAssignmentDecision>,
1024    terminal_anchor_link: Option<AssignmentDecisionLink>,
1025}
1026
1027#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1028#[serde(deny_unknown_fields)]
1029struct AssignmentHandoffPin {
1030    target: CheckpointAssignmentFence,
1031    checkpoint: CommittedCheckpointRef,
1032}
1033
1034impl AssignmentHandoffPin {
1035    fn validate(&self) -> Result<(), LeaseError> {
1036        if !self.target.is_canonical() {
1037            return Err(LeaseError::Invalid(
1038                "assignment handoff pin has an invalid target fence".into(),
1039            ));
1040        }
1041        self.checkpoint.validate().map_err(LeaseError::Invalid)
1042    }
1043}
1044
1045/// One immutable entry in the cluster's shared leadership and checkpoint-decision sequence.
1046#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1047#[serde(deny_unknown_fields)]
1048struct LeaderAuthorityRecord {
1049    version: u32,
1050    lease: LeaderLease,
1051    /// Present only on the sequence that admitted this terminal outcome.
1052    checkpoint_outcome: Option<CheckpointOutcome>,
1053    /// Link to the preceding admitted outcome, present only on an outcome-bearing record.
1054    previous_outcome: Option<OutcomeLink>,
1055    /// Latest admitted outcome. Renewals, takeovers, catalog seals, and floor advances preserve it.
1056    outcome_head: Option<OutcomeLink>,
1057    /// Link to the preceding admitted Commit, present only on a Commit-bearing record.
1058    previous_commit: Option<OutcomeLink>,
1059    /// Latest admitted Commit. Every authority mutation preserves it; Abort does not advance it.
1060    commit_head: Option<OutcomeLink>,
1061    /// Monotonic cluster outcome retention boundary and its continuity anchors.
1062    outcome_floor: Option<AuthorityOutcomeFloor>,
1063    /// Exact leader-fenced artifact cleanup position, preserved across leadership terms.
1064    artifact_cleanup: Option<ClusterArtifactCleanupCursor>,
1065    /// Latest replay-retention floor linearized through this authority sequence.
1066    #[serde(default, skip_serializing_if = "Option::is_none")]
1067    subscription_cleanup_commit: Option<subscription_replay::SubscriptionCleanupCommit>,
1068    /// Exact unresolved checkpoint attempt whose artifacts require a terminal decision or cleanup.
1069    active_checkpoint_artifacts: Option<CheckpointArtifactInventory>,
1070    /// Leader term that admitted `active_checkpoint_artifacts` before any artifact write.
1071    active_checkpoint_artifact_leader_proof: Option<LeaderProof>,
1072    /// Present only on the sequence that admitted an assignment decision.
1073    assignment_decision: Option<AuthorityAssignmentDecision>,
1074    /// Link to the preceding assignment decision, present only on a decision-bearing record.
1075    previous_assignment_decision: Option<AssignmentDecisionLink>,
1076    /// Latest admitted assignment decision. Every other authority mutation preserves it.
1077    assignment_decision_head: Option<AssignmentDecisionLink>,
1078    /// Monotonic assignment-decision retention boundary and its continuity anchor.
1079    assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
1080    /// State handoff cut retained until a complete checkpoint commits under its target fence.
1081    assignment_handoff_pin: Option<AssignmentHandoffPin>,
1082    /// Authority sequence that admitted the current recovery-fault slot state.
1083    recovery_fault_revision: u64,
1084    /// One active or tombstoned request identity per stable node.
1085    recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
1086    /// Present only on the sequence that admitted the latest recovery release.
1087    recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
1088    /// Latest admitted recovery release, preserved by every later authority mutation.
1089    recovery_release_head: Option<RecoveryReleaseLink>,
1090    /// Explicit baseline, present only after the coordinated authority-format upgrade.
1091    #[serde(default, skip_serializing_if = "Option::is_none")]
1092    topology_baseline: Option<super::topology::LegacyTopologyBaseline>,
1093    /// Frozen drain intent, admitted before any assignment snapshot publication.
1094    #[serde(default, skip_serializing_if = "Option::is_none")]
1095    assignment_drain_reservation: Option<AssignmentDrainReservation>,
1096    /// Bounded payload-bound pre-cut request journal, preserved by every authority append.
1097    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1098    topology_operations: Vec<super::topology::TopologyAdmissionStatus>,
1099}
1100
1101#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1102#[serde(deny_unknown_fields)]
1103struct AssignmentDrainReservation {
1104    proposal: AssignmentSnapshotRef,
1105    transition: AssignmentDrainTransition,
1106    authority_sequence: u64,
1107}
1108
1109impl LeaderLease {
1110    fn validate(&self) -> Result<(), LeaseError> {
1111        self.owner.validate()?;
1112        if self.seq == 0
1113            || self.renewal_sequence == 0
1114            || self.renewal_sequence > self.seq
1115            || self.token == 0
1116        {
1117            return Err(LeaseError::Invalid(
1118                "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
1119                    .into(),
1120            ));
1121        }
1122        if let Some(reference) = &self.catalog_manifest {
1123            reference
1124                .validate()
1125                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1126        }
1127        Ok(())
1128    }
1129
1130    /// Exact feature-neutral proof for this ownership term.
1131    #[must_use]
1132    pub fn proof(&self) -> LeaderProof {
1133        LeaderProof {
1134            owner: self.owner.proof_owner(),
1135            fencing_token: self.token,
1136        }
1137    }
1138
1139    /// Whether `proof` names this exact owner and fencing token.
1140    #[must_use]
1141    pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
1142        proof.is_canonical()
1143            && proof.owner == self.owner.proof_owner()
1144            && proof.fencing_token == self.token
1145    }
1146
1147    fn has_same_liveness_identity(&self, other: &Self) -> bool {
1148        self.owner == other.owner
1149            && self.token == other.token
1150            && self.renewal_sequence == other.renewal_sequence
1151    }
1152}
1153
1154impl AuthorityOutcomeFloor {
1155    fn validate(&self) -> Result<(), LeaseError> {
1156        let deployment = Uuid::parse_str(&self.deployment_id)
1157            .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
1158        if deployment.is_nil()
1159            || deployment.to_string() != self.deployment_id
1160            || self.authority_before_epoch == 0
1161            || self.artifact_before_epoch > self.authority_before_epoch
1162        {
1163            return Err(LeaseError::Invalid(
1164                "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
1165                    .into(),
1166            ));
1167        }
1168        for link in [self.terminal_anchor_link, self.committed_anchor_link]
1169            .into_iter()
1170            .flatten()
1171        {
1172            link.validate()?;
1173        }
1174        for (name, anchor) in [
1175            ("terminal", self.terminal_anchor.as_ref()),
1176            ("committed", self.committed_anchor.as_ref()),
1177        ] {
1178            let Some(anchor) = anchor else { continue };
1179            anchor
1180                .validate_shape(anchor.epoch)
1181                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1182            if anchor.scope != CheckpointScope::Cluster
1183                || anchor.deployment_id != self.deployment_id
1184            {
1185                return Err(LeaseError::Invalid(format!(
1186                    "outcome floor has an invalid {name} anchor"
1187                )));
1188            }
1189        }
1190        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1191            (Some(anchor), Some(link))
1192                if anchor.epoch < self.authority_before_epoch
1193                    && link.sequence != 0
1194                    && link.epoch == anchor.epoch
1195                    && link.checkpoint_id == anchor.checkpoint_id => {}
1196            (None, None) => {}
1197            _ => {
1198                return Err(LeaseError::Invalid(
1199                    "outcome floor terminal anchor does not match its exact authority link".into(),
1200                ));
1201            }
1202        }
1203        match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
1204            (Some(anchor), Some(link))
1205                if self.artifact_before_epoch != 0
1206                    && anchor.is_commit()
1207                    && anchor.epoch < self.artifact_before_epoch
1208                    && link.sequence != 0
1209                    && link.epoch == anchor.epoch
1210                    && link.checkpoint_id == anchor.checkpoint_id => {}
1211            (None, None) => {}
1212            _ => {
1213                return Err(LeaseError::Invalid(
1214                    "outcome floor committed anchor does not match its exact Commit link".into(),
1215                ));
1216            }
1217        }
1218        if let (Some(committed), Some(terminal)) = (
1219            self.committed_anchor.as_ref(),
1220            self.terminal_anchor.as_ref(),
1221        ) {
1222            let ordered = if committed.epoch == terminal.epoch {
1223                committed == terminal
1224            } else {
1225                committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
1226            };
1227            if !ordered {
1228                return Err(LeaseError::Invalid(
1229                    "outcome floor anchors are not monotonically ordered".into(),
1230                ));
1231            }
1232            let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
1233                (Some(committed_link), Some(terminal_link)) if committed == terminal => {
1234                    committed_link == terminal_link
1235                }
1236                (Some(committed_link), Some(terminal_link)) => {
1237                    committed_link.sequence < terminal_link.sequence
1238                }
1239                _ => false,
1240            };
1241            if !links_ordered {
1242                return Err(LeaseError::Invalid(
1243                    "outcome floor anchor links are not ordered with their exact outcomes".into(),
1244                ));
1245            }
1246        } else if self.committed_anchor.is_some() {
1247            return Err(LeaseError::Invalid(
1248                "outcome floor has a committed anchor without terminal continuity".into(),
1249            ));
1250        }
1251        Ok(())
1252    }
1253}
1254
1255impl AuthorityAssignmentDecisionFloor {
1256    fn validate(&self) -> Result<(), LeaseError> {
1257        if self.before_target_version == 0 {
1258            return Err(LeaseError::Invalid(
1259                "assignment decision floor requires a nonzero target version".into(),
1260            ));
1261        }
1262        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1263            (Some(anchor), Some(link)) => {
1264                anchor.validate()?;
1265                if link.sequence == 0
1266                    || link.target_version != anchor.target_version()
1267                    || anchor.target_version() >= self.before_target_version
1268                {
1269                    return Err(LeaseError::Invalid(
1270                        "assignment decision floor anchor does not match its exact authority link"
1271                            .into(),
1272                    ));
1273                }
1274            }
1275            (None, None) => {}
1276            _ => {
1277                return Err(LeaseError::Invalid(
1278                    "assignment decision floor has an incomplete terminal anchor".into(),
1279                ));
1280            }
1281        }
1282        Ok(())
1283    }
1284}
1285
1286impl LeaderAuthorityRecord {
1287    fn initial(lease: LeaderLease) -> Self {
1288        Self {
1289            version: AUTHORITY_RECORD_VERSION,
1290            lease,
1291            checkpoint_outcome: None,
1292            previous_outcome: None,
1293            outcome_head: None,
1294            previous_commit: None,
1295            commit_head: None,
1296            outcome_floor: None,
1297            artifact_cleanup: None,
1298            subscription_cleanup_commit: None,
1299            active_checkpoint_artifacts: None,
1300            active_checkpoint_artifact_leader_proof: None,
1301            assignment_decision: None,
1302            previous_assignment_decision: None,
1303            assignment_decision_head: None,
1304            assignment_decision_floor: None,
1305            assignment_handoff_pin: None,
1306            recovery_fault_revision: 0,
1307            recovery_fault_slots: Vec::new(),
1308            recovery_release_commit: None,
1309            recovery_release_head: None,
1310            topology_baseline: None,
1311            assignment_drain_reservation: None,
1312            topology_operations: Vec::new(),
1313        }
1314    }
1315
1316    fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1317        let leader_changed = self.lease.owner != lease.owner || self.lease.token != lease.token;
1318        let mut preserved = Self {
1319            version: self.version,
1320            lease,
1321            checkpoint_outcome: None,
1322            previous_outcome: None,
1323            outcome_head: self.outcome_head,
1324            previous_commit: None,
1325            commit_head: self.commit_head,
1326            outcome_floor: self.outcome_floor.clone(),
1327            artifact_cleanup: self.artifact_cleanup.clone(),
1328            subscription_cleanup_commit: self.subscription_cleanup_commit.clone(),
1329            active_checkpoint_artifacts: self.active_checkpoint_artifacts.clone(),
1330            active_checkpoint_artifact_leader_proof: self
1331                .active_checkpoint_artifact_leader_proof
1332                .clone(),
1333            assignment_decision: None,
1334            previous_assignment_decision: None,
1335            assignment_decision_head: self.assignment_decision_head,
1336            assignment_decision_floor: self.assignment_decision_floor.clone(),
1337            assignment_handoff_pin: self.assignment_handoff_pin.clone(),
1338            recovery_fault_revision: self.recovery_fault_revision,
1339            recovery_fault_slots: self.recovery_fault_slots.clone(),
1340            recovery_release_commit: None,
1341            recovery_release_head: self.recovery_release_head.clone(),
1342            topology_baseline: self.topology_baseline.clone(),
1343            assignment_drain_reservation: self.assignment_drain_reservation.clone(),
1344            topology_operations: self.topology_operations.clone(),
1345        };
1346        if leader_changed {
1347            preserved
1348                .abort_topology_preparation(super::topology::TopologyAbortReason::LeaderChanged);
1349        }
1350        preserved
1351    }
1352
1353    fn validate(&self) -> Result<(), LeaseError> {
1354        if self.version != AUTHORITY_RECORD_VERSION
1355            && self.version != TOPOLOGY_AUTHORITY_RECORD_VERSION
1356            && self.version != TOPOLOGY_ADMISSION_RECORD_VERSION
1357            && self.version != TOPOLOGY_CUT_RECORD_VERSION
1358            && self.version != TOPOLOGY_PREPARATION_RECORD_VERSION
1359            && self.version != TOPOLOGY_MIGRATION_ROOT_RECORD_VERSION
1360            && self.version != TOPOLOGY_SOURCE_ROOT_RECORD_VERSION
1361            && self.version != TOPOLOGY_TARGET_PREPARATION_RECORD_VERSION
1362            && self.version != TOPOLOGY_COMMIT_RECORD_VERSION
1363            && self.version != TOPOLOGY_INSTALLATION_RECORD_VERSION
1364            && self.version != TOPOLOGY_RECOVERY_RECORD_VERSION
1365            && self.version != TOPOLOGY_SUBMISSION_RECORD_VERSION
1366        {
1367            return Err(LeaseError::Invalid(format!(
1368                "authority record version {} is unsupported",
1369                self.version
1370            )));
1371        }
1372        self.lease.validate()?;
1373        self.validate_topology_baseline()?;
1374        self.validate_topology_admission()?;
1375        for link in [
1376            self.previous_outcome,
1377            self.outcome_head,
1378            self.previous_commit,
1379            self.commit_head,
1380        ]
1381        .into_iter()
1382        .flatten()
1383        {
1384            link.validate()?;
1385        }
1386        self.subscription_cleanup_commit
1387            .iter()
1388            .try_for_each(subscription_replay::SubscriptionCleanupCommit::validate)?;
1389        self.validate_recovery_fault_inventory()?;
1390        self.validate_checkpoint_outcome_chain()?;
1391        self.validate_outcome_floor()?;
1392        self.validate_artifact_cleanup()?;
1393        self.validate_checkpoint_artifact_inventory()?;
1394        self.validate_assignment_decision_chain()?;
1395        self.validate_assignment_decision_floor()?;
1396        self.validate_assignment_handoff_pin()?;
1397        self.validate_recovery_release()?;
1398        Ok(())
1399    }
1400
1401    fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1402        if self.recovery_fault_slots.windows(2).any(|pair| {
1403            pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1404        }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1405            || (self.recovery_fault_revision != 0
1406                && self.recovery_fault_slots.is_empty()
1407                && self
1408                    .recovery_release_head
1409                    .as_ref()
1410                    .map(|head| head.sequence)
1411                    != Some(self.recovery_fault_revision))
1412            || self.recovery_fault_revision > self.lease.seq
1413            || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1414        {
1415            return Err(LeaseError::Invalid(
1416                "leader authority recovery-fault inventory is not canonical".into(),
1417            ));
1418        }
1419        let mut fault_sequences = BTreeSet::new();
1420        for slot in &self.recovery_fault_slots {
1421            slot.validate()?;
1422            if slot.fault_sequence > self.recovery_fault_revision
1423                || slot.fault_sequence > self.lease.seq
1424                || !fault_sequences.insert(slot.fault_sequence)
1425            {
1426                return Err(LeaseError::Invalid(
1427                    "leader authority recovery-fault sequences are not canonical".into(),
1428                ));
1429            }
1430        }
1431        if [
1432            self.checkpoint_outcome.is_some(),
1433            self.assignment_decision.is_some(),
1434            self.recovery_release_commit.is_some(),
1435        ]
1436        .into_iter()
1437        .filter(|present| *present)
1438        .count()
1439            > 1
1440        {
1441            return Err(LeaseError::Invalid(
1442                "one authority sequence cannot admit two terminal domains".into(),
1443            ));
1444        }
1445        Ok(())
1446    }
1447
1448    fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1449        if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1450            outcome
1451                .validate_shape(outcome.epoch)
1452                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1453            if outcome.scope != CheckpointScope::Cluster {
1454                return Err(LeaseError::Invalid(
1455                    "leader authority can only admit cluster checkpoint outcomes".into(),
1456                ));
1457            }
1458            let proof = outcome
1459                .leader_proof
1460                .as_ref()
1461                .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1462            let current_link = OutcomeLink {
1463                sequence: self.lease.seq,
1464                epoch: outcome.epoch,
1465                checkpoint_id: outcome.checkpoint_id,
1466            };
1467            if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1468                return Err(LeaseError::Invalid(
1469                    "cluster outcome is not bound to its exact authority sequence and term".into(),
1470                ));
1471            }
1472            if let Some(previous) = self.previous_outcome {
1473                if previous.sequence >= self.lease.seq
1474                    || previous.epoch >= outcome.epoch
1475                    || previous.checkpoint_id >= outcome.checkpoint_id
1476                {
1477                    return Err(LeaseError::Invalid(
1478                        "cluster outcome link does not move backward in sequence and epoch".into(),
1479                    ));
1480                }
1481            }
1482            if outcome.is_commit() {
1483                if self.commit_head != Some(current_link) {
1484                    return Err(LeaseError::Invalid(
1485                        "cluster Commit is not bound to its exact Commit-chain head".into(),
1486                    ));
1487                }
1488                if let Some(previous) = self.previous_commit {
1489                    if previous.sequence >= self.lease.seq
1490                        || previous.epoch >= outcome.epoch
1491                        || previous.checkpoint_id >= outcome.checkpoint_id
1492                    {
1493                        return Err(LeaseError::Invalid(
1494                            "cluster Commit link does not move backward in sequence and epoch"
1495                                .into(),
1496                        ));
1497                    }
1498                    if self.outcome_floor.as_ref().is_some_and(|floor| {
1499                        previous.epoch < floor.artifact_before_epoch
1500                            && Some(previous) != floor.committed_anchor_link
1501                    }) {
1502                        return Err(LeaseError::Invalid(
1503                            "cluster Commit link crosses the artifact floor without its exact anchor"
1504                                .into(),
1505                        ));
1506                    }
1507                }
1508            } else if self.previous_commit.is_some() {
1509                return Err(LeaseError::Invalid(
1510                    "cluster Abort carries a previous-Commit link".into(),
1511                ));
1512            }
1513            if let Some(floor) = &self.outcome_floor {
1514                if outcome.deployment_id != floor.deployment_id
1515                    || outcome.epoch < floor.authority_before_epoch
1516                {
1517                    return Err(LeaseError::Invalid(
1518                        "cluster outcome is below or outside its durable authority floor".into(),
1519                    ));
1520                }
1521            }
1522        } else {
1523            if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1524                return Err(LeaseError::Invalid(
1525                    "non-outcome authority record carries an outcome-chain link".into(),
1526                ));
1527            }
1528            if self.outcome_head.is_some_and(|head| {
1529                head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1530            }) {
1531                return Err(LeaseError::Invalid(
1532                    "authority outcome head is outside the durable sequence".into(),
1533                ));
1534            }
1535        }
1536        if self.commit_head.is_some_and(|head| {
1537            head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1538        }) {
1539            return Err(LeaseError::Invalid(
1540                "authority Commit head is outside the durable sequence".into(),
1541            ));
1542        }
1543        match (self.commit_head, self.outcome_head) {
1544            (Some(commit), Some(terminal)) => {
1545                let equal = commit == terminal;
1546                let strictly_older = commit.sequence < terminal.sequence
1547                    && commit.epoch < terminal.epoch
1548                    && commit.checkpoint_id < terminal.checkpoint_id;
1549                if !equal && !strictly_older {
1550                    return Err(LeaseError::Invalid(
1551                        "authority Commit head is not ordered behind the terminal head".into(),
1552                    ));
1553                }
1554                if self
1555                    .checkpoint_outcome
1556                    .as_ref()
1557                    .is_some_and(|outcome| !outcome.is_commit())
1558                    && !strictly_older
1559                {
1560                    return Err(LeaseError::Invalid(
1561                        "cluster Abort did not preserve a strictly older Commit head".into(),
1562                    ));
1563                }
1564            }
1565            (Some(_), None) => {
1566                return Err(LeaseError::Invalid(
1567                    "authority Commit head exists without terminal continuity".into(),
1568                ));
1569            }
1570            (None, _) => {}
1571        }
1572        Ok(())
1573    }
1574
1575    fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1576        if let Some(floor) = self.outcome_floor.as_ref() {
1577            floor.validate()?;
1578            if floor
1579                .terminal_anchor_link
1580                .is_some_and(|link| link.sequence >= self.lease.seq)
1581            {
1582                return Err(LeaseError::Invalid(
1583                    "outcome floor anchor is outside the authority sequence".into(),
1584                ));
1585            }
1586            if floor
1587                .committed_anchor_link
1588                .is_some_and(|link| link.sequence >= self.lease.seq)
1589            {
1590                return Err(LeaseError::Invalid(
1591                    "outcome floor Commit anchor is outside the authority sequence".into(),
1592                ));
1593            }
1594            if let Some(head) = self.outcome_head {
1595                if head.epoch < floor.authority_before_epoch
1596                    && Some(head) != floor.terminal_anchor_link
1597                {
1598                    return Err(LeaseError::Invalid(
1599                        "authority outcome head does not meet its durable compaction floor".into(),
1600                    ));
1601                }
1602            } else if floor.terminal_anchor_link.is_some() {
1603                return Err(LeaseError::Invalid(
1604                    "outcome floor anchor is disconnected from the authority head".into(),
1605                ));
1606            }
1607            if let Some(head) = self.commit_head {
1608                if head.epoch < floor.artifact_before_epoch
1609                    && Some(head) != floor.committed_anchor_link
1610                {
1611                    return Err(LeaseError::Invalid(
1612                        "authority Commit head does not meet its durable artifact floor".into(),
1613                    ));
1614                }
1615            } else if floor.committed_anchor_link.is_some() {
1616                return Err(LeaseError::Invalid(
1617                    "outcome floor Commit anchor is disconnected from the Commit head".into(),
1618                ));
1619            }
1620        }
1621        Ok(())
1622    }
1623
1624    fn validate_artifact_cleanup(&self) -> Result<(), LeaseError> {
1625        let Some(cursor) = self.artifact_cleanup.as_ref() else {
1626            return Ok(());
1627        };
1628        cursor.validate()?;
1629        let floor = self.outcome_floor.as_ref().ok_or_else(|| {
1630            LeaseError::Invalid(
1631                "cluster artifact cleanup exists without an outcome retention floor".into(),
1632            )
1633        })?;
1634        if floor.artifact_before_epoch != cursor.protected.epoch
1635            || self
1636                .commit_head
1637                .is_none_or(|head| head.epoch < cursor.protected.epoch)
1638        {
1639            return Err(LeaseError::Invalid(
1640                "cluster artifact cleanup is not bound to its protected retention cut".into(),
1641            ));
1642        }
1643        Ok(())
1644    }
1645
1646    fn validate_checkpoint_artifact_inventory(&self) -> Result<(), LeaseError> {
1647        let (Some(inventory), Some(admitting_proof)) = (
1648            self.active_checkpoint_artifacts.as_ref(),
1649            self.active_checkpoint_artifact_leader_proof.as_ref(),
1650        ) else {
1651            if self.active_checkpoint_artifacts.is_some()
1652                || self.active_checkpoint_artifact_leader_proof.is_some()
1653            {
1654                return Err(LeaseError::Invalid(
1655                    "cluster checkpoint artifact inventory has incomplete leader authority".into(),
1656                ));
1657            }
1658            return Ok(());
1659        };
1660
1661        inventory.validate().map_err(|error| {
1662            LeaseError::Invalid(format!("cluster checkpoint artifact inventory: {error}"))
1663        })?;
1664        let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
1665            LeaseError::Invalid(
1666                "cluster checkpoint artifact inventory has no assignment fence".into(),
1667            )
1668        })?;
1669        if !admitting_proof.is_canonical()
1670            || assignment_fence.participant_incarnation(admitting_proof.owner.node_id)
1671                != Some(admitting_proof.owner.boot_id)
1672        {
1673            return Err(LeaseError::Invalid(
1674                "cluster checkpoint artifact inventory has no canonical admitting leader".into(),
1675            ));
1676        }
1677        if self
1678            .outcome_floor
1679            .as_ref()
1680            .is_some_and(|floor| floor.deployment_id != inventory.deployment_id)
1681        {
1682            return Err(LeaseError::Invalid(
1683                "cluster checkpoint artifact inventory belongs to a foreign deployment".into(),
1684            ));
1685        }
1686        if let Some(head) = self.outcome_head {
1687            let attempt = crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
1688            if matches!(
1689                inventory.attempt.relation_to(attempt),
1690                crate::checkpoint::CheckpointAttemptRelation::Older
1691                    | crate::checkpoint::CheckpointAttemptRelation::Conflict
1692            ) {
1693                return Err(LeaseError::Invalid(
1694                    "cluster checkpoint artifact inventory is behind the terminal outcome head"
1695                        .into(),
1696                ));
1697            }
1698        }
1699        if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1700            let exact = outcome.deployment_id == inventory.deployment_id
1701                && outcome.epoch == inventory.attempt.epoch
1702                && outcome.checkpoint_id == inventory.attempt.checkpoint_id
1703                && outcome.assignment_fence.as_ref() == inventory.assignment_fence.as_ref();
1704            if outcome.is_commit() || !exact {
1705                return Err(LeaseError::Invalid(
1706                    "terminal cluster outcome has inconsistent checkpoint artifact inventory"
1707                        .into(),
1708                ));
1709            }
1710        }
1711        Ok(())
1712    }
1713
1714    fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1715        if let Some(decision) = self.assignment_decision.as_ref() {
1716            decision.validate()?;
1717            if self
1718                .assignment_decision_floor
1719                .as_ref()
1720                .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1721            {
1722                return Err(LeaseError::Invalid(
1723                    "assignment decision is below its durable authority floor".into(),
1724                ));
1725            }
1726            if !self.lease.matches_proof(decision.leader_proof())
1727                || self.assignment_decision_head
1728                    != Some(AssignmentDecisionLink {
1729                        sequence: self.lease.seq,
1730                        target_version: decision.target_version(),
1731                    })
1732            {
1733                return Err(LeaseError::Invalid(
1734                    "assignment decision is not bound to its exact authority sequence and term"
1735                        .into(),
1736                ));
1737            }
1738            if let Some(previous) = self.previous_assignment_decision {
1739                if previous.sequence >= self.lease.seq
1740                    || previous.target_version >= decision.target_version()
1741                {
1742                    return Err(LeaseError::Invalid(
1743                        "assignment decision link does not move backward".into(),
1744                    ));
1745                }
1746            }
1747        } else {
1748            if self.previous_assignment_decision.is_some() {
1749                return Err(LeaseError::Invalid(
1750                    "non-decision authority record carries a previous assignment-decision link"
1751                        .into(),
1752                ));
1753            }
1754            if self
1755                .assignment_decision_head
1756                .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1757            {
1758                return Err(LeaseError::Invalid(
1759                    "authority assignment-decision head is outside the durable sequence".into(),
1760                ));
1761            }
1762        }
1763        Ok(())
1764    }
1765
1766    fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1767        if let Some(floor) = self.assignment_decision_floor.as_ref() {
1768            floor.validate()?;
1769            if floor
1770                .terminal_anchor_link
1771                .is_some_and(|link| link.sequence >= self.lease.seq)
1772            {
1773                return Err(LeaseError::Invalid(
1774                    "assignment decision floor anchor is outside the authority sequence".into(),
1775                ));
1776            }
1777            if let Some(head) = self.assignment_decision_head {
1778                if head.target_version < floor.before_target_version
1779                    && Some(head) != floor.terminal_anchor_link
1780                {
1781                    return Err(LeaseError::Invalid(
1782                        "authority assignment-decision head does not meet its durable floor".into(),
1783                    ));
1784                }
1785            } else if floor.terminal_anchor_link.is_some() {
1786                return Err(LeaseError::Invalid(
1787                    "assignment decision floor anchor is disconnected from the authority head"
1788                        .into(),
1789                ));
1790            }
1791        }
1792        Ok(())
1793    }
1794
1795    fn validate_assignment_handoff_pin(&self) -> Result<(), LeaseError> {
1796        let Some(pin) = self.assignment_handoff_pin.as_ref() else {
1797            return Ok(());
1798        };
1799        pin.validate()?;
1800        if self.commit_head.is_none_or(|head| {
1801            head.epoch != pin.checkpoint.epoch || head.checkpoint_id != pin.checkpoint.checkpoint_id
1802        }) || self
1803            .outcome_floor
1804            .as_ref()
1805            .is_some_and(|floor| floor.artifact_before_epoch > pin.checkpoint.epoch)
1806            || self
1807                .artifact_cleanup
1808                .as_ref()
1809                .is_some_and(|cursor| cursor.protected.epoch > pin.checkpoint.epoch)
1810        {
1811            return Err(LeaseError::Invalid(
1812                "assignment handoff pin is outside its live checkpoint retention range".into(),
1813            ));
1814        }
1815        Ok(())
1816    }
1817
1818    fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1819        match self.recovery_release_commit.as_ref() {
1820            Some(commit) => {
1821                commit.validate()?;
1822                let expected = RecoveryReleaseLink {
1823                    sequence: self.lease.seq,
1824                    terminal: commit.terminal.clone(),
1825                };
1826                if !self.lease.matches_proof(&commit.leader_proof)
1827                    || self.recovery_release_head.as_ref() != Some(&expected)
1828                    || self.recovery_fault_revision != self.lease.seq
1829                    || self.recovery_fault_slots.iter().any(|slot| {
1830                        slot.active || slot.disposition == RecoveryFaultDisposition::Terminal
1831                    })
1832                {
1833                    return Err(LeaseError::Invalid(
1834                        "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1835                            .into(),
1836                    ));
1837                }
1838            }
1839            None => {
1840                if let Some(head) = self.recovery_release_head.as_ref() {
1841                    head.validate()?;
1842                    if head.sequence >= self.lease.seq {
1843                        return Err(LeaseError::Invalid(
1844                            "recovery release head is outside the durable authority sequence"
1845                                .into(),
1846                        ));
1847                    }
1848                    if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1849                        && self.recovery_fault_revision != head.sequence
1850                    {
1851                        return Err(LeaseError::Invalid(
1852                            "settled recovery fault inventory is not bound to its release head"
1853                                .into(),
1854                        ));
1855                    }
1856                }
1857            }
1858        }
1859        Ok(())
1860    }
1861}
1862
1863/// Result of an acquisition or renewal attempt.
1864#[derive(Debug, Clone, PartialEq, Eq)]
1865pub enum LeaseOutcome {
1866    /// The caller's exact process incarnation owns the returned record.
1867    Acquired(LeaderLease),
1868    /// A rival exact process incarnation owns the returned record.
1869    Held(LeaderLease),
1870}
1871
1872enum AuthorityCreateOutcome {
1873    Created,
1874    ExistingIdentical,
1875    Contended(Box<LeaderAuthorityRecord>),
1876}
1877
1878#[derive(Clone, Copy)]
1879enum SameOwnerToken {
1880    #[cfg(test)]
1881    Preserve,
1882    Rotate,
1883    Exact(u64),
1884}
1885
1886/// Candidate-local proof that one rival liveness identity remained current for a full TTL.
1887#[derive(Debug)]
1888pub struct LeaderLeaseObservation {
1889    lease: LeaderLease,
1890    started: Instant,
1891}
1892
1893/// Coalescing-safe local candidacy state. A generation change means a published grant was lost
1894/// and must never reuse its fencing token, even when the current state is eligible again.
1895#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1896pub struct LeaderCandidacy {
1897    eligible: bool,
1898    generation: u64,
1899}
1900
1901impl LeaderCandidacy {
1902    pub(crate) const fn initial(eligible: bool) -> Self {
1903        Self {
1904            eligible,
1905            generation: 1,
1906        }
1907    }
1908
1909    pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1910        let generation = if self.eligible && !eligible {
1911            self.generation.checked_add(1)?
1912        } else {
1913            self.generation
1914        };
1915        Some(Self {
1916            eligible,
1917            generation,
1918        })
1919    }
1920
1921    pub(crate) const fn terminal() -> Self {
1922        Self {
1923            eligible: false,
1924            generation: u64::MAX,
1925        }
1926    }
1927
1928    /// Whether this process is currently eligible to contend for the leader lease.
1929    #[must_use]
1930    pub const fn is_eligible(self) -> bool {
1931        self.eligible
1932    }
1933}
1934
1935/// Leader lease storage or validation failure.
1936#[derive(Debug, thiserror::Error)]
1937pub enum LeaseError {
1938    /// Underlying object-store failure.
1939    #[error("object store I/O: {0}")]
1940    Io(String),
1941    /// Malformed configuration, owner, or durable record.
1942    #[error("invalid leader lease: {0}")]
1943    Invalid(String),
1944    /// A newer durable authority term superseded an exact renewal.
1945    #[error("leader lease fenced: {0}")]
1946    Fenced(String),
1947    /// JSON encoding or decoding failure.
1948    #[error("JSON: {0}")]
1949    Json(#[from] serde_json::Error),
1950}
1951
1952/// Failure while using the leader sequence as the cluster checkpoint-decision authority.
1953#[derive(Debug, thiserror::Error)]
1954pub enum ClusterCheckpointAuthorityError {
1955    /// Cluster runtime did not wire the durable leader authority.
1956    #[error("cluster checkpoint authority is not installed")]
1957    NotConfigured,
1958    /// Shared append-only authority failed.
1959    #[error("leader authority: {0}")]
1960    Authority(#[from] LeaseError),
1961    /// Checkpoint metadata or content-addressed recovery state was invalid.
1962    #[error("checkpoint decision: {0}")]
1963    Decision(#[from] DecisionError),
1964    /// The supplied proof no longer names the exact durable leader term.
1965    #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1966    Fenced,
1967}
1968
1969/// Exact continuity retained when old cluster checkpoint outcomes are compacted.
1970#[derive(Debug, Clone, PartialEq, Eq)]
1971pub struct ClusterOutcomeRetentionBoundary {
1972    /// Checkpoint artifacts below this epoch are eligible for garbage collection.
1973    pub artifact_before_epoch: u64,
1974    /// Terminal outcome links below this epoch have been compacted.
1975    pub terminal_before_epoch: u64,
1976    /// Greatest committed outcome below the artifact-retention horizon.
1977    pub committed_anchor: Option<CheckpointOutcome>,
1978    /// Greatest terminal outcome compacted from the authority chain, including aborts.
1979    pub terminal_anchor: Option<CheckpointOutcome>,
1980}
1981
1982impl ClusterOutcomeRetentionBoundary {
1983    fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1984        floor.map_or(
1985            Self {
1986                artifact_before_epoch: 0,
1987                terminal_before_epoch: 0,
1988                committed_anchor: None,
1989                terminal_anchor: None,
1990            },
1991            |floor| Self {
1992                artifact_before_epoch: floor.artifact_before_epoch,
1993                terminal_before_epoch: floor.authority_before_epoch,
1994                committed_anchor: floor.committed_anchor.clone(),
1995                terminal_anchor: floor.terminal_anchor.clone(),
1996            },
1997        )
1998    }
1999}
2000
2001/// Live cluster outcomes and their retention horizons from one audited authority head.
2002#[derive(Debug, Clone, PartialEq, Eq)]
2003pub struct ClusterOutcomeInventory {
2004    /// Outcomes at or above the artifact-retention horizon, in ascending epoch order.
2005    pub outcomes: Vec<CheckpointOutcome>,
2006    /// Artifact and terminal-history horizons paired with `outcomes`.
2007    pub retention_boundary: ClusterOutcomeRetentionBoundary,
2008}
2009
2010/// Append-only object-store authority for the cluster leader.
2011pub struct LeaderLeaseStore {
2012    store: Arc<dyn ObjectStore>,
2013    ttl_ms: i64,
2014    prune_running: Arc<AtomicBool>,
2015    outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
2016    outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
2017}
2018
2019#[derive(Clone, PartialEq, Eq)]
2020struct ClusterOutcomeAuditKey {
2021    terminal_head: Option<OutcomeLink>,
2022    commit_head: Option<OutcomeLink>,
2023    floor: Option<AuthorityOutcomeFloor>,
2024}
2025
2026struct ClusterOutcomeAuditCache {
2027    key: ClusterOutcomeAuditKey,
2028    authority_sequence: u64,
2029    snapshot: ClusterOutcomeAuditSnapshot,
2030}
2031
2032#[derive(Clone)]
2033struct ClusterOutcomeAuditSnapshot {
2034    outcomes: Arc<[CheckpointOutcome]>,
2035    terminal_links: Arc<[OutcomeLink]>,
2036    commit_links: Arc<[OutcomeLink]>,
2037}
2038
2039struct ClusterOutcomeAuditFlight {
2040    key: ClusterOutcomeAuditKey,
2041    gate: Weak<tokio::sync::Mutex<()>>,
2042}
2043
2044struct PruneLatchGuard(Arc<AtomicBool>);
2045
2046impl Drop for PruneLatchGuard {
2047    fn drop(&mut self) {
2048        self.0.store(false, Ordering::Release);
2049    }
2050}
2051
2052impl std::fmt::Debug for LeaderLeaseStore {
2053    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2054        formatter
2055            .debug_struct("LeaderLeaseStore")
2056            .field("ttl_ms", &self.ttl_ms)
2057            .finish_non_exhaustive()
2058    }
2059}
2060
2061impl LeaderLeaseStore {
2062    /// Create a leader lease authority.
2063    ///
2064    /// The store must provide linearizable `PutMode::Create`/`Update` and GET `ETag` or version
2065    /// metadata; unsupported conditional updates fail closed.
2066    #[must_use]
2067    pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
2068        Self {
2069            store,
2070            ttl_ms,
2071            prune_running: Arc::new(AtomicBool::new(false)),
2072            outcome_audit_cache: Mutex::new(None),
2073            outcome_audit_flights: Mutex::new(Vec::new()),
2074        }
2075    }
2076
2077    /// Verify the conditional-write contract required by the mutable authority head.
2078    ///
2079    /// Validation and cleanup each receive the supplied timeout. The probe uses a unique path and
2080    /// cleanup is attempted after every validation outcome.
2081    ///
2082    /// # Errors
2083    /// Returns an error when the store does not enforce native conditional writes, omits update
2084    /// metadata, times out, or cannot remove the probe.
2085    pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
2086        probe_object_store_conditional_update(
2087            self.store.as_ref(),
2088            STORE_CONTRACT_PROBE_PREFIX,
2089            timeout,
2090        )
2091        .await
2092        .map_err(|error| match error {
2093            CheckpointStoreError::Invalid(message) => LeaseError::Invalid(message),
2094            error => LeaseError::Io(error.to_string()),
2095        })
2096    }
2097
2098    fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
2099        RecoveryFaultInventory {
2100            revision: record.recovery_fault_revision,
2101            faults: record
2102                .recovery_fault_slots
2103                .iter()
2104                .filter(|slot| slot.active)
2105                .map(AuthorityRecoveryFaultSlot::fault)
2106                .collect(),
2107        }
2108    }
2109
2110    pub(crate) async fn recovery_fault_inventory(
2111        &self,
2112    ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
2113        let record = self
2114            .load_record()
2115            .await?
2116            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2117        Ok(Self::recovery_fault_inventory_from(&record))
2118    }
2119
2120    #[cfg(test)]
2121    pub(crate) async fn record_recovery_fault(
2122        &self,
2123        publisher: RecoveryFaultPublisher,
2124        request_sequence: u64,
2125    ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2126        self.record_recovery_fault_with_disposition(
2127            publisher,
2128            request_sequence,
2129            RecoveryFaultDisposition::Recoverable,
2130        )
2131        .await
2132    }
2133
2134    pub(crate) async fn record_recovery_fault_with_disposition(
2135        &self,
2136        publisher: RecoveryFaultPublisher,
2137        request_sequence: u64,
2138        disposition: RecoveryFaultDisposition,
2139    ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2140        publisher.validate().map_err(LeaseError::Invalid)?;
2141        if request_sequence == 0 {
2142            return Err(LeaseError::Invalid(
2143                "recovery fault request sequence must be nonzero".into(),
2144            )
2145            .into());
2146        }
2147
2148        loop {
2149            let head = self
2150                .load_published_authority_head()
2151                .await?
2152                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2153            let current = &head.record;
2154            let node_id = publisher.participant.node_id;
2155            let slot_index = current
2156                .recovery_fault_slots
2157                .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
2158            let (insert_at, replace) = match slot_index {
2159                Ok(index) => {
2160                    let slot = &current.recovery_fault_slots[index];
2161                    if slot.matches_request(publisher, request_sequence, disposition) {
2162                        return Ok(if slot.active {
2163                            RecordRecoveryFaultResult::Active
2164                        } else {
2165                            RecordRecoveryFaultResult::AlreadyCleared
2166                        });
2167                    }
2168                    // A terminal slot is an operator-owned cluster fence. Neither a later request
2169                    // nor a replacement process may silently downgrade or tombstone it.
2170                    if slot.disposition == RecoveryFaultDisposition::Terminal {
2171                        return Ok(RecordRecoveryFaultResult::TerminalFenceActive);
2172                    }
2173                    if publisher.process_term < slot.publisher.process_term {
2174                        return Ok(RecordRecoveryFaultResult::Superseded);
2175                    }
2176                    if publisher.process_term == slot.publisher.process_term
2177                        && publisher.participant.boot_incarnation
2178                            != slot.publisher.participant.boot_incarnation
2179                    {
2180                        return Err(LeaseError::Invalid(format!(
2181                            "stable node {node_id} has two recovery fault publishers for process term {}",
2182                            publisher.process_term
2183                        ))
2184                        .into());
2185                    }
2186                    if publisher.process_term == slot.publisher.process_term
2187                        && request_sequence < slot.request_sequence
2188                    {
2189                        return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
2190                    }
2191                    (index, true)
2192                }
2193                Err(index) => (index, false),
2194            };
2195
2196            let sequence =
2197                current.lease.seq.checked_add(1).ok_or_else(|| {
2198                    LeaseError::Invalid("leader authority sequence exhausted".into())
2199                })?;
2200            let mut lease = current.lease.clone();
2201            lease.seq = sequence;
2202            let mut candidate = current.preserve_with_lease(lease);
2203            let slot = AuthorityRecoveryFaultSlot {
2204                publisher,
2205                request_sequence,
2206                // Authority sequence is globally monotonic even after an obsolete tombstone is
2207                // compacted. A delayed retry can therefore become a conservative fresh fault,
2208                // but can never alias a sequence already remembered by a recovery monitor.
2209                fault_sequence: sequence,
2210                disposition,
2211                active: true,
2212            };
2213            if replace {
2214                candidate.recovery_fault_slots[insert_at] = slot;
2215            } else {
2216                candidate.recovery_fault_slots.insert(insert_at, slot);
2217            }
2218            candidate.recovery_fault_revision = sequence;
2219            candidate.abort_topology_preparation(super::topology::TopologyAbortReason::Recovery);
2220            candidate.validate()?;
2221
2222            match self
2223                .create_authority_record(Some(&head), &candidate)
2224                .await?
2225            {
2226                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2227                    return Ok(RecordRecoveryFaultResult::Active);
2228                }
2229                AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
2230            }
2231        }
2232    }
2233
2234    pub(crate) async fn authorize_recovery_release(
2235        &self,
2236        clearer: RecoveryFaultPublisher,
2237        terminal: &RecoveryAnnouncement,
2238    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2239        self.authorize_recovery_release_with_topology(clearer, terminal, None)
2240            .await
2241    }
2242
2243    pub(crate) async fn authorize_recovery_release_with_topology(
2244        &self,
2245        clearer: RecoveryFaultPublisher,
2246        terminal: &RecoveryAnnouncement,
2247        context: Option<(&AssignmentSnapshotStore, &super::ProcessLeaseAuthority)>,
2248    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2249        clearer.validate().map_err(LeaseError::Invalid)?;
2250        let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2251        let reporter = NodeId(clearer.participant.node_id);
2252        let expected = terminal.round.fault_sequence(reporter);
2253        let current = self
2254            .load_record()
2255            .await?
2256            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2257        self.audit_topology_recovery_terminal(&current, terminal, context)
2258            .await?;
2259        if terminal.round.topology_binding().is_some()
2260            && current
2261                .committed_topology_operation()
2262                .is_none_or(|operation| operation.phase != super::TopologyAdmissionPhase::Active)
2263        {
2264            return Ok(false);
2265        }
2266        let Some(release_head) = current.recovery_release_head.as_ref() else {
2267            return Ok(false);
2268        };
2269        if release_head.terminal != terminal_reference
2270            || current.recovery_fault_revision != release_head.sequence
2271        {
2272            return Ok(false);
2273        }
2274        if current
2275            .recovery_fault_slots
2276            .iter()
2277            .any(|slot| slot.active || slot.disposition == RecoveryFaultDisposition::Terminal)
2278        {
2279            return Ok(false);
2280        }
2281        let slot_index = current
2282            .recovery_fault_slots
2283            .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2284        let Some(expected_sequence) = expected else {
2285            return Ok(match slot_index {
2286                Ok(index) => !current.recovery_fault_slots[index].active,
2287                Err(_) => true,
2288            });
2289        };
2290        let Ok(index) = slot_index else {
2291            return Ok(false);
2292        };
2293        let slot = &current.recovery_fault_slots[index];
2294        Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2295    }
2296
2297    pub(crate) async fn stage_recovery_release_terminal(
2298        &self,
2299        terminal: &RecoveryAnnouncement,
2300    ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2301        let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2302        let path = recovery_release_terminal_path(&reference);
2303        let put_error = match self
2304            .store
2305            .put_opts(
2306                &path,
2307                PutPayload::from(encoded),
2308                PutOptions {
2309                    mode: PutMode::Create,
2310                    ..PutOptions::default()
2311                },
2312            )
2313            .await
2314        {
2315            Ok(_)
2316            | Err(
2317                object_store::Error::AlreadyExists { .. }
2318                | object_store::Error::Precondition { .. },
2319            ) => None,
2320            Err(error) => Some(error),
2321        };
2322        match self.load_recovery_release_terminal(&reference).await {
2323            Ok(stored) if stored == *terminal => Ok(reference),
2324            Ok(_) => Err(LeaseError::Invalid(format!(
2325                "recovery release terminal '{}' differs from the proposed content",
2326                reference.sha256
2327            ))
2328            .into()),
2329            Err(reconcile_error) => {
2330                if let Some(put_error) = put_error {
2331                    Err(LeaseError::Io(format!(
2332                        "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2333                    ))
2334                    .into())
2335                } else {
2336                    Err(reconcile_error)
2337                }
2338            }
2339        }
2340    }
2341
2342    pub(crate) async fn load_recovery_release_terminal(
2343        &self,
2344        reference: &RecoveryReleaseTerminalRef,
2345    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2346        reference.validate()?;
2347        let result = match self
2348            .store
2349            .get(&recovery_release_terminal_path(reference))
2350            .await
2351        {
2352            Ok(result) => result,
2353            Err(object_store::Error::NotFound { .. }) => {
2354                return Err(LeaseError::Invalid(format!(
2355                    "recovery release terminal '{}' is missing",
2356                    reference.sha256
2357                ))
2358                .into());
2359            }
2360            Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2361        };
2362        if result.meta.size != reference.encoded_len {
2363            return Err(LeaseError::Invalid(format!(
2364                "recovery release terminal '{}' is {} bytes, expected {}",
2365                reference.sha256, result.meta.size, reference.encoded_len
2366            ))
2367            .into());
2368        }
2369        let bytes = result
2370            .bytes()
2371            .await
2372            .map_err(|error| LeaseError::Io(error.to_string()))?;
2373        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2374            return Err(LeaseError::Invalid(format!(
2375                "recovery release terminal '{}' payload length changed while reading",
2376                reference.sha256
2377            ))
2378            .into());
2379        }
2380        let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2381            LeaseError::Invalid(format!(
2382                "recovery release terminal '{}': {error}",
2383                reference.sha256
2384            ))
2385        })?;
2386        let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2387        if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2388            return Err(LeaseError::Invalid(format!(
2389                "recovery release terminal '{}' does not match its content-addressed reference",
2390                reference.sha256
2391            ))
2392            .into());
2393        }
2394        Ok(terminal)
2395    }
2396
2397    pub(crate) async fn record_recovery_release_commit(
2398        &self,
2399        proof: &LeaderProof,
2400        reference: RecoveryReleaseTerminalRef,
2401    ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2402        self.record_recovery_release_commit_with_topology(proof, reference, None)
2403            .await
2404    }
2405
2406    pub(crate) async fn record_recovery_release_commit_with_topology(
2407        &self,
2408        proof: &LeaderProof,
2409        reference: RecoveryReleaseTerminalRef,
2410        context: Option<(&AssignmentSnapshotStore, &super::ProcessLeaseAuthority)>,
2411    ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2412        reference.validate()?;
2413        let terminal = self.load_recovery_release_terminal(&reference).await?;
2414        if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2415            return Err(ClusterCheckpointAuthorityError::Fenced);
2416        }
2417
2418        loop {
2419            let published = self
2420                .load_published_authority_head()
2421                .await?
2422                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2423            let current = &published.record;
2424            self.audit_topology_recovery_terminal(current, &terminal, context)
2425                .await?;
2426            if let Some(winner) = current.recovery_release_head.as_ref() {
2427                if winner.terminal == reference
2428                    || winner.terminal.generation() >= reference.generation()
2429                {
2430                    self.load_recovery_release_terminal(&winner.terminal)
2431                        .await?;
2432                    return if winner.terminal == reference {
2433                        Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2434                    } else {
2435                        Ok(RecordRecoveryReleaseCommitResult::Conflict {
2436                            winner: winner.terminal.clone(),
2437                        })
2438                    };
2439                }
2440            }
2441            if !current.lease.matches_proof(proof) {
2442                return Err(ClusterCheckpointAuthorityError::Fenced);
2443            }
2444            let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
2445                return Err(DecisionError::Conflict(
2446                    "recovery terminal must be a committed Release".into(),
2447                )
2448                .into());
2449            };
2450            self.audit_recovery_topology_from(current, &terminal.round, Some(epoch), context)
2451                .await?;
2452            let fault_inventory = Self::recovery_fault_inventory_from(current);
2453            if fault_inventory.revision != terminal.round.fault_revision()
2454                || fault_inventory.faults != terminal.round.faults
2455                || fault_inventory
2456                    .faults
2457                    .iter()
2458                    .copied()
2459                    .any(RecoveryFault::is_terminal)
2460            {
2461                return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2462            }
2463
2464            let base_sequence = current.lease.seq;
2465            let sequence = base_sequence
2466                .checked_add(1)
2467                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2468            let mut candidate = current.preserve_with_lease(LeaderLease {
2469                seq: sequence,
2470                renewal_sequence: current.lease.renewal_sequence,
2471                token: current.lease.token,
2472                owner: current.lease.owner.clone(),
2473                expires_at_ms: current.lease.expires_at_ms,
2474                catalog_manifest: current.lease.catalog_manifest.clone(),
2475            });
2476            if terminal.round.topology_binding().is_some() {
2477                candidate.version = candidate.version.max(TOPOLOGY_RECOVERY_RECORD_VERSION);
2478                Self::release_recovered_topology(&mut candidate, &terminal.round, sequence)?;
2479            }
2480            // Only a process frozen into the stopped roster may consume this terminal. Covered
2481            // unavailable publishers remain fenced and conservatively republish if they return.
2482            candidate.recovery_fault_slots.retain(|slot| {
2483                slot.active
2484                    && terminal
2485                        .round
2486                        .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2487                        == Some(slot.publisher.participant.boot_incarnation)
2488            });
2489            for slot in &mut candidate.recovery_fault_slots {
2490                slot.active = false;
2491            }
2492            candidate.recovery_fault_revision = sequence;
2493            candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2494                terminal: reference.clone(),
2495                leader_proof: proof.clone(),
2496            });
2497            candidate.recovery_release_head = Some(RecoveryReleaseLink {
2498                sequence,
2499                terminal: reference.clone(),
2500            });
2501            candidate.validate()?;
2502
2503            match self
2504                .create_authority_record(Some(&published), &candidate)
2505                .await?
2506            {
2507                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2508                    return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2509                }
2510                AuthorityCreateOutcome::Contended(winner_head) => {
2511                    if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2512                        if winner.terminal == reference
2513                            || winner.terminal.generation() >= reference.generation()
2514                        {
2515                            self.load_recovery_release_terminal(&winner.terminal)
2516                                .await?;
2517                            return if winner.terminal == reference {
2518                                Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2519                            } else {
2520                                Ok(RecordRecoveryReleaseCommitResult::Conflict {
2521                                    winner: winner.terminal.clone(),
2522                                })
2523                            };
2524                        }
2525                    }
2526                    if !winner_head.lease.matches_proof(proof) {
2527                        return Err(ClusterCheckpointAuthorityError::Fenced);
2528                    }
2529                    if winner_head.lease.seq <= base_sequence {
2530                        return Err(LeaseError::Invalid(
2531                            "recovery release authority contention did not advance the sequence"
2532                                .into(),
2533                        )
2534                        .into());
2535                    }
2536                    tokio::task::yield_now().await;
2537                }
2538            }
2539        }
2540    }
2541
2542    async fn recovery_release_terminal_from(
2543        &self,
2544        head: &LeaderAuthorityRecord,
2545        link: &RecoveryReleaseLink,
2546    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2547        let admission = if link.sequence == head.lease.seq {
2548            head.clone()
2549        } else {
2550            read_authority_record(self.store.as_ref(), link.sequence)
2551                .await?
2552                .ok_or_else(|| {
2553                    LeaseError::Invalid(
2554                        "recovery release admission is missing from retained authority history"
2555                            .into(),
2556                    )
2557                })?
2558        };
2559        let commit = admission
2560            .recovery_release_commit
2561            .as_ref()
2562            .filter(|commit| {
2563                commit.terminal == link.terminal
2564                    && admission.recovery_release_head.as_ref() == Some(link)
2565            })
2566            .ok_or_else(|| {
2567                LeaseError::Invalid(
2568                    "recovery release head does not match its admitting authority record".into(),
2569                )
2570            })?;
2571        let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2572        if terminal.round.leader_proof != commit.leader_proof {
2573            return Err(LeaseError::Invalid(
2574                "recovery release terminal does not match its admitting leader proof".into(),
2575            )
2576            .into());
2577        }
2578        Ok(terminal)
2579    }
2580
2581    pub(crate) async fn recovery_admission_snapshot(
2582        &self,
2583    ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2584        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2585            let head = self
2586                .load_record()
2587                .await?
2588                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2589            if let Some(operation) = head.committed_topology_operation() {
2590                self.audit_topology_operation(operation).await?;
2591            }
2592            let Some(link) = head.recovery_release_head.clone() else {
2593                return Ok(RecoveryAdmissionSnapshot {
2594                    committed_release: None,
2595                    fault_inventory: Self::recovery_fault_inventory_from(&head),
2596                    authority_sequence: head.lease.seq,
2597                    release_head: None,
2598                    topology: head
2599                        .committed_topology_operation()
2600                        .and_then(|operation| operation.commit.clone()),
2601                });
2602            };
2603            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2604            let rechecked = self
2605                .load_record()
2606                .await?
2607                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2608            if rechecked.lease.seq >= head.lease.seq
2609                && rechecked.recovery_release_head == head.recovery_release_head
2610                && rechecked.recovery_fault_revision == head.recovery_fault_revision
2611                && rechecked.recovery_fault_slots == head.recovery_fault_slots
2612                && rechecked.committed_topology_operation() == head.committed_topology_operation()
2613            {
2614                return Ok(RecoveryAdmissionSnapshot {
2615                    committed_release: Some(terminal?),
2616                    fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2617                    authority_sequence: rechecked.lease.seq,
2618                    release_head: rechecked.recovery_release_head.clone(),
2619                    topology: rechecked
2620                        .committed_topology_operation()
2621                        .and_then(|operation| operation.commit.clone()),
2622                });
2623            }
2624            tokio::task::yield_now().await;
2625        }
2626        Err(LeaseError::Io(format!(
2627            "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2628        ))
2629        .into())
2630    }
2631
2632    pub(crate) async fn recovery_admission_is_current(
2633        &self,
2634        snapshot: &RecoveryAdmissionSnapshot,
2635        leader_proof: &LeaderProof,
2636    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2637        let current = self
2638            .load_record()
2639            .await?
2640            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2641        let inventory = Self::recovery_fault_inventory_from(&current);
2642        let topology = current
2643            .committed_topology_operation()
2644            .and_then(|operation| operation.commit.as_ref());
2645        let release_matches_topology = current.live_topology_migration().is_none_or(|operation| {
2646            operation.phase == super::TopologyAdmissionPhase::Active
2647                && snapshot
2648                    .committed_release
2649                    .as_ref()
2650                    .and_then(|release| release.round.topology_binding())
2651                    .is_some_and(|binding| Some(binding.commit()) == operation.commit.as_ref())
2652        });
2653        Ok(topology == snapshot.topology.as_ref()
2654            && release_matches_topology
2655            && current.lease.matches_proof(leader_proof)
2656            && current.lease.seq >= snapshot.authority_sequence
2657            && current.recovery_release_head == snapshot.release_head
2658            && inventory == snapshot.fault_inventory
2659            && inventory.faults().is_empty())
2660    }
2661
2662    pub(crate) async fn latest_recovery_release_terminal(
2663        &self,
2664    ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2665        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2666            let Some(head) = self.load_record().await? else {
2667                return Ok(None);
2668            };
2669            let Some(link) = head.recovery_release_head.clone() else {
2670                return Ok(None);
2671            };
2672            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2673            let rechecked = self
2674                .load_record()
2675                .await?
2676                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2677            if rechecked.recovery_release_head.as_ref() == Some(&link) {
2678                return Ok(Some(terminal?));
2679            }
2680            tokio::task::yield_now().await;
2681        }
2682        Err(LeaseError::Io(format!(
2683            "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2684        ))
2685        .into())
2686    }
2687
2688    fn ttl(&self) -> Result<Duration, LeaseError> {
2689        let ttl = u64::try_from(self.ttl_ms)
2690            .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2691        if ttl == 0 {
2692            return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2693        }
2694        Ok(Duration::from_millis(ttl))
2695    }
2696
2697    fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2698        now_ms
2699            .checked_add(self.ttl_ms)
2700            .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2701    }
2702
2703    #[cfg(test)]
2704    async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2705        let prefix = OsPath::from(LEASE_PREFIX);
2706        let mut entries = self.store.list(Some(&prefix));
2707        let mut sequences = Vec::new();
2708        while let Some(entry) = entries.next().await {
2709            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2710            if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2711                return Err(LeaseError::Invalid(format!(
2712                    "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2713                )));
2714            }
2715            sequences.push(lease_sequence_from_path(&entry.location)?);
2716        }
2717        sequences.sort_unstable();
2718        sequences.dedup();
2719        Ok(sequences)
2720    }
2721
2722    fn schedule_history_prune(&self) {
2723        if self
2724            .prune_running
2725            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2726            .is_err()
2727        {
2728            return;
2729        }
2730        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2731            self.prune_running.store(false, Ordering::Release);
2732            return;
2733        };
2734        let store = Arc::clone(&self.store);
2735        let prune_running = Arc::clone(&self.prune_running);
2736        let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2737        runtime.spawn(async move {
2738            let _latch = PruneLatchGuard(prune_running);
2739            match tokio::time::timeout(
2740                LEADER_LEASE_PRUNE_TIMEOUT,
2741                Self::prune_history(&store, grace_ms),
2742            )
2743            .await
2744            {
2745                Ok(Ok(())) => {}
2746                Ok(Err(error)) => {
2747                    tracing::warn!(%error, "leader lease history prune failed");
2748                }
2749                Err(_) => {
2750                    tracing::warn!(
2751                        timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2752                        "leader lease history prune timed out"
2753                    );
2754                }
2755            }
2756        });
2757    }
2758
2759    async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2760        let authority = Self::new(Arc::clone(store), 1);
2761        let Some(head) = authority.load_record().await? else {
2762            return Ok(());
2763        };
2764        let head_sequence = head.lease.seq;
2765        let mut retained = BTreeSet::from([head_sequence]);
2766        authority
2767            .retain_topology_history(&head, &mut retained)
2768            .await?;
2769        if let Some(reservation) = &head.assignment_drain_reservation {
2770            retained.insert(reservation.authority_sequence);
2771        }
2772        if let Some(previous) = head_sequence
2773            .checked_sub(1)
2774            .filter(|sequence| *sequence != 0)
2775        {
2776            retained.insert(previous);
2777        }
2778        let floor = head
2779            .outcome_floor
2780            .as_ref()
2781            .map_or(0, |floor| floor.authority_before_epoch);
2782        let artifact_floor = head
2783            .outcome_floor
2784            .as_ref()
2785            .map_or(0, |floor| floor.artifact_before_epoch);
2786        let mut link = head.outcome_head;
2787        let mut outcome_links = 0;
2788        let mut terminal_commit_links = BTreeSet::new();
2789        let mut expected_commit_head = head.commit_head;
2790        while let Some(current) = link {
2791            if current.epoch < floor {
2792                if head
2793                    .outcome_floor
2794                    .as_ref()
2795                    .and_then(|floor| floor.terminal_anchor_link)
2796                    != Some(current)
2797                {
2798                    return Err(LeaseError::Invalid(
2799                        "outcome floor does not anchor the retained authority chain".into(),
2800                    ));
2801                }
2802                break;
2803            }
2804            if !consume_live_authority_link(&mut outcome_links) {
2805                return Err(LeaseError::Invalid(format!(
2806                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2807                )));
2808            }
2809            let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2810                .await?
2811                .filter(|record| {
2812                    record.outcome_head == Some(current)
2813                        && record
2814                            .checkpoint_outcome
2815                            .as_ref()
2816                            .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2817                            == Some((current.epoch, current.checkpoint_id))
2818                })
2819                .ok_or_else(|| {
2820                    LeaseError::Invalid("retained outcome authority chain is broken".into())
2821                })?;
2822            if outcome_record.commit_head != expected_commit_head {
2823                return Err(LeaseError::Invalid(
2824                    "retained outcome chain did not preserve the exact Commit head".into(),
2825                ));
2826            }
2827            retained.insert(current.sequence);
2828            if outcome_record
2829                .checkpoint_outcome
2830                .as_ref()
2831                .is_some_and(CheckpointOutcome::is_commit)
2832            {
2833                terminal_commit_links.insert(current);
2834                expected_commit_head = outcome_record.previous_commit;
2835            } else {
2836                expected_commit_head = outcome_record.commit_head;
2837            }
2838            link = outcome_record.previous_outcome;
2839        }
2840        let mut commit_link = head.commit_head;
2841        let mut commit_links = 0;
2842        let mut retained_commit_links = BTreeSet::new();
2843        while let Some(current) = commit_link {
2844            if current.epoch < artifact_floor {
2845                break;
2846            }
2847            if !consume_live_authority_link(&mut commit_links) {
2848                return Err(LeaseError::Invalid(format!(
2849                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2850                )));
2851            }
2852            let commit_record = read_authority_record(store.as_ref(), current.sequence)
2853                .await?
2854                .filter(|record| {
2855                    record.commit_head == Some(current)
2856                        && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2857                            outcome.is_commit()
2858                                && outcome.epoch == current.epoch
2859                                && outcome.checkpoint_id == current.checkpoint_id
2860                        })
2861                })
2862                .ok_or_else(|| {
2863                    LeaseError::Invalid("retained Commit authority chain is broken".into())
2864                })?;
2865            retained.insert(current.sequence);
2866            retained_commit_links.insert(current);
2867            commit_link = commit_record.previous_commit;
2868        }
2869        match (
2870            commit_link,
2871            head.outcome_floor
2872                .as_ref()
2873                .and_then(|floor| floor.committed_anchor_link),
2874            head.outcome_floor
2875                .as_ref()
2876                .and_then(|floor| floor.committed_anchor.as_ref()),
2877        ) {
2878            (Some(link), Some(anchor_link), Some(anchor))
2879                if link == anchor_link
2880                    && link.epoch == anchor.epoch
2881                    && link.checkpoint_id == anchor.checkpoint_id => {}
2882            (None, None, None) => {}
2883            _ => {
2884                return Err(LeaseError::Invalid(
2885                    "Commit chain does not meet the retained artifact-floor anchor".into(),
2886                ));
2887            }
2888        }
2889        Self::validate_retained_commit_boundary(
2890            &head,
2891            expected_commit_head,
2892            &retained_commit_links,
2893            &terminal_commit_links,
2894            artifact_floor,
2895        )?;
2896        let mut assignment_link = head.assignment_decision_head;
2897        let mut assignment_links = 0;
2898        while let Some(current) = assignment_link {
2899            if !consume_live_authority_link(&mut assignment_links) {
2900                return Err(LeaseError::Invalid(format!(
2901                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2902                )));
2903            }
2904            if head
2905                .assignment_decision_floor
2906                .as_ref()
2907                .is_some_and(|floor| current.target_version < floor.before_target_version)
2908            {
2909                if head
2910                    .assignment_decision_floor
2911                    .as_ref()
2912                    .and_then(|floor| floor.terminal_anchor_link)
2913                    != Some(current)
2914                {
2915                    return Err(LeaseError::Invalid(
2916                        "assignment-decision floor does not anchor the retained authority chain"
2917                            .into(),
2918                    ));
2919                }
2920                break;
2921            }
2922            let decision_record = read_authority_record(store.as_ref(), current.sequence)
2923                .await?
2924                .filter(|record| {
2925                    record.assignment_decision_head == Some(current)
2926                        && record
2927                            .assignment_decision
2928                            .as_ref()
2929                            .map(AuthorityAssignmentDecision::target_version)
2930                            == Some(current.target_version)
2931                })
2932                .ok_or_else(|| {
2933                    LeaseError::Invalid(
2934                        "retained assignment-decision authority chain is broken".into(),
2935                    )
2936                })?;
2937            retained.insert(current.sequence);
2938            assignment_link = decision_record.previous_assignment_decision;
2939        }
2940        let retained_release = match head.recovery_release_head.as_ref() {
2941            Some(link) => {
2942                let admission = read_authority_record(store.as_ref(), link.sequence)
2943                    .await?
2944                    .filter(|record| {
2945                        record.recovery_release_head.as_ref() == Some(link)
2946                            && record
2947                                .recovery_release_commit
2948                                .as_ref()
2949                                .is_some_and(|commit| commit.terminal == link.terminal)
2950                    })
2951                    .ok_or_else(|| {
2952                        LeaseError::Invalid(
2953                            "retained recovery release authority link is broken".into(),
2954                        )
2955                    })?;
2956                if admission.lease.seq != link.sequence {
2957                    return Err(LeaseError::Invalid(
2958                        "recovery release admission sequence does not match its retained link"
2959                            .into(),
2960                    ));
2961                }
2962                let terminal = authority
2963                    .load_recovery_release_terminal(&link.terminal)
2964                    .await
2965                    .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2966                if admission
2967                    .recovery_release_commit
2968                    .as_ref()
2969                    .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2970                {
2971                    return Err(LeaseError::Invalid(
2972                        "retained recovery release terminal does not match its admission".into(),
2973                    ));
2974                }
2975                retained.insert(link.sequence);
2976                Some(link.terminal.clone())
2977            }
2978            None => None,
2979        };
2980
2981        let mut history_exhausted = false;
2982        for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2983            let (candidates, exhausted) =
2984                Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2985            if candidates.is_empty() {
2986                history_exhausted = true;
2987                break;
2988            }
2989            let deletions =
2990                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2991            let mut results = store.delete_stream(Box::pin(deletions));
2992            while let Some(result) = results.next().await {
2993                if let Err(error) = result {
2994                    if !matches!(error, object_store::Error::NotFound { .. }) {
2995                        return Err(LeaseError::Io(error.to_string()));
2996                    }
2997                }
2998            }
2999            if exhausted {
3000                history_exhausted = true;
3001                break;
3002            }
3003            tokio::task::yield_now().await;
3004        }
3005        if !history_exhausted {
3006            return Err(LeaseError::Io(
3007                "leader lease history still exceeds the bounded prune budget".into(),
3008            ));
3009        }
3010        Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await?;
3011        if let Some(floor) = head.assignment_decision_floor.as_ref() {
3012            let before = head.assignment_drain_reservation.as_ref().map_or(
3013                floor.before_target_version,
3014                |reservation| {
3015                    floor
3016                        .before_target_version
3017                        .min(reservation.proposal.version)
3018                },
3019            );
3020            authority.prune_drain_proposals(before).await?;
3021        }
3022        Ok(())
3023    }
3024
3025    async fn prune_recovery_release_terminals(
3026        store: &Arc<dyn ObjectStore>,
3027        retained: Option<&RecoveryReleaseTerminalRef>,
3028        grace_ms: i64,
3029    ) -> Result<(), LeaseError> {
3030        let Some(retained) = retained else {
3031            return Ok(());
3032        };
3033        retained.validate()?;
3034        for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
3035            let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
3036            let mut listed = store.list(Some(&prefix));
3037            let now = now_millis();
3038            let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
3039            let mut exhausted = true;
3040            while let Some(entry) = listed.next().await {
3041                let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3042                let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
3043                let canonical = OsPath::from(format!(
3044                    "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
3045                ));
3046                if canonical != entry.location {
3047                    return Err(LeaseError::Invalid(format!(
3048                        "noncanonical recovery release terminal path {}",
3049                        entry.location
3050                    )));
3051                }
3052                if entry.location == recovery_release_terminal_path(retained)
3053                    || generation > retained.generation()
3054                    || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
3055                {
3056                    continue;
3057                }
3058                candidates.push(entry.location);
3059                if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
3060                    exhausted = false;
3061                    break;
3062                }
3063            }
3064            if candidates.is_empty() {
3065                return Ok(());
3066            }
3067            let deletions =
3068                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
3069            let mut results = store.delete_stream(Box::pin(deletions));
3070            while let Some(result) = results.next().await {
3071                if let Err(error) = result {
3072                    if !matches!(error, object_store::Error::NotFound { .. }) {
3073                        return Err(LeaseError::Io(error.to_string()));
3074                    }
3075                }
3076            }
3077            if exhausted {
3078                return Ok(());
3079            }
3080            tokio::task::yield_now().await;
3081        }
3082        Ok(())
3083    }
3084
3085    async fn prune_candidates(
3086        store: &Arc<dyn ObjectStore>,
3087        retained: &BTreeSet<u64>,
3088        snapshot_head_sequence: u64,
3089        grace_ms: i64,
3090    ) -> Result<(Vec<OsPath>, bool), LeaseError> {
3091        let prefix = OsPath::from(LEASE_PREFIX);
3092        let mut listed = store.list(Some(&prefix));
3093        let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
3094        let now = now_millis();
3095        let mut exhausted = true;
3096        while let Some(entry) = listed.next().await {
3097            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3098            let sequence = lease_sequence_from_path(&entry.location)?;
3099            if sequence >= snapshot_head_sequence
3100                || retained.contains(&sequence)
3101                || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
3102            {
3103                continue;
3104            }
3105            candidates.push(entry.location);
3106            if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
3107                exhausted = false;
3108                break;
3109            }
3110        }
3111        Ok((candidates, exhausted))
3112    }
3113
3114    /// Load the highest durable sequence.
3115    ///
3116    /// # Errors
3117    /// Fails closed on object-store I/O or malformed durable state.
3118    pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
3119        Ok(self.load_record().await?.map(|record| record.lease))
3120    }
3121
3122    async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3123        Ok(self
3124            .load_published_authority_head()
3125            .await?
3126            .map(|head| head.record))
3127    }
3128
3129    async fn load_published_authority_head(
3130        &self,
3131    ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
3132        let pointer = if let Some(pointer) =
3133            read_authority_head_pointer(self.store.as_ref()).await?
3134        {
3135            pointer
3136        } else {
3137            let Some(discovered) = self.discover_authority_head().await? else {
3138                let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
3139                    return Ok(None);
3140                };
3141                return self.load_authority_head_target(pointer).await.map(Some);
3142            };
3143            let discovered_sequence = discovered.lease.seq;
3144            let published_sequence = self
3145                .publish_authority_head(discovered_sequence, None)
3146                .await?;
3147            let pointer = self
3148                .reload_authority_head_pointer(published_sequence)
3149                .await?;
3150            if published_sequence > discovered_sequence
3151                || pointer.pointer.sequence > published_sequence
3152            {
3153                return self.load_authority_head_target(pointer).await.map(Some);
3154            }
3155            pointer
3156        };
3157
3158        let sequence = pointer.pointer.sequence;
3159        let successor_sequence = sequence.checked_add(1);
3160        let (record, successor) = if let Some(successor_sequence) = successor_sequence {
3161            tokio::try_join!(
3162                read_authority_record(self.store.as_ref(), sequence),
3163                read_authority_record(self.store.as_ref(), successor_sequence)
3164            )?
3165        } else {
3166            (
3167                read_authority_record(self.store.as_ref(), sequence).await?,
3168                None,
3169            )
3170        };
3171        let Some(record) = record else {
3172            let rechecked = read_authority_head_pointer(self.store.as_ref())
3173                .await?
3174                .ok_or_else(|| {
3175                    LeaseError::Invalid(
3176                        "leader authority head disappeared while reading its target".into(),
3177                    )
3178                })?;
3179            if rechecked.pointer.sequence > sequence {
3180                return self
3181                    .reload_published_authority_snapshot(rechecked.pointer.sequence)
3182                    .await
3183                    .map(Some);
3184            }
3185            if rechecked.pointer.sequence < sequence {
3186                return Err(LeaseError::Invalid(format!(
3187                    "leader authority head regressed from sequence {sequence} to {}",
3188                    rechecked.pointer.sequence
3189                )));
3190            }
3191            return Err(LeaseError::Invalid(format!(
3192                "leader authority head points ahead to missing sequence {sequence}"
3193            )));
3194        };
3195        let Some(successor_sequence) = successor_sequence else {
3196            return Ok(Some(PublishedAuthorityHead { record, pointer }));
3197        };
3198        if successor.is_none() {
3199            return Ok(Some(PublishedAuthorityHead { record, pointer }));
3200        }
3201
3202        if let Some(after_successor) = successor_sequence.checked_add(1) {
3203            if read_authority_record(self.store.as_ref(), after_successor)
3204                .await?
3205                .is_some()
3206            {
3207                let rechecked = read_authority_head_pointer(self.store.as_ref())
3208                    .await?
3209                    .ok_or_else(|| {
3210                        LeaseError::Invalid(
3211                            "leader authority head disappeared while checking pointer lag".into(),
3212                        )
3213                    })?;
3214                if rechecked.pointer.sequence == sequence {
3215                    return Err(LeaseError::Invalid(format!(
3216                        "leader authority head at sequence {sequence} lags by more than one record"
3217                    )));
3218                }
3219                if rechecked.pointer.sequence < sequence {
3220                    return Err(LeaseError::Invalid(format!(
3221                        "leader authority head regressed from sequence {sequence} to {}",
3222                        rechecked.pointer.sequence
3223                    )));
3224                }
3225                return self
3226                    .reload_published_authority_snapshot(rechecked.pointer.sequence)
3227                    .await
3228                    .map(Some);
3229            }
3230        }
3231
3232        let published_sequence = self
3233            .publish_authority_head(successor_sequence, Some(&pointer))
3234            .await?;
3235        self.reload_published_authority_snapshot(published_sequence)
3236            .await
3237            .map(Some)
3238    }
3239
3240    async fn reload_published_authority_snapshot(
3241        &self,
3242        minimum_sequence: u64,
3243    ) -> Result<PublishedAuthorityHead, LeaseError> {
3244        let pointer = self.reload_authority_head_pointer(minimum_sequence).await?;
3245        self.load_authority_head_target(pointer).await
3246    }
3247
3248    async fn reload_authority_head_pointer(
3249        &self,
3250        minimum_sequence: u64,
3251    ) -> Result<VersionedAuthorityHeadPointer, LeaseError> {
3252        let pointer = read_authority_head_pointer(self.store.as_ref())
3253            .await?
3254            .ok_or_else(|| {
3255                LeaseError::Invalid(
3256                    "leader authority head disappeared while reloading its published snapshot"
3257                        .into(),
3258                )
3259            })?;
3260        if pointer.pointer.sequence < minimum_sequence {
3261            return Err(LeaseError::Invalid(format!(
3262                "leader authority head regressed from sequence {minimum_sequence} to {}",
3263                pointer.pointer.sequence
3264            )));
3265        }
3266        Ok(pointer)
3267    }
3268
3269    async fn load_authority_head_target(
3270        &self,
3271        mut pointer: VersionedAuthorityHeadPointer,
3272    ) -> Result<PublishedAuthorityHead, LeaseError> {
3273        for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
3274            let sequence = pointer.pointer.sequence;
3275            if let Some(record) = read_authority_record(self.store.as_ref(), sequence).await? {
3276                return Ok(PublishedAuthorityHead { record, pointer });
3277            }
3278
3279            let rechecked = read_authority_head_pointer(self.store.as_ref())
3280                .await?
3281                .ok_or_else(|| {
3282                    LeaseError::Invalid(
3283                        "leader authority head disappeared while reading its target".into(),
3284                    )
3285                })?;
3286            if rechecked.pointer.sequence < sequence {
3287                return Err(LeaseError::Invalid(format!(
3288                    "leader authority head regressed from sequence {sequence} to {}",
3289                    rechecked.pointer.sequence
3290                )));
3291            }
3292            if rechecked.pointer.sequence == sequence {
3293                return Err(LeaseError::Invalid(format!(
3294                    "leader authority head points ahead to missing sequence {sequence}"
3295                )));
3296            }
3297            pointer = rechecked;
3298            if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
3299                tokio::task::yield_now().await;
3300            }
3301        }
3302        Err(LeaseError::Io(format!(
3303            "leader authority head target changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
3304        )))
3305    }
3306
3307    async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3308        let prefix = OsPath::from(LEASE_PREFIX);
3309        let mut listed = self.store.list(Some(&prefix));
3310        let mut discovered = 0usize;
3311        let mut maximum = None;
3312        while let Some(entry) = listed.next().await {
3313            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3314            discovered = discovered.checked_add(1).ok_or_else(|| {
3315                LeaseError::Invalid("leader authority discovery count overflowed".into())
3316            })?;
3317            if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
3318                return Err(LeaseError::Invalid(format!(
3319                    "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
3320                )));
3321            }
3322            let sequence = lease_sequence_from_path(&entry.location)?;
3323            maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
3324        }
3325        let Some(sequence) = maximum else {
3326            return Ok(None);
3327        };
3328        read_authority_record(self.store.as_ref(), sequence)
3329            .await?
3330            .map(Some)
3331            .ok_or_else(|| {
3332                LeaseError::Io(format!(
3333                    "discovered leader authority sequence {sequence} vanished before publication"
3334                ))
3335            })
3336    }
3337
3338    async fn publish_authority_head(
3339        &self,
3340        sequence: u64,
3341        expected: Option<&VersionedAuthorityHeadPointer>,
3342    ) -> Result<u64, LeaseError> {
3343        if let Some(expected) = expected {
3344            if expected.pointer.sequence.checked_add(1) != Some(sequence) {
3345                return Err(LeaseError::Invalid(
3346                    "leader authority head update is not the exact successor".into(),
3347                ));
3348            }
3349            if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3350            {
3351                return Err(LeaseError::Invalid(
3352                    "leader authority store did not provide a native conditional update version"
3353                        .into(),
3354                ));
3355            }
3356        }
3357
3358        let options = PutOptions {
3359            mode: expected.map_or(PutMode::Create, |expected| {
3360                PutMode::Update(expected.update_version.clone())
3361            }),
3362            ..PutOptions::default()
3363        };
3364        let result = self
3365            .store
3366            .put_opts(
3367                &authority_head_path(),
3368                PutPayload::from(encode_authority_head_pointer(sequence)?),
3369                options,
3370            )
3371            .await;
3372        if result.is_ok() {
3373            return Ok(sequence);
3374        }
3375        let write_error = result
3376            .expect_err("successful authority head writes return above")
3377            .to_string();
3378        let current = read_authority_head_pointer(self.store.as_ref())
3379            .await?
3380            .ok_or_else(|| {
3381                LeaseError::Io(format!(
3382                    "leader authority head write failed ({write_error}) and no pointer was durable"
3383                ))
3384            })?;
3385        if let Some(expected) = expected {
3386            if current.pointer.sequence < expected.pointer.sequence {
3387                return Err(LeaseError::Invalid(format!(
3388                    "leader authority head regressed from sequence {} to {}",
3389                    expected.pointer.sequence, current.pointer.sequence
3390                )));
3391            }
3392        }
3393        if current.pointer.sequence >= sequence {
3394            return Ok(current.pointer.sequence);
3395        }
3396        Err(LeaseError::Io(format!(
3397            "leader authority head write failed ({write_error}) and remained at sequence {}",
3398            current.pointer.sequence
3399        )))
3400    }
3401
3402    async fn read_catalog_manifest_blob(
3403        &self,
3404        reference: &CatalogManifestRef,
3405    ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3406        reference.validate()?;
3407        let path = reference.object_path();
3408        let result = match self.store.get(&path).await {
3409            Ok(result) => result,
3410            Err(object_store::Error::NotFound { .. }) => {
3411                return Err(CatalogManifestError::Invalid(format!(
3412                    "catalog manifest blob '{}' is missing",
3413                    reference.sha256
3414                )));
3415            }
3416            Err(error) => {
3417                return Err(CatalogManifestError::Authority(LeaseError::Io(
3418                    error.to_string(),
3419                )));
3420            }
3421        };
3422        if result.meta.size != reference.encoded_len {
3423            return Err(CatalogManifestError::Invalid(format!(
3424                "catalog manifest blob '{}' is {} bytes, expected {}",
3425                reference.sha256, result.meta.size, reference.encoded_len
3426            )));
3427        }
3428        let bytes = result
3429            .bytes()
3430            .await
3431            .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3432        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3433            return Err(CatalogManifestError::Invalid(format!(
3434                "catalog manifest blob '{}' payload length changed while reading",
3435                reference.sha256
3436            )));
3437        }
3438        let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3439        let (canonical, actual_reference) = manifest.encode_and_reference()?;
3440        if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3441            return Err(CatalogManifestError::Invalid(format!(
3442                "catalog manifest blob '{}' does not match its sealed reference",
3443                reference.sha256
3444            )));
3445        }
3446        Ok((manifest, bytes))
3447    }
3448
3449    pub(super) async fn load_catalog_manifest(
3450        &self,
3451        reference: &CatalogManifestRef,
3452    ) -> Result<CatalogManifest, CatalogManifestError> {
3453        self.read_catalog_manifest_blob(reference)
3454            .await
3455            .map(|(manifest, _)| manifest)
3456    }
3457
3458    async fn ensure_catalog_manifest_blob(
3459        &self,
3460        encoded: &[u8],
3461        reference: &CatalogManifestRef,
3462    ) -> Result<(), CatalogManifestError> {
3463        reference.validate()?;
3464        let options = PutOptions {
3465            mode: PutMode::Create,
3466            ..PutOptions::default()
3467        };
3468        let path = reference.object_path();
3469        let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3470        let put_error = match self.store.put_opts(&path, payload, options).await {
3471            Ok(_)
3472            | Err(
3473                object_store::Error::AlreadyExists { .. }
3474                | object_store::Error::Precondition { .. },
3475            ) => None,
3476            Err(error) => Some(error),
3477        };
3478
3479        match self.read_catalog_manifest_blob(reference).await {
3480            Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3481            Ok(_) => Err(CatalogManifestError::Invalid(format!(
3482                "catalog manifest blob '{}' differs from the proposed content",
3483                reference.sha256
3484            ))),
3485            Err(error) => {
3486                if let Some(put_error) = put_error {
3487                    Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3488                        "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3489                    ))))
3490                } else {
3491                    Err(error)
3492                }
3493            }
3494        }
3495    }
3496
3497    async fn create_authority_record(
3498        &self,
3499        expected: Option<&PublishedAuthorityHead>,
3500        candidate: &LeaderAuthorityRecord,
3501    ) -> Result<AuthorityCreateOutcome, LeaseError> {
3502        let encoded = encode_authority_record(candidate)?;
3503        let expected_pointer = match expected {
3504            None => {
3505                if candidate.lease.seq != 1 {
3506                    return Err(LeaseError::Invalid(format!(
3507                        "cannot append authority sequence {} to an empty published namespace",
3508                        candidate.lease.seq
3509                    )));
3510                }
3511                None
3512            }
3513            Some(head) if head.record.lease.seq == candidate.lease.seq => {
3514                return if head.record == *candidate {
3515                    Ok(AuthorityCreateOutcome::ExistingIdentical)
3516                } else {
3517                    Ok(AuthorityCreateOutcome::Contended(Box::new(
3518                        head.record.clone(),
3519                    )))
3520                };
3521            }
3522            Some(head) if head.record.lease.seq > candidate.lease.seq => {
3523                return Ok(AuthorityCreateOutcome::Contended(Box::new(
3524                    head.record.clone(),
3525                )));
3526            }
3527            Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3528                head.record.validate_topology_successor(candidate)?;
3529                head.record
3530                    .validate_topology_admission_successor(candidate)?;
3531                Some(&head.pointer)
3532            }
3533            Some(head) => {
3534                return Err(LeaseError::Invalid(format!(
3535                    "cannot append authority sequence {} after published sequence {}",
3536                    candidate.lease.seq, head.record.lease.seq
3537                )));
3538            }
3539        };
3540        let options = PutOptions {
3541            mode: PutMode::Create,
3542            ..PutOptions::default()
3543        };
3544        let result = self
3545            .store
3546            .put_opts(
3547                &lease_path(candidate.lease.seq),
3548                PutPayload::from(encoded),
3549                options,
3550            )
3551            .await;
3552        let created = result.is_ok();
3553        if let Err(error) = result {
3554            let Some(at_sequence) =
3555                read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3556            else {
3557                return Err(LeaseError::Io(format!(
3558                    "authority record {} write failed ({error}) and no record was durable",
3559                    candidate.lease.seq
3560                )));
3561            };
3562            if at_sequence != *candidate {
3563                let winner = self.load_record().await?.ok_or_else(|| {
3564                    LeaseError::Io("authority contender was not published or readable".into())
3565                })?;
3566                return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3567            }
3568        }
3569
3570        let published_sequence = self
3571            .publish_authority_head(candidate.lease.seq, expected_pointer)
3572            .await?;
3573        if published_sequence > candidate.lease.seq {
3574            if created {
3575                self.schedule_history_prune();
3576            }
3577            let winner = self
3578                .load_record()
3579                .await?
3580                .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3581            return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3582        }
3583        self.schedule_history_prune();
3584        if created {
3585            Ok(AuthorityCreateOutcome::Created)
3586        } else {
3587            Ok(AuthorityCreateOutcome::ExistingIdentical)
3588        }
3589    }
3590
3591    /// Seal the complete catalog by appending it to the exact durable leader term.
3592    ///
3593    /// Renewals that race this operation are retried under the same proof. A takeover either
3594    /// observes and preserves the seal or wins the next sequence and fences this writer.
3595    ///
3596    /// # Errors
3597    /// Rejects malformed inventories, divergent existing inventories, stale proofs, and durable
3598    /// storage failures.
3599    pub(super) async fn seal_catalog(
3600        &self,
3601        proof: &LeaderProof,
3602        manifest: &CatalogManifest,
3603    ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3604        if !proof.is_canonical() {
3605            return Err(CatalogManifestError::Fenced);
3606        }
3607        let (encoded, reference) = manifest.encode_and_reference()?;
3608
3609        let current = self
3610            .load_record()
3611            .await?
3612            .ok_or(CatalogManifestError::Fenced)?;
3613        if let Some(sealed) = &current.lease.catalog_manifest {
3614            let durable = self.load_catalog_manifest(sealed).await?;
3615            return if durable == *manifest {
3616                Ok(CatalogSealOutcome::ExistingIdentical)
3617            } else {
3618                Err(CatalogManifestError::Conflict)
3619            };
3620        }
3621        if !current.lease.matches_proof(proof) {
3622            return Err(CatalogManifestError::Fenced);
3623        }
3624        self.ensure_catalog_manifest_blob(&encoded, &reference)
3625            .await?;
3626
3627        loop {
3628            let published = self
3629                .load_published_authority_head()
3630                .await?
3631                .ok_or(CatalogManifestError::Fenced)?;
3632            let current = &published.record;
3633            if let Some(sealed) = &current.lease.catalog_manifest {
3634                let durable = self.load_catalog_manifest(sealed).await?;
3635                return if durable == *manifest {
3636                    Ok(CatalogSealOutcome::ExistingIdentical)
3637                } else {
3638                    Err(CatalogManifestError::Conflict)
3639                };
3640            }
3641            if !current.lease.matches_proof(proof) {
3642                return Err(CatalogManifestError::Fenced);
3643            }
3644
3645            let base_sequence = current.lease.seq;
3646            let candidate_lease = LeaderLease {
3647                seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3648                    CatalogManifestError::Authority(LeaseError::Invalid(
3649                        "lease sequence exhausted".into(),
3650                    ))
3651                })?,
3652                renewal_sequence: current.lease.renewal_sequence,
3653                token: current.lease.token,
3654                owner: current.lease.owner.clone(),
3655                expires_at_ms: current.lease.expires_at_ms,
3656                catalog_manifest: Some(reference.clone()),
3657            };
3658            let candidate = current.preserve_with_lease(candidate_lease);
3659            match self
3660                .create_authority_record(Some(&published), &candidate)
3661                .await?
3662            {
3663                AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3664                AuthorityCreateOutcome::ExistingIdentical => {
3665                    return Ok(CatalogSealOutcome::ExistingIdentical);
3666                }
3667                AuthorityCreateOutcome::Contended(winner) => {
3668                    if let Some(sealed) = &winner.lease.catalog_manifest {
3669                        let durable = self.load_catalog_manifest(sealed).await?;
3670                        return if durable == *manifest {
3671                            Ok(CatalogSealOutcome::ExistingIdentical)
3672                        } else {
3673                            Err(CatalogManifestError::Conflict)
3674                        };
3675                    }
3676                    if !winner.lease.matches_proof(proof) {
3677                        return Err(CatalogManifestError::Fenced);
3678                    }
3679                    if winner.lease.seq <= base_sequence {
3680                        return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3681                            "catalog seal contention did not advance the authority sequence".into(),
3682                        )));
3683                    }
3684                    tokio::task::yield_now().await;
3685                }
3686            }
3687        }
3688    }
3689
3690    async fn audited_cluster_outcomes_from(
3691        &self,
3692        head: &LeaderAuthorityRecord,
3693    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3694        let floor = head.outcome_floor.as_ref();
3695        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3696        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3697        let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3698        let mut terminal_newest_first = Vec::new();
3699        let mut terminal_links = Vec::new();
3700        let mut link = head.outcome_head;
3701        let mut expected_commit_head = head.commit_head;
3702        let mut traversed = 0;
3703        while let Some(current) = link {
3704            if current.epoch < authority_before_epoch {
3705                break;
3706            }
3707            if !consume_live_authority_link(&mut traversed) {
3708                return Err(DecisionError::Conflict(format!(
3709                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3710                ))
3711                .into());
3712            }
3713            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3714                record.clone()
3715            } else {
3716                let record = read_authority_record(self.store.as_ref(), current.sequence)
3717                    .await?
3718                    .ok_or_else(|| {
3719                        DecisionError::InventoryChanged(format!(
3720                            "cluster outcome authority record {} disappeared during audit",
3721                            current.sequence
3722                        ))
3723                    })?;
3724                terminal_records.insert(current.sequence, record.clone());
3725                record
3726            };
3727            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3728                DecisionError::Conflict(format!(
3729                    "cluster outcome head epoch {} points to non-outcome authority record {}",
3730                    current.epoch, current.sequence
3731                ))
3732            })?;
3733            if record.outcome_head != Some(current)
3734                || outcome.epoch != current.epoch
3735                || outcome.checkpoint_id != current.checkpoint_id
3736            {
3737                return Err(DecisionError::Conflict(format!(
3738                    "cluster outcome link epoch {} sequence {} does not match its authority record",
3739                    current.epoch, current.sequence
3740                ))
3741                .into());
3742            }
3743            if record.commit_head != expected_commit_head {
3744                return Err(DecisionError::Conflict(format!(
3745                    "cluster outcome epoch {} did not preserve the exact Commit head",
3746                    outcome.epoch
3747                ))
3748                .into());
3749            }
3750            expected_commit_head = if outcome.is_commit() {
3751                record.previous_commit
3752            } else {
3753                record.commit_head
3754            };
3755            terminal_newest_first.push(outcome);
3756            terminal_links.push(current);
3757            link = record.previous_outcome;
3758        }
3759
3760        if let Some(floor) = floor {
3761            match (
3762                link,
3763                floor.terminal_anchor_link,
3764                floor.terminal_anchor.as_ref(),
3765            ) {
3766                (Some(link), Some(anchor_link), Some(anchor))
3767                    if link == anchor_link
3768                        && link.epoch == anchor.epoch
3769                        && link.checkpoint_id == anchor.checkpoint_id => {}
3770                (None, None, None) => {}
3771                _ => {
3772                    return Err(DecisionError::Conflict(format!(
3773                        "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3774                        floor.authority_before_epoch
3775                    ))
3776                    .into());
3777                }
3778            }
3779        } else if link.is_some() {
3780            return Err(DecisionError::Conflict(
3781                "cluster outcome chain stopped without a durable retention floor".into(),
3782            )
3783            .into());
3784        }
3785
3786        let mut commit_newest_first = Vec::new();
3787        let mut commit_links = Vec::new();
3788        link = head.commit_head;
3789        traversed = 0;
3790        while let Some(current) = link {
3791            if current.epoch < artifact_before_epoch {
3792                break;
3793            }
3794            if !consume_live_authority_link(&mut traversed) {
3795                return Err(DecisionError::Conflict(format!(
3796                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3797                ))
3798                .into());
3799            }
3800            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3801                record.clone()
3802            } else {
3803                read_authority_record(self.store.as_ref(), current.sequence)
3804                    .await?
3805                    .ok_or_else(|| {
3806                        DecisionError::InventoryChanged(format!(
3807                            "cluster Commit authority record {} disappeared during audit",
3808                            current.sequence
3809                        ))
3810                    })?
3811            };
3812            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3813                DecisionError::Conflict(format!(
3814                    "cluster Commit head epoch {} points to non-outcome authority record {}",
3815                    current.epoch, current.sequence
3816                ))
3817            })?;
3818            if !outcome.is_commit()
3819                || record.commit_head != Some(current)
3820                || outcome.epoch != current.epoch
3821                || outcome.checkpoint_id != current.checkpoint_id
3822            {
3823                return Err(DecisionError::Conflict(format!(
3824                    "cluster Commit link epoch {} sequence {} does not match its authority record",
3825                    current.epoch, current.sequence
3826                ))
3827                .into());
3828            }
3829            commit_newest_first.push(outcome);
3830            commit_links.push(current);
3831            link = record.previous_commit;
3832        }
3833
3834        if let Some(floor) = floor {
3835            match (
3836                link,
3837                floor.committed_anchor_link,
3838                floor.committed_anchor.as_ref(),
3839            ) {
3840                (Some(link), Some(anchor_link), Some(anchor))
3841                    if link == anchor_link
3842                        && link.epoch == anchor.epoch
3843                        && link.checkpoint_id == anchor.checkpoint_id => {}
3844                (None, None, None) => {}
3845                _ => {
3846                    return Err(DecisionError::Conflict(format!(
3847                        "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3848                        floor.artifact_before_epoch
3849                    ))
3850                    .into());
3851                }
3852            }
3853        } else if link.is_some() {
3854            return Err(DecisionError::Conflict(
3855                "cluster Commit chain stopped without a durable retention floor".into(),
3856            )
3857            .into());
3858        }
3859
3860        terminal_newest_first.reverse();
3861        terminal_links.reverse();
3862        commit_newest_first.reverse();
3863        commit_links.reverse();
3864        let boundary_commit_head = floor
3865            .and_then(|floor| floor.terminal_anchor.as_ref())
3866            .and_then(|anchor| {
3867                commit_links
3868                    .iter()
3869                    .rev()
3870                    .find(|link| link.epoch <= anchor.epoch)
3871                    .copied()
3872                    .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3873            });
3874        if expected_commit_head != boundary_commit_head {
3875            return Err(DecisionError::Conflict(
3876                "cluster terminal chain does not preserve Commit continuity at its durable floor"
3877                    .into(),
3878            )
3879            .into());
3880        }
3881        for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3882            if !outcome.is_commit() {
3883                continue;
3884            }
3885            let commit_link = commit_links
3886                .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3887                .ok()
3888                .map(|index| commit_links[index]);
3889            if commit_link != Some(*terminal_link) {
3890                return Err(DecisionError::Conflict(format!(
3891                    "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3892                    outcome.epoch
3893                ))
3894                .into());
3895            }
3896        }
3897        if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3898            floor
3899                .terminal_anchor
3900                .as_ref()
3901                .zip(floor.terminal_anchor_link)
3902        }) {
3903            if terminal_anchor.is_commit() {
3904                let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3905                    commit_links
3906                        .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3907                        .ok()
3908                        .map(|index| commit_links[index])
3909                } else {
3910                    floor.and_then(|floor| {
3911                        (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3912                            .then_some(floor.committed_anchor_link)
3913                            .flatten()
3914                    })
3915                };
3916                if commit_link != Some(terminal_anchor_link) {
3917                    return Err(DecisionError::Conflict(format!(
3918                        "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3919                        terminal_anchor.epoch
3920                    ))
3921                    .into());
3922                }
3923            }
3924        }
3925        let mut outcomes = Vec::with_capacity(
3926            terminal_newest_first
3927                .len()
3928                .saturating_add(commit_newest_first.len())
3929                .saturating_add(2),
3930        );
3931        outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3932        outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3933        outcomes.extend(terminal_newest_first);
3934        outcomes.extend(commit_newest_first);
3935        outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3936        let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3937        for outcome in outcomes {
3938            if let Some(previous) = merged.last() {
3939                if previous.epoch == outcome.epoch {
3940                    if previous != &outcome {
3941                        return Err(DecisionError::Conflict(format!(
3942                            "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3943                            outcome.epoch
3944                        ))
3945                        .into());
3946                    }
3947                    continue;
3948                }
3949            }
3950            merged.push(outcome);
3951        }
3952
3953        let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3954            .load_or_create_deployment_id()
3955            .await?;
3956        for outcome in &merged {
3957            if outcome.deployment_id != expected_deployment {
3958                return Err(DecisionError::Conflict(format!(
3959                    "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3960                    outcome.epoch, outcome.deployment_id, expected_deployment
3961                ))
3962                .into());
3963            }
3964        }
3965        for pair in merged.windows(2) {
3966            let previous = &pair[0];
3967            let current = &pair[1];
3968            if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3969                return Err(DecisionError::Conflict(format!(
3970                    "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3971                    previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3972                ))
3973                .into());
3974            }
3975        }
3976        Ok(ClusterOutcomeAuditSnapshot {
3977            outcomes: Arc::from(merged),
3978            terminal_links: Arc::from(terminal_links),
3979            commit_links: Arc::from(commit_links),
3980        })
3981    }
3982
3983    fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3984        ClusterOutcomeAuditKey {
3985            terminal_head: head.outcome_head,
3986            commit_head: head.commit_head,
3987            floor: head.outcome_floor.clone(),
3988        }
3989    }
3990
3991    fn cached_cluster_outcome_audit(
3992        &self,
3993        key: &ClusterOutcomeAuditKey,
3994        authority_sequence: u64,
3995    ) -> Option<ClusterOutcomeAuditSnapshot> {
3996        let mut cache = self.outcome_audit_cache.lock();
3997        let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3998        cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3999        Some(cached.snapshot.clone())
4000    }
4001
4002    fn install_cluster_outcome_audit(
4003        &self,
4004        key: ClusterOutcomeAuditKey,
4005        authority_sequence: u64,
4006        snapshot: ClusterOutcomeAuditSnapshot,
4007    ) {
4008        let mut cache = self.outcome_audit_cache.lock();
4009        if cache.as_ref().is_none_or(|cached| {
4010            cached.authority_sequence < authority_sequence
4011                || (cached.authority_sequence == authority_sequence && cached.key == key)
4012        }) {
4013            *cache = Some(ClusterOutcomeAuditCache {
4014                key,
4015                authority_sequence,
4016                snapshot,
4017            });
4018        }
4019    }
4020
4021    fn cluster_outcome_audit_gate(
4022        &self,
4023        key: &ClusterOutcomeAuditKey,
4024    ) -> Arc<tokio::sync::Mutex<()>> {
4025        let mut flights = self.outcome_audit_flights.lock();
4026        flights.retain(|flight| flight.gate.strong_count() != 0);
4027        if let Some(gate) = flights
4028            .iter()
4029            .find(|flight| flight.key == *key)
4030            .and_then(|flight| flight.gate.upgrade())
4031        {
4032            return gate;
4033        }
4034        let gate = Arc::new(tokio::sync::Mutex::new(()));
4035        flights.push(ClusterOutcomeAuditFlight {
4036            key: key.clone(),
4037            gate: Arc::downgrade(&gate),
4038        });
4039        gate
4040    }
4041
4042    async fn cached_audited_cluster_outcomes_from(
4043        &self,
4044        head: &LeaderAuthorityRecord,
4045    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
4046        let key = Self::cluster_outcome_audit_key(head);
4047        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
4048            return Ok(snapshot);
4049        }
4050
4051        let gate = self.cluster_outcome_audit_gate(&key);
4052        let _guard = gate.lock().await;
4053        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
4054            return Ok(snapshot);
4055        }
4056
4057        let snapshot = self.audited_cluster_outcomes_from(head).await?;
4058        self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
4059        Ok(snapshot)
4060    }
4061
4062    fn outcomes_retained_by_floor(
4063        floor: &AuthorityOutcomeFloor,
4064        snapshot: &ClusterOutcomeAuditSnapshot,
4065    ) -> ClusterOutcomeAuditSnapshot {
4066        let outcomes = &snapshot.outcomes;
4067        let mut retained = Vec::with_capacity(outcomes.len());
4068        if let Some(anchor) = floor.committed_anchor.as_ref() {
4069            retained.push(anchor.clone());
4070        }
4071        if let Some(anchor) = floor.terminal_anchor.as_ref() {
4072            if retained.last() != Some(anchor) {
4073                retained.push(anchor.clone());
4074            }
4075        }
4076        retained.extend(
4077            outcomes
4078                .iter()
4079                .filter(|outcome| {
4080                    outcome.epoch >= floor.authority_before_epoch
4081                        || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
4082                })
4083                .cloned(),
4084        );
4085        retained.sort_unstable_by_key(|outcome| outcome.epoch);
4086        retained.dedup();
4087        ClusterOutcomeAuditSnapshot {
4088            outcomes: Arc::from(retained),
4089            terminal_links: Arc::from(
4090                snapshot
4091                    .terminal_links
4092                    .iter()
4093                    .filter(|link| link.epoch >= floor.authority_before_epoch)
4094                    .copied()
4095                    .collect::<Vec<_>>(),
4096            ),
4097            commit_links: Arc::from(
4098                snapshot
4099                    .commit_links
4100                    .iter()
4101                    .filter(|link| link.epoch >= floor.artifact_before_epoch)
4102                    .copied()
4103                    .collect::<Vec<_>>(),
4104            ),
4105        }
4106    }
4107
4108    async fn build_cluster_outcome_floor(
4109        &self,
4110        current: &LeaderAuthorityRecord,
4111        snapshot: &ClusterOutcomeAuditSnapshot,
4112        artifact_before_epoch: u64,
4113        authority_before_epoch: u64,
4114    ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
4115        let terminal_anchor = snapshot
4116            .outcomes
4117            .iter()
4118            .rev()
4119            .find(|outcome| outcome.epoch < authority_before_epoch)
4120            .cloned();
4121        let terminal_anchor_link = match terminal_anchor.as_ref() {
4122            Some(anchor)
4123                if current
4124                    .outcome_floor
4125                    .as_ref()
4126                    .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
4127            {
4128                current
4129                    .outcome_floor
4130                    .as_ref()
4131                    .and_then(|floor| floor.terminal_anchor_link)
4132            }
4133            Some(anchor) => {
4134                let index = snapshot
4135                    .terminal_links
4136                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4137                    .map_err(|_| {
4138                        DecisionError::Conflict(format!(
4139                            "cluster outcome epoch {} has no audited authority link",
4140                            anchor.epoch
4141                        ))
4142                    })?;
4143                let link = snapshot.terminal_links[index];
4144                if link.checkpoint_id != anchor.checkpoint_id {
4145                    return Err(DecisionError::Conflict(format!(
4146                        "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
4147                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4148                    ))
4149                    .into());
4150                }
4151                Some(link)
4152            }
4153            None => None,
4154        };
4155        let committed_anchor = snapshot
4156            .outcomes
4157            .iter()
4158            .rev()
4159            .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
4160            .cloned();
4161        let committed_anchor_link = match committed_anchor.as_ref() {
4162            Some(anchor)
4163                if current
4164                    .outcome_floor
4165                    .as_ref()
4166                    .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
4167            {
4168                current
4169                    .outcome_floor
4170                    .as_ref()
4171                    .and_then(|floor| floor.committed_anchor_link)
4172            }
4173            Some(anchor) => {
4174                let index = snapshot
4175                    .commit_links
4176                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4177                    .map_err(|_| {
4178                        DecisionError::Conflict(format!(
4179                            "cluster Commit epoch {} has no audited Commit link",
4180                            anchor.epoch
4181                        ))
4182                    })?;
4183                let link = snapshot.commit_links[index];
4184                if link.checkpoint_id != anchor.checkpoint_id {
4185                    return Err(DecisionError::Conflict(format!(
4186                        "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
4187                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4188                    ))
4189                    .into());
4190                }
4191                Some(link)
4192            }
4193            None => None,
4194        };
4195        let floor = AuthorityOutcomeFloor {
4196            deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
4197                .load_or_create_deployment_id()
4198                .await?,
4199            artifact_before_epoch,
4200            authority_before_epoch,
4201            terminal_anchor,
4202            terminal_anchor_link,
4203            committed_anchor,
4204            committed_anchor_link,
4205        };
4206        floor.validate()?;
4207        Ok(floor)
4208    }
4209
4210    async fn compact_cluster_outcome_history_before_append(
4211        &self,
4212        proof: &LeaderProof,
4213        current: &LeaderAuthorityRecord,
4214        snapshot: &ClusterOutcomeAuditSnapshot,
4215    ) -> Result<bool, ClusterCheckpointAuthorityError> {
4216        if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
4217            return Ok(false);
4218        }
4219        let authority_before_epoch = snapshot.terminal_links
4220            [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
4221            .epoch;
4222        let artifact_before_epoch = current
4223            .outcome_floor
4224            .as_ref()
4225            .map_or(0, |floor| floor.artifact_before_epoch);
4226        let floor = self
4227            .build_cluster_outcome_floor(
4228                current,
4229                snapshot,
4230                artifact_before_epoch,
4231                authority_before_epoch,
4232            )
4233            .await?;
4234        // Floor construction may perform remote reads. Publish only if the audited chain and both
4235        // horizons are still the exact authority snapshot used to select the anchors.
4236        let published = self
4237            .load_published_authority_head()
4238            .await?
4239            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4240        let rechecked = &published.record;
4241        if !rechecked.lease.matches_proof(proof) {
4242            return Err(ClusterCheckpointAuthorityError::Fenced);
4243        }
4244        if rechecked.outcome_head != current.outcome_head
4245            || rechecked.commit_head != current.commit_head
4246            || rechecked.outcome_floor != current.outcome_floor
4247        {
4248            return Ok(true);
4249        }
4250
4251        let sequence = rechecked
4252            .lease
4253            .seq
4254            .checked_add(1)
4255            .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4256        let mut lease = rechecked.lease.clone();
4257        lease.seq = sequence;
4258        let mut next = rechecked.preserve_with_lease(lease);
4259        next.outcome_floor = Some(floor.clone());
4260        next.validate()?;
4261        match self
4262            .create_authority_record(Some(&published), &next)
4263            .await?
4264        {
4265            AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4266                self.install_cluster_outcome_audit(
4267                    Self::cluster_outcome_audit_key(&next),
4268                    next.lease.seq,
4269                    Self::outcomes_retained_by_floor(&floor, snapshot),
4270                );
4271                Ok(true)
4272            }
4273            AuthorityCreateOutcome::Contended(winner) => {
4274                if !winner.lease.matches_proof(proof) {
4275                    return Err(ClusterCheckpointAuthorityError::Fenced);
4276                }
4277                if winner.lease.seq <= current.lease.seq {
4278                    return Err(LeaseError::Invalid(
4279                        "outcome history compaction contention did not advance the authority sequence"
4280                            .into(),
4281                    )
4282                    .into());
4283                }
4284                Ok(true)
4285            }
4286        }
4287    }
4288
4289    async fn audited_cluster_outcomes(
4290        &self,
4291    ) -> Result<
4292        (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
4293        ClusterCheckpointAuthorityError,
4294    > {
4295        const AUDIT_RETRIES: usize = 3;
4296        for attempt in 0..AUDIT_RETRIES {
4297            let Some(head) = self.load_record().await? else {
4298                *self.outcome_audit_cache.lock() = None;
4299                return Ok((None, Arc::from([])));
4300            };
4301            match self.cached_audited_cluster_outcomes_from(&head).await {
4302                Err(ClusterCheckpointAuthorityError::Decision(
4303                    DecisionError::InventoryChanged(_),
4304                )) if attempt + 1 < AUDIT_RETRIES => {
4305                    tokio::task::yield_now().await;
4306                }
4307                Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
4308                Err(error) => return Err(error),
4309            }
4310        }
4311        Err(DecisionError::InventoryChanged(
4312            "cluster outcome audit exhausted stability retries".into(),
4313        )
4314        .into())
4315    }
4316
4317    async fn audited_assignment_decisions_from(
4318        &self,
4319        head: &LeaderAuthorityRecord,
4320    ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4321        let mut newest_first = Vec::new();
4322        let floor = head.assignment_decision_floor.as_ref();
4323        let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
4324        let mut stopped_at_anchor = false;
4325        let mut link = head.assignment_decision_head;
4326        let mut traversed = 0;
4327        while let Some(current) = link {
4328            if !consume_live_authority_link(&mut traversed) {
4329                return Err(DecisionError::Conflict(format!(
4330                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
4331                ))
4332                .into());
4333            }
4334            if current.target_version < before_target_version {
4335                if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
4336                    return Err(DecisionError::Conflict(format!(
4337                        "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
4338                    ))
4339                    .into());
4340                }
4341                stopped_at_anchor = true;
4342                break;
4343            }
4344            let record = read_authority_record(self.store.as_ref(), current.sequence)
4345                .await?
4346                .ok_or_else(|| {
4347                    DecisionError::InventoryChanged(format!(
4348                        "assignment decision authority record {} disappeared during audit",
4349                        current.sequence
4350                    ))
4351                })?;
4352            let decision = record.assignment_decision.clone().ok_or_else(|| {
4353                DecisionError::Conflict(format!(
4354                    "assignment decision head version {} points to non-decision authority record {}",
4355                    current.target_version, current.sequence
4356                ))
4357            })?;
4358            if record.assignment_decision_head != Some(current)
4359                || decision.target_version() != current.target_version
4360            {
4361                return Err(DecisionError::Conflict(format!(
4362                    "assignment decision link version {} sequence {} does not match its authority record",
4363                    current.target_version, current.sequence
4364                ))
4365                .into());
4366            }
4367            newest_first.push(decision);
4368            link = record.previous_assignment_decision;
4369        }
4370        if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4371            return Err(DecisionError::Conflict(
4372                "assignment decision chain stopped without its durable retention anchor".into(),
4373            )
4374            .into());
4375        }
4376        newest_first.reverse();
4377        if let Some(pair) = newest_first
4378            .windows(2)
4379            .find(|pair| pair[0].target_version() >= pair[1].target_version())
4380        {
4381            return Err(DecisionError::Conflict(format!(
4382                "assignment decisions regress from version {} to {}",
4383                pair[0].target_version(),
4384                pair[1].target_version()
4385            ))
4386            .into());
4387        }
4388        Ok(newest_first)
4389    }
4390
4391    async fn exact_assignment_decision_link(
4392        &self,
4393        head: &LeaderAuthorityRecord,
4394        decision: &AuthorityAssignmentDecision,
4395    ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4396        if let Some(floor) = head.assignment_decision_floor.as_ref() {
4397            if floor.terminal_anchor.as_ref() == Some(decision) {
4398                return floor.terminal_anchor_link.ok_or_else(|| {
4399                    DecisionError::Conflict(
4400                        "assignment decision floor lost its terminal authority link".into(),
4401                    )
4402                    .into()
4403                });
4404            }
4405        }
4406        let before_target_version = head
4407            .assignment_decision_floor
4408            .as_ref()
4409            .map_or(0, |floor| floor.before_target_version);
4410        let mut link = head.assignment_decision_head;
4411        let mut traversed = 0;
4412        while let Some(current) = link {
4413            if !consume_live_authority_link(&mut traversed) {
4414                return Err(DecisionError::Conflict(format!(
4415                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4416                ))
4417                .into());
4418            }
4419            if current.target_version < before_target_version {
4420                break;
4421            }
4422            let record = read_authority_record(self.store.as_ref(), current.sequence)
4423                .await?
4424                .ok_or_else(|| {
4425                    DecisionError::InventoryChanged(format!(
4426                        "assignment decision authority record {} disappeared during exact lookup",
4427                        current.sequence
4428                    ))
4429                })?;
4430            if record.assignment_decision.as_ref() == Some(decision) {
4431                return Ok(current);
4432            }
4433            link = record.previous_assignment_decision;
4434        }
4435        Err(DecisionError::Conflict(format!(
4436            "assignment decision version {} is not linked from the durable authority head",
4437            decision.target_version()
4438        ))
4439        .into())
4440    }
4441
4442    async fn reject_consumed_checkpoint_assignment(
4443        &self,
4444        current: &LeaderAuthorityRecord,
4445        assignment_fence: &CheckpointAssignmentFence,
4446    ) -> Result<(), ClusterCheckpointAuthorityError> {
4447        let decisions = self.audited_assignment_decisions_from(current).await?;
4448        if let Some(floor) = current.assignment_decision_floor.as_ref() {
4449            // The floor is half-open: only target versions below it were compacted. Equality is
4450            // not evidence that the fence was consumed (notably for a bootstrap assignment),
4451            // while any decision at the boundary remains in `decisions` and is checked below.
4452            if assignment_fence.assignment_version < floor.before_target_version {
4453                return Err(DecisionError::Conflict(format!(
4454                    "checkpoint assignment version {} is below durable assignment-decision floor {}",
4455                    assignment_fence.assignment_version, floor.before_target_version
4456                ))
4457                .into());
4458            }
4459        }
4460        let Some(newest) = decisions.last() else {
4461            return Ok(());
4462        };
4463        if newest.target_version() > assignment_fence.assignment_version {
4464            return Err(DecisionError::Conflict(format!(
4465                "checkpoint assignment version {} was consumed by assignment decision version {}",
4466                assignment_fence.assignment_version,
4467                newest.target_version()
4468            ))
4469            .into());
4470        }
4471        if newest.target_version() == assignment_fence.assignment_version
4472            && newest.materialized_target() != *assignment_fence
4473        {
4474            return Err(DecisionError::Conflict(format!(
4475                "checkpoint assignment version {} does not match its materialized assignment decision",
4476                assignment_fence.assignment_version
4477            ))
4478            .into());
4479        }
4480        Ok(())
4481    }
4482
4483    /// Admit one exact cluster checkpoint attempt before any participant writes artifacts.
4484    ///
4485    /// An identical retry returns the durable inventory. A later attempt cannot begin until the
4486    /// current attempt commits or its aborted artifacts are cleaned exactly.
4487    ///
4488    /// # Errors
4489    /// Fails for a stale proof, foreign deployment, malformed inventory, another active attempt,
4490    /// or object-store failure.
4491    pub async fn begin_cluster_checkpoint_artifacts(
4492        &self,
4493        proof: &LeaderProof,
4494        inventory: CheckpointArtifactInventory,
4495    ) -> Result<CheckpointArtifactInventory, ClusterCheckpointAuthorityError> {
4496        if !proof.is_canonical() {
4497            return Err(ClusterCheckpointAuthorityError::Fenced);
4498        }
4499        inventory.validate().map_err(DecisionError::Conflict)?;
4500        let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
4501            DecisionError::Conflict(
4502                "cluster checkpoint artifact inventory requires an assignment fence".into(),
4503            )
4504        })?;
4505        if assignment_fence.participant_incarnation(proof.owner.node_id)
4506            != Some(proof.owner.boot_id)
4507        {
4508            return Err(DecisionError::Conflict(
4509                "checkpoint artifact admitting leader is outside the assignment fence".into(),
4510            )
4511            .into());
4512        }
4513        let initial = self
4514            .load_record()
4515            .await?
4516            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4517        if !initial.lease.matches_proof(proof) {
4518            return Err(ClusterCheckpointAuthorityError::Fenced);
4519        }
4520        let deployment_id = CheckpointDecisionStore::new(Arc::clone(&self.store))
4521            .load_or_create_deployment_id()
4522            .await?;
4523        if inventory.deployment_id != deployment_id {
4524            return Err(DecisionError::Conflict(format!(
4525                "checkpoint artifact inventory deployment {} does not match authority deployment {deployment_id}",
4526                inventory.deployment_id
4527            ))
4528            .into());
4529        }
4530
4531        loop {
4532            let published = self
4533                .load_published_authority_head()
4534                .await?
4535                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4536            let current = &published.record;
4537            if !current.lease.matches_proof(proof) {
4538                return Err(ClusterCheckpointAuthorityError::Fenced);
4539            }
4540            self.reject_consumed_checkpoint_assignment(current, assignment_fence)
4541                .await?;
4542            current.validate_topology_checkpoint_inventory(&inventory)?;
4543            self.validate_committed_topology_checkpoint_inventory(current, &inventory)
4544                .await?;
4545            if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
4546                if active == &inventory
4547                    && current.active_checkpoint_artifact_leader_proof.as_ref() == Some(proof)
4548                    && current.outcome_head.is_none_or(|head| {
4549                        head.epoch != inventory.attempt.epoch
4550                            || head.checkpoint_id != inventory.attempt.checkpoint_id
4551                    })
4552                {
4553                    return Ok(active.clone());
4554                }
4555                return Err(DecisionError::Conflict(format!(
4556                    "checkpoint {} still has unresolved or inherited cluster artifacts",
4557                    active.attempt.checkpoint_id
4558                ))
4559                .into());
4560            }
4561            if let Some(head) = current.outcome_head {
4562                let terminal =
4563                    crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
4564                if inventory.attempt.relation_to(terminal)
4565                    != crate::checkpoint::CheckpointAttemptRelation::Newer
4566                {
4567                    return Err(DecisionError::Conflict(format!(
4568                        "checkpoint {} does not advance terminal checkpoint {}",
4569                        inventory.attempt.checkpoint_id, head.checkpoint_id
4570                    ))
4571                    .into());
4572                }
4573            }
4574
4575            let sequence =
4576                current.lease.seq.checked_add(1).ok_or_else(|| {
4577                    LeaseError::Invalid("leader authority sequence exhausted".into())
4578                })?;
4579            let mut lease = current.lease.clone();
4580            lease.seq = sequence;
4581            let mut next = current.preserve_with_lease(lease);
4582            next.active_checkpoint_artifacts = Some(inventory.clone());
4583            next.active_checkpoint_artifact_leader_proof = Some(proof.clone());
4584            next.validate()?;
4585            match self
4586                .create_authority_record(Some(&published), &next)
4587                .await?
4588            {
4589                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4590                    return Ok(inventory);
4591                }
4592                AuthorityCreateOutcome::Contended(winner) => {
4593                    if !winner.lease.matches_proof(proof) {
4594                        return Err(ClusterCheckpointAuthorityError::Fenced);
4595                    }
4596                    tokio::task::yield_now().await;
4597                }
4598            }
4599        }
4600    }
4601
4602    /// Clear one exact retained inventory after its durable Abort artifact paths are sealed.
4603    ///
4604    /// # Errors
4605    /// Fails for a stale proof, a different active attempt, a missing matching Abort, malformed
4606    /// inventory, or object-store failure.
4607    pub async fn finish_cluster_checkpoint_artifact_cleanup(
4608        &self,
4609        proof: &LeaderProof,
4610        expected: &CheckpointArtifactInventory,
4611    ) -> Result<(), ClusterCheckpointAuthorityError> {
4612        if !proof.is_canonical() {
4613            return Err(ClusterCheckpointAuthorityError::Fenced);
4614        }
4615        expected.validate().map_err(DecisionError::Conflict)?;
4616        if expected.assignment_fence.is_none() {
4617            return Err(DecisionError::Conflict(
4618                "cluster checkpoint artifact inventory requires an assignment fence".into(),
4619            )
4620            .into());
4621        }
4622
4623        loop {
4624            let published = self
4625                .load_published_authority_head()
4626                .await?
4627                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4628            let current = &published.record;
4629            if !current.lease.matches_proof(proof) {
4630                return Err(ClusterCheckpointAuthorityError::Fenced);
4631            }
4632            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4633            let matching_abort = snapshot.outcomes.iter().any(|outcome| {
4634                matches!(outcome.verdict, CheckpointVerdict::Abort)
4635                    && outcome.deployment_id == expected.deployment_id
4636                    && outcome.epoch == expected.attempt.epoch
4637                    && outcome.checkpoint_id == expected.attempt.checkpoint_id
4638                    && outcome.assignment_fence.as_ref() == expected.assignment_fence.as_ref()
4639            });
4640            match current.active_checkpoint_artifacts.as_ref() {
4641                None if matching_abort => return Ok(()),
4642                None => {
4643                    return Err(DecisionError::Conflict(format!(
4644                        "checkpoint {} has no retained cluster artifact inventory",
4645                        expected.attempt.checkpoint_id
4646                    ))
4647                    .into());
4648                }
4649                Some(active) if active != expected => {
4650                    return Err(DecisionError::Conflict(format!(
4651                        "checkpoint {} still has unresolved cluster artifacts",
4652                        active.attempt.checkpoint_id
4653                    ))
4654                    .into());
4655                }
4656                Some(_) if !matching_abort => {
4657                    return Err(DecisionError::Conflict(format!(
4658                        "checkpoint {} has no matching durable Abort",
4659                        expected.attempt.checkpoint_id
4660                    ))
4661                    .into());
4662                }
4663                Some(_) => {}
4664            }
4665
4666            let sequence =
4667                current.lease.seq.checked_add(1).ok_or_else(|| {
4668                    LeaseError::Invalid("leader authority sequence exhausted".into())
4669                })?;
4670            let mut lease = current.lease.clone();
4671            lease.seq = sequence;
4672            let mut next = current.preserve_with_lease(lease);
4673            next.active_checkpoint_artifacts = None;
4674            next.active_checkpoint_artifact_leader_proof = None;
4675            next.validate()?;
4676            match self
4677                .create_authority_record(Some(&published), &next)
4678                .await?
4679            {
4680                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4681                    return Ok(());
4682                }
4683                AuthorityCreateOutcome::Contended(winner) => {
4684                    if !winner.lease.matches_proof(proof) {
4685                        return Err(ClusterCheckpointAuthorityError::Fenced);
4686                    }
4687                    tokio::task::yield_now().await;
4688                }
4689            }
4690        }
4691    }
4692
4693    /// Admit one cluster terminal outcome through the exact next leader-authority sequence.
4694    ///
4695    /// Renewals, takeovers, catalog seals, floor advances, and other decisions all contend on the
4696    /// same create-only object. An identical retry converges on the durable winner.
4697    ///
4698    /// # Errors
4699    /// Fails closed for a stale proof, non-monotonic or conflicting outcome, malformed committed
4700    /// checkpoint index, or object-store failure.
4701    pub async fn record_cluster_outcome(
4702        &self,
4703        proof: &LeaderProof,
4704        epoch: u64,
4705        checkpoint_id: u64,
4706        assignment_fence: CheckpointAssignmentFence,
4707        verdict: CheckpointVerdict,
4708        committed_checkpoint: Option<CommittedCheckpointRef>,
4709    ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4710        self.record_cluster_outcome_inner(
4711            proof,
4712            epoch,
4713            checkpoint_id,
4714            assignment_fence,
4715            verdict,
4716            committed_checkpoint,
4717            false,
4718        )
4719        .await
4720    }
4721
4722    /// Retire an unused checkpoint reservation only while artifact admission is absent.
4723    /// The same authority append serializes this condition with checkpoint/cut admission.
4724    /// An existing Abort is reusable only if its original append had no admitted artifacts.
4725    ///
4726    /// # Errors
4727    /// Rejects changed leader proof, admitted artifacts, incompatible terminal outcomes or I/O.
4728    pub async fn abort_unadmitted_cluster_checkpoint(
4729        &self,
4730        proof: &LeaderProof,
4731        attempt: crate::checkpoint::CheckpointAttempt,
4732        assignment_fence: CheckpointAssignmentFence,
4733    ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4734        self.record_cluster_outcome_inner(
4735            proof,
4736            attempt.epoch,
4737            attempt.checkpoint_id,
4738            assignment_fence,
4739            CheckpointVerdict::Abort,
4740            None,
4741            true,
4742        )
4743        .await
4744    }
4745
4746    async fn validate_assignment_recovery_snapshot(
4747        &self,
4748        decision: &AssignmentRecoveryDecision,
4749    ) -> Result<(), ClusterCheckpointAuthorityError> {
4750        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4751        let predecessor = snapshots
4752            .load()
4753            .await
4754            .map_err(|error| {
4755                assignment_snapshot_error("load assignment recovery predecessor", error)
4756            })?
4757            .ok_or_else(|| {
4758                LeaseError::Invalid(
4759                    "assignment recovery requires a durable predecessor snapshot".into(),
4760                )
4761            })?;
4762        let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4763            LeaseError::Invalid(format!(
4764                "assignment recovery predecessor has no valid fence: {error}"
4765            ))
4766        })?;
4767        if predecessor.draining || predecessor_fence != decision.predecessor {
4768            return Err(LeaseError::Invalid(
4769                "assignment recovery predecessor is not the exact committed assignment head".into(),
4770            )
4771            .into());
4772        }
4773        let proposal = snapshots
4774            .load_recovery_proposal(&decision.proposal)
4775            .await
4776            .map_err(|error| {
4777                assignment_snapshot_error("load assignment recovery proposal", error)
4778            })?;
4779        let proposal_fence = proposal.assignment_fence().map_err(|error| {
4780            LeaseError::Invalid(format!(
4781                "assignment recovery proposal has no valid fence: {error}"
4782            ))
4783        })?;
4784        if proposal_fence != decision.target {
4785            return Err(LeaseError::Invalid(
4786                "assignment recovery proposal does not match its authorized target fence".into(),
4787            )
4788            .into());
4789        }
4790        Ok(())
4791    }
4792
4793    async fn current_assignment_checkpoint(
4794        &self,
4795        current: &LeaderAuthorityRecord,
4796        predecessor: &CheckpointAssignmentFence,
4797        purpose: &str,
4798    ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
4799        let Some(commit_head) = current.commit_head else {
4800            return Ok(None);
4801        };
4802        let outcome = self
4803            .cluster_outcome_from_snapshot(current, commit_head.epoch)
4804            .await?
4805            .ok_or_else(|| {
4806                DecisionError::Conflict(format!(
4807                    "{purpose} checkpoint {} is not live",
4808                    commit_head.checkpoint_id
4809                ))
4810            })?;
4811        if !outcome.is_commit()
4812            || outcome.checkpoint_id != commit_head.checkpoint_id
4813            || outcome.assignment_fence.as_ref() != Some(predecessor)
4814        {
4815            return Err(DecisionError::Conflict(format!(
4816                "{purpose} checkpoint {} does not bind its predecessor fence",
4817                commit_head.checkpoint_id
4818            ))
4819            .into());
4820        }
4821        let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
4822            DecisionError::Conflict(format!(
4823                "{purpose} checkpoint {} has no committed index reference",
4824                commit_head.checkpoint_id
4825            ))
4826        })?;
4827        let index = CheckpointDecisionStore::new(Arc::clone(&self.store))
4828            .validate_committed_checkpoint_for_outcome(&outcome)
4829            .await?;
4830        if index.assignment_fence.as_ref() != Some(predecessor) {
4831            return Err(DecisionError::Conflict(format!(
4832                "{purpose} checkpoint index {} does not bind its predecessor fence",
4833                commit_head.checkpoint_id
4834            ))
4835            .into());
4836        }
4837        Ok(Some(reference.clone()))
4838    }
4839
4840    async fn validate_current_assignment_checkpoint(
4841        &self,
4842        current: &LeaderAuthorityRecord,
4843        reference: &CommittedCheckpointRef,
4844        predecessor: &CheckpointAssignmentFence,
4845        purpose: &str,
4846    ) -> Result<(), ClusterCheckpointAuthorityError> {
4847        let current_reference = self
4848            .current_assignment_checkpoint(current, predecessor, purpose)
4849            .await?
4850            .ok_or_else(|| {
4851                DecisionError::Conflict(format!("{purpose} has no authoritative checkpoint Commit"))
4852            })?;
4853        if current_reference != *reference {
4854            return Err(DecisionError::Conflict(format!(
4855                "{purpose} checkpoint {} is not the current Commit head {}",
4856                reference.checkpoint_id, current_reference.checkpoint_id
4857            ))
4858            .into());
4859        }
4860        Ok(())
4861    }
4862
4863    fn aborted_handoff_target(
4864        transition: &AssignmentDrainTransition,
4865    ) -> Result<CheckpointAssignmentFence, ClusterCheckpointAuthorityError> {
4866        let mut target = transition.predecessor.clone();
4867        target.assignment_version = transition.target.assignment_version;
4868        if !target.is_canonical() {
4869            return Err(LeaseError::Invalid(
4870                "assignment drain abort produced an invalid handoff target".into(),
4871            )
4872            .into());
4873        }
4874        Ok(target)
4875    }
4876
4877    async fn next_assignment_handoff_pin(
4878        &self,
4879        current: &LeaderAuthorityRecord,
4880        decision: &AuthorityAssignmentDecision,
4881    ) -> Result<Option<AssignmentHandoffPin>, ClusterCheckpointAuthorityError> {
4882        match decision {
4883            AuthorityAssignmentDecision::Drain(drain) => match drain.verdict {
4884                AssignmentDrainVerdict::Commit => {
4885                    if current.assignment_handoff_pin.is_some() {
4886                        return Err(DecisionError::Conflict(
4887                            "assignment decision cannot overtake an unresolved state handoff"
4888                                .into(),
4889                        )
4890                        .into());
4891                    }
4892                    let reference = drain.handoff_checkpoint.as_ref().ok_or_else(|| {
4893                        LeaseError::Invalid(
4894                            "committed assignment drain has no handoff checkpoint".into(),
4895                        )
4896                    })?;
4897                    self.validate_current_assignment_checkpoint(
4898                        current,
4899                        reference,
4900                        &drain.transition.predecessor,
4901                        "assignment drain handoff",
4902                    )
4903                    .await?;
4904                    Ok(Some(AssignmentHandoffPin {
4905                        target: drain.transition.target.clone(),
4906                        checkpoint: reference.clone(),
4907                    }))
4908                }
4909                AssignmentDrainVerdict::Abort => match current.assignment_handoff_pin.as_ref() {
4910                    None => {
4911                        let checkpoint = self
4912                            .current_assignment_checkpoint(
4913                                current,
4914                                &drain.transition.predecessor,
4915                                "assignment drain abort",
4916                            )
4917                            .await?;
4918                        match checkpoint {
4919                            Some(checkpoint) => Ok(Some(AssignmentHandoffPin {
4920                                target: Self::aborted_handoff_target(&drain.transition)?,
4921                                checkpoint,
4922                            })),
4923                            None => Ok(None),
4924                        }
4925                    }
4926                    Some(pin) if pin.target == drain.transition.predecessor => {
4927                        Ok(Some(AssignmentHandoffPin {
4928                            target: Self::aborted_handoff_target(&drain.transition)?,
4929                            checkpoint: pin.checkpoint.clone(),
4930                        }))
4931                    }
4932                    Some(_) => Err(DecisionError::Conflict(
4933                        "assignment decision cannot overtake an unresolved state handoff".into(),
4934                    )
4935                    .into()),
4936                },
4937            },
4938            AuthorityAssignmentDecision::Recovery(recovery) => {
4939                if let Some(pin) = current.assignment_handoff_pin.as_ref() {
4940                    if pin.target != recovery.predecessor
4941                        || pin.checkpoint != recovery.recovery_checkpoint
4942                    {
4943                        return Err(DecisionError::Conflict(
4944                            "assignment recovery does not continue the exact unresolved state handoff"
4945                                .into(),
4946                        )
4947                        .into());
4948                    }
4949                } else {
4950                    self.validate_current_assignment_checkpoint(
4951                        current,
4952                        &recovery.recovery_checkpoint,
4953                        &recovery.predecessor,
4954                        "assignment recovery",
4955                    )
4956                    .await?;
4957                }
4958                Ok(Some(AssignmentHandoffPin {
4959                    target: recovery.target.clone(),
4960                    checkpoint: recovery.recovery_checkpoint.clone(),
4961                }))
4962            }
4963        }
4964    }
4965
4966    async fn record_assignment_decision(
4967        &self,
4968        proof: &LeaderProof,
4969        decision: AuthorityAssignmentDecision,
4970        assignments: Option<&AssignmentSnapshotStore>,
4971    ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
4972        decision.validate()?;
4973        if decision.leader_proof() != proof || !proof.is_canonical() {
4974            return Err(ClusterCheckpointAuthorityError::Fenced);
4975        }
4976
4977        loop {
4978            let published = self
4979                .load_published_authority_head()
4980                .await?
4981                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4982            let current = &published.record;
4983            if !current.lease.matches_proof(proof) {
4984                return Err(ClusterCheckpointAuthorityError::Fenced);
4985            }
4986            if let Some(floor) = current.assignment_decision_floor.as_ref() {
4987                if decision.target_version() < floor.before_target_version {
4988                    return Err(DecisionError::Conflict(format!(
4989                        "assignment decision version {} is below durable retention floor {}",
4990                        decision.target_version(),
4991                        floor.before_target_version
4992                    ))
4993                    .into());
4994                }
4995            }
4996            let decisions = self.audited_assignment_decisions_from(current).await?;
4997            if let Some(winner) = decisions
4998                .iter()
4999                .find(|winner| winner.target_version() == decision.target_version())
5000            {
5001                return if winner == &decision {
5002                    Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5003                        winner.clone(),
5004                    ))
5005                } else {
5006                    Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5007                        winner: winner.clone(),
5008                    })
5009                };
5010            }
5011            self.validate_assignment_topology_settlement(current, &decision, assignments)
5012                .await?;
5013            current.reject_assignment_checkpoint_overtake(&decision)?;
5014            if let Some(last) = decisions.last() {
5015                if decision.target_version() <= last.target_version() {
5016                    return Err(DecisionError::Conflict(format!(
5017                        "assignment decision version {} does not advance durable version {}",
5018                        decision.target_version(),
5019                        last.target_version()
5020                    ))
5021                    .into());
5022                }
5023            }
5024            if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
5025                self.validate_assignment_recovery_snapshot(recovery).await?;
5026            }
5027            let assignment_handoff_pin =
5028                self.next_assignment_handoff_pin(current, &decision).await?;
5029
5030            let base_sequence = current.lease.seq;
5031            let sequence = base_sequence
5032                .checked_add(1)
5033                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5034            let mut candidate = current.preserve_with_lease(LeaderLease {
5035                seq: sequence,
5036                renewal_sequence: current.lease.renewal_sequence,
5037                token: current.lease.token,
5038                owner: current.lease.owner.clone(),
5039                expires_at_ms: current.lease.expires_at_ms,
5040                catalog_manifest: current.lease.catalog_manifest.clone(),
5041            });
5042            candidate.assignment_decision = Some(decision.clone());
5043            candidate.previous_assignment_decision = current.assignment_decision_head;
5044            candidate.assignment_decision_head = Some(AssignmentDecisionLink {
5045                sequence,
5046                target_version: decision.target_version(),
5047            });
5048            candidate.assignment_handoff_pin = assignment_handoff_pin;
5049            candidate.assignment_drain_reservation = None;
5050            candidate.validate()?;
5051
5052            match self
5053                .create_authority_record(Some(&published), &candidate)
5054                .await?
5055            {
5056                AuthorityCreateOutcome::Created => {
5057                    return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
5058                }
5059                AuthorityCreateOutcome::ExistingIdentical => {
5060                    return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
5061                }
5062                AuthorityCreateOutcome::Contended(winner_head) => {
5063                    let winners = self.audited_assignment_decisions_from(&winner_head).await?;
5064                    if let Some(winner) = winners
5065                        .iter()
5066                        .find(|winner| winner.target_version() == decision.target_version())
5067                    {
5068                        return if winner == &decision {
5069                            Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5070                                winner.clone(),
5071                            ))
5072                        } else {
5073                            Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5074                                winner: winner.clone(),
5075                            })
5076                        };
5077                    }
5078                    if !winner_head.lease.matches_proof(proof) {
5079                        return Err(ClusterCheckpointAuthorityError::Fenced);
5080                    }
5081                    if winner_head.lease.seq > base_sequence {
5082                        tokio::task::yield_now().await;
5083                        continue;
5084                    }
5085                    return Err(LeaseError::Invalid(
5086                        "assignment authority contention did not advance the sequence".into(),
5087                    )
5088                    .into());
5089                }
5090            }
5091        }
5092    }
5093
5094    /// Admit one assignment-drain settlement through the exact next authority sequence.
5095    ///
5096    /// Lease renewals, takeovers, checkpoint outcomes, and other decisions contend on that same
5097    /// create-only sequence. An identical retry converges on the durable winner.
5098    ///
5099    /// # Errors
5100    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
5101    pub async fn record_assignment_drain_decision(
5102        &self,
5103        proof: &LeaderProof,
5104        decision: AssignmentDrainDecision,
5105    ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5106        let assignments = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5107        self.record_assignment_drain_decision_with_assignments(proof, decision, &assignments)
5108            .await
5109    }
5110
5111    /// Settle a drain using the cluster's configured, namespace-verified assignment store.
5112    /// An admitted intent is materialized before its reservation can be cleared.
5113    ///
5114    /// # Errors
5115    /// Rejects stale authority, conflicting evidence, or failed materialization/decision I/O.
5116    pub async fn record_assignment_drain_decision_with_assignments(
5117        &self,
5118        proof: &LeaderProof,
5119        decision: AssignmentDrainDecision,
5120        assignments: &AssignmentSnapshotStore,
5121    ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5122        match Box::pin(self.record_assignment_decision(
5123            proof,
5124            AuthorityAssignmentDecision::Drain(decision),
5125            Some(assignments),
5126        ))
5127        .await?
5128        {
5129            RecordAuthorityAssignmentDecisionResult::Created(
5130                AuthorityAssignmentDecision::Drain(decision),
5131            ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
5132            RecordAuthorityAssignmentDecisionResult::Unchanged(
5133                AuthorityAssignmentDecision::Drain(decision),
5134            ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
5135            RecordAuthorityAssignmentDecisionResult::Conflict {
5136                winner: AuthorityAssignmentDecision::Drain(winner),
5137            } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
5138            RecordAuthorityAssignmentDecisionResult::Conflict {
5139                winner: AuthorityAssignmentDecision::Recovery(winner),
5140            } => Err(DecisionError::Conflict(format!(
5141                "assignment recovery decision already settled target version {}",
5142                winner.target_version()
5143            ))
5144            .into()),
5145            RecordAuthorityAssignmentDecisionResult::Created(_)
5146            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5147                "assignment decision recorder returned the wrong decision kind".into(),
5148            )
5149            .into()),
5150        }
5151    }
5152
5153    /// Admit one failure-recovery assignment through the exact next authority sequence.
5154    ///
5155    /// # Errors
5156    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
5157    pub(crate) async fn record_assignment_recovery_decision(
5158        &self,
5159        proof: &LeaderProof,
5160        decision: AssignmentRecoveryDecision,
5161    ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
5162        match Box::pin(self.record_assignment_decision(
5163            proof,
5164            AuthorityAssignmentDecision::Recovery(decision),
5165            None,
5166        ))
5167        .await?
5168        {
5169            RecordAuthorityAssignmentDecisionResult::Created(
5170                AuthorityAssignmentDecision::Recovery(decision),
5171            ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
5172            RecordAuthorityAssignmentDecisionResult::Unchanged(
5173                AuthorityAssignmentDecision::Recovery(decision),
5174            ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
5175            RecordAuthorityAssignmentDecisionResult::Conflict {
5176                winner: AuthorityAssignmentDecision::Recovery(winner),
5177            } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
5178            RecordAuthorityAssignmentDecisionResult::Conflict {
5179                winner: AuthorityAssignmentDecision::Drain(winner),
5180            } => Err(DecisionError::Conflict(format!(
5181                "assignment drain decision already settled target version {}",
5182                winner.target_version()
5183            ))
5184            .into()),
5185            RecordAuthorityAssignmentDecisionResult::Created(_)
5186            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5187                "assignment decision recorder returned the wrong decision kind".into(),
5188            )
5189            .into()),
5190        }
5191    }
5192
5193    async fn assignment_decision(
5194        &self,
5195        target_version: u64,
5196    ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
5197        if target_version == 0 {
5198            return Err(LeaseError::Invalid(
5199                "assignment decision target version must be nonzero".into(),
5200            )
5201            .into());
5202        }
5203        let Some(head) = self.load_record().await? else {
5204            return Ok(None);
5205        };
5206        if let Some(floor) = head.assignment_decision_floor.as_ref() {
5207            if target_version < floor.before_target_version {
5208                return Err(DecisionError::Conflict(format!(
5209                    "assignment decision version {target_version} is below durable retention floor {}",
5210                    floor.before_target_version
5211                ))
5212                .into());
5213            }
5214        }
5215        Ok(self
5216            .audited_assignment_decisions_from(&head)
5217            .await?
5218            .into_iter()
5219            .find(|decision| decision.target_version() == target_version))
5220    }
5221
5222    /// Read the immutable settlement for one exact target assignment version.
5223    ///
5224    /// # Errors
5225    /// Fails closed on malformed or incomplete authority history.
5226    pub async fn assignment_drain_decision(
5227        &self,
5228        target_version: u64,
5229    ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
5230        match self.assignment_decision(target_version).await? {
5231            Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
5232            Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
5233                format!("target version {target_version} was settled by assignment recovery"),
5234            )
5235            .into()),
5236            None => Ok(None),
5237        }
5238    }
5239
5240    /// Read the immutable recovery authorization for one exact target assignment version.
5241    ///
5242    /// # Errors
5243    /// Fails closed on malformed, incomplete, or cross-kind authority history.
5244    pub async fn assignment_recovery_decision(
5245        &self,
5246        target_version: u64,
5247    ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
5248        match self.assignment_decision(target_version).await? {
5249            Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
5250            Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
5251                "target version {target_version} was settled by assignment drain"
5252            ))
5253            .into()),
5254            None => Ok(None),
5255        }
5256    }
5257
5258    /// Read the state handoff checkpoint pinned for one exact target assignment.
5259    ///
5260    /// # Errors
5261    /// Fails when `target` is invalid or the durable authority head is unavailable.
5262    pub async fn assignment_handoff_checkpoint(
5263        &self,
5264        target: &CheckpointAssignmentFence,
5265    ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
5266        if !target.is_canonical() {
5267            return Err(LeaseError::Invalid(
5268                "assignment handoff checkpoint lookup requires a canonical target fence".into(),
5269            )
5270            .into());
5271        }
5272        Ok(self.load_record().await?.and_then(|head| {
5273            head.assignment_handoff_pin
5274                .filter(|pin| pin.target == *target)
5275                .map(|pin| pin.checkpoint)
5276        }))
5277    }
5278
5279    /// Materialize the immutable authority winner for one recovery target version.
5280    ///
5281    /// The caller supplies only the version; an arbitrary staged proposal can never bypass the
5282    /// shared leader-decision chain.
5283    ///
5284    /// # Errors
5285    /// Fails closed when no recovery decision exists, its proposal is invalid, or storage fails.
5286    pub async fn materialize_assignment_recovery(
5287        &self,
5288        target_version: u64,
5289    ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
5290        let decision = self
5291            .assignment_recovery_decision(target_version)
5292            .await?
5293            .ok_or_else(|| {
5294                DecisionError::Conflict(format!(
5295                    "assignment recovery target {target_version} has no authority decision"
5296                ))
5297            })?;
5298        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5299        let proposal = snapshots
5300            .load_recovery_proposal(&decision.proposal)
5301            .await
5302            .map_err(|error| {
5303                assignment_snapshot_error("load authorized assignment recovery proposal", error)
5304            })?;
5305        if proposal.assignment_fence().map_err(|error| {
5306            LeaseError::Invalid(format!(
5307                "authorized assignment recovery proposal has no valid fence: {error}"
5308            ))
5309        })? != decision.target
5310        {
5311            return Err(LeaseError::Invalid(
5312                "authorized assignment recovery proposal does not match its target".into(),
5313            )
5314            .into());
5315        }
5316        snapshots
5317            .materialize_recovery(&decision.proposal)
5318            .await
5319            .map_err(|error| {
5320                assignment_snapshot_error("materialize authorized assignment recovery", error)
5321                    .into()
5322            })
5323    }
5324
5325    /// Advance the shared assignment-decision floor through the exact next authority sequence.
5326    ///
5327    /// The compatibility-shaped API retains both drain and recovery decisions on one ordered
5328    /// chain. The caller must first durably prune assignment snapshots below the same
5329    /// target-version horizon. Decision-bearing authority records below the durable floor then
5330    /// become eligible for best-effort deletion while the exact terminal anchor preserves chain
5331    /// continuity.
5332    ///
5333    /// # Errors
5334    ///
5335    /// Fails for a stale proof, invalid horizon, corrupt history, or storage failure.
5336    pub async fn prune_assignment_drain_decisions_before(
5337        &self,
5338        proof: &LeaderProof,
5339        before_target_version: u64,
5340    ) -> Result<u64, ClusterCheckpointAuthorityError> {
5341        if before_target_version == 0 {
5342            return Ok(self
5343                .load_record()
5344                .await?
5345                .and_then(|head| head.assignment_decision_floor)
5346                .map_or(0, |floor| floor.before_target_version));
5347        }
5348        if !proof.is_canonical() {
5349            return Err(ClusterCheckpointAuthorityError::Fenced);
5350        }
5351        loop {
5352            let published = self
5353                .load_published_authority_head()
5354                .await?
5355                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5356            let current = &published.record;
5357            if !current.lease.matches_proof(proof) {
5358                return Err(ClusterCheckpointAuthorityError::Fenced);
5359            }
5360            if let Some(floor) = current.assignment_decision_floor.as_ref() {
5361                if floor.before_target_version >= before_target_version {
5362                    self.schedule_history_prune();
5363                    return Ok(floor.before_target_version);
5364                }
5365            }
5366
5367            let decisions = self.audited_assignment_decisions_from(current).await?;
5368            let terminal_anchor = decisions
5369                .iter()
5370                .rev()
5371                .find(|decision| decision.target_version() < before_target_version)
5372                .cloned()
5373                .or_else(|| {
5374                    current
5375                        .assignment_decision_floor
5376                        .as_ref()
5377                        .and_then(|floor| floor.terminal_anchor.clone())
5378                });
5379            let terminal_anchor_link = match terminal_anchor.as_ref() {
5380                Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
5381                None => None,
5382            };
5383            let floor = AuthorityAssignmentDecisionFloor {
5384                before_target_version,
5385                terminal_anchor,
5386                terminal_anchor_link,
5387            };
5388            floor.validate()?;
5389
5390            let base_sequence = current.lease.seq;
5391            let sequence = base_sequence
5392                .checked_add(1)
5393                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5394            let mut next = current.preserve_with_lease(LeaderLease {
5395                seq: sequence,
5396                renewal_sequence: current.lease.renewal_sequence,
5397                token: current.lease.token,
5398                owner: current.lease.owner.clone(),
5399                expires_at_ms: current.lease.expires_at_ms,
5400                catalog_manifest: current.lease.catalog_manifest.clone(),
5401            });
5402            next.assignment_decision_floor = Some(floor);
5403            next.validate()?;
5404
5405            match self
5406                .create_authority_record(Some(&published), &next)
5407                .await?
5408            {
5409                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5410                    return Ok(before_target_version);
5411                }
5412                AuthorityCreateOutcome::Contended(winner) => {
5413                    if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
5414                        if winner_floor.before_target_version >= before_target_version {
5415                            self.schedule_history_prune();
5416                            return Ok(winner_floor.before_target_version);
5417                        }
5418                    }
5419                    if !winner.lease.matches_proof(proof) {
5420                        return Err(ClusterCheckpointAuthorityError::Fenced);
5421                    }
5422                    if winner.lease.seq > base_sequence {
5423                        tokio::task::yield_now().await;
5424                        continue;
5425                    }
5426                    return Err(LeaseError::Invalid(
5427                        "assignment decision floor contention did not advance the sequence".into(),
5428                    )
5429                    .into());
5430                }
5431            }
5432        }
5433    }
5434
5435    async fn cluster_outcome_from_snapshot(
5436        &self,
5437        head: &LeaderAuthorityRecord,
5438        epoch: u64,
5439    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5440        let floor = head.outcome_floor.as_ref();
5441        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5442        if epoch < artifact_before_epoch {
5443            return Ok(None);
5444        }
5445        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
5446        let cache_key = Self::cluster_outcome_audit_key(head);
5447        if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
5448            return Ok(snapshot
5449                .outcomes
5450                .binary_search_by_key(&epoch, |outcome| outcome.epoch)
5451                .ok()
5452                .map(|index| snapshot.outcomes[index].clone())
5453                .filter(|outcome| {
5454                    outcome.epoch >= authority_before_epoch
5455                        || outcome.is_commit()
5456                        || floor
5457                            .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
5458                }));
5459        }
5460        if let Some(anchor) = floor
5461            .and_then(|floor| floor.terminal_anchor.as_ref())
5462            .filter(|anchor| anchor.epoch == epoch)
5463        {
5464            let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5465                .load_or_create_deployment_id()
5466                .await?;
5467            if anchor.deployment_id != expected_deployment {
5468                return Err(DecisionError::Conflict(format!(
5469                    "cluster terminal anchor epoch {} does not belong to current deployment {}",
5470                    anchor.epoch, expected_deployment
5471                ))
5472                .into());
5473            }
5474            return Ok(Some(anchor.clone()));
5475        }
5476
5477        let commit_only = epoch < authority_before_epoch;
5478        let before_epoch = if commit_only {
5479            artifact_before_epoch
5480        } else {
5481            authority_before_epoch
5482        };
5483        let mut current = if commit_only {
5484            head.commit_head
5485        } else {
5486            head.outcome_head
5487        };
5488        let expected_anchor = if commit_only {
5489            floor.and_then(|floor| floor.committed_anchor_link)
5490        } else {
5491            floor.and_then(|floor| floor.terminal_anchor_link)
5492        };
5493        let expected_anchor_outcome = if commit_only {
5494            floor.and_then(|floor| floor.committed_anchor.as_ref())
5495        } else {
5496            floor.and_then(|floor| floor.terminal_anchor.as_ref())
5497        };
5498        let mut traversed = 0;
5499        while let Some(link) = current {
5500            if link.epoch < before_epoch {
5501                match (expected_anchor, expected_anchor_outcome) {
5502                    (Some(anchor_link), Some(anchor))
5503                        if link == anchor_link
5504                            && link.epoch == anchor.epoch
5505                            && link.checkpoint_id == anchor.checkpoint_id =>
5506                    {
5507                        return Ok(None)
5508                    }
5509                    (None, None) => return Ok(None),
5510                    _ => {
5511                        return Err(DecisionError::Conflict(format!(
5512                            "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
5513                            if commit_only { "Commit" } else { "terminal outcome" }
5514                        ))
5515                        .into());
5516                    }
5517                }
5518            }
5519            if link.epoch < epoch {
5520                return Ok(None);
5521            }
5522            if !consume_live_authority_link(&mut traversed) {
5523                return Err(DecisionError::Conflict(format!(
5524                    "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
5525                    if commit_only { "Commit" } else { "outcome" }
5526                ))
5527                .into());
5528            }
5529
5530            let admission;
5531            let record = if link.sequence == head.lease.seq {
5532                head
5533            } else {
5534                admission = read_authority_record(self.store.as_ref(), link.sequence)
5535                    .await?
5536                    .ok_or_else(|| {
5537                        DecisionError::InventoryChanged(format!(
5538                            "cluster {} authority record {} disappeared during exact lookup",
5539                            if commit_only { "Commit" } else { "outcome" },
5540                            link.sequence
5541                        ))
5542                    })?;
5543                &admission
5544            };
5545            let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
5546                DecisionError::Conflict(format!(
5547                    "cluster {} head epoch {} points to non-outcome authority record {}",
5548                    if commit_only { "Commit" } else { "outcome" },
5549                    link.epoch,
5550                    link.sequence
5551                ))
5552            })?;
5553            let record_head = if commit_only {
5554                record.commit_head
5555            } else {
5556                record.outcome_head
5557            };
5558            if record_head != Some(link)
5559                || (commit_only && !outcome.is_commit())
5560                || outcome.epoch != link.epoch
5561                || outcome.checkpoint_id != link.checkpoint_id
5562            {
5563                return Err(DecisionError::Conflict(format!(
5564                    "cluster {} link epoch {} sequence {} does not match its authority record",
5565                    if commit_only { "Commit" } else { "outcome" },
5566                    link.epoch,
5567                    link.sequence
5568                ))
5569                .into());
5570            }
5571            if outcome.epoch == epoch {
5572                let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5573                    .load_or_create_deployment_id()
5574                    .await?;
5575                if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
5576                    || outcome.deployment_id != expected_deployment
5577                {
5578                    return Err(DecisionError::Conflict(format!(
5579                        "cluster outcome epoch {} does not belong to current deployment {}",
5580                        outcome.epoch, expected_deployment
5581                    ))
5582                    .into());
5583                }
5584                return Ok(Some(outcome.clone()));
5585            }
5586            current = if commit_only {
5587                record.previous_commit
5588            } else {
5589                record.previous_outcome
5590            };
5591        }
5592        if expected_anchor.is_some() {
5593            return Err(DecisionError::Conflict(format!(
5594                "cluster {} chain stopped without its durable floor anchor",
5595                if commit_only { "Commit" } else { "outcome" }
5596            ))
5597            .into());
5598        }
5599        Ok(None)
5600    }
5601
5602    async fn stable_cluster_outcome(
5603        &self,
5604        epoch: u64,
5605    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5606        const LOOKUP_RETRIES: usize = 3;
5607        for attempt in 0..LOOKUP_RETRIES {
5608            let Some(head) = self.load_record().await? else {
5609                return Ok(None);
5610            };
5611            match self.cluster_outcome_from_snapshot(&head, epoch).await {
5612                Err(ClusterCheckpointAuthorityError::Decision(
5613                    DecisionError::InventoryChanged(_),
5614                )) if attempt + 1 < LOOKUP_RETRIES => {
5615                    tokio::task::yield_now().await;
5616                }
5617                result => return result,
5618            }
5619        }
5620        Err(DecisionError::InventoryChanged(
5621            "cluster outcome exact lookup exhausted stability retries".into(),
5622        )
5623        .into())
5624    }
5625
5626    /// Read one live cluster outcome from the shared authority.
5627    ///
5628    /// # Errors
5629    ///
5630    /// Fails when the durable authority history is unavailable or invalid.
5631    pub async fn cluster_outcome(
5632        &self,
5633        epoch: u64,
5634    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5635        self.stable_cluster_outcome(epoch).await
5636    }
5637
5638    /// Store one immutable committed-checkpoint index before publishing its Commit outcome.
5639    ///
5640    /// # Errors
5641    ///
5642    /// Fails when the durable authority cannot create the immutable index.
5643    pub async fn create_committed_checkpoint(
5644        &self,
5645        index: &CommittedCheckpointIndex,
5646    ) -> Result<CommittedCheckpointRef, ClusterCheckpointAuthorityError> {
5647        CheckpointDecisionStore::new(Arc::clone(&self.store))
5648            .create_committed_checkpoint(index)
5649            .await
5650            .map_err(Into::into)
5651    }
5652
5653    /// Load one exact content-addressed committed-checkpoint index.
5654    ///
5655    /// # Errors
5656    ///
5657    /// Fails when the durable authority cannot load or validate the index.
5658    pub async fn load_committed_checkpoint(
5659        &self,
5660        reference: &CommittedCheckpointRef,
5661    ) -> Result<CommittedCheckpointIndex, ClusterCheckpointAuthorityError> {
5662        CheckpointDecisionStore::new(Arc::clone(&self.store))
5663            .load_committed_checkpoint(reference)
5664            .await
5665            .map_err(Into::into)
5666    }
5667
5668    /// Read one live cluster outcome together with its committed checkpoint index.
5669    /// Commit always returns a validated index; Abort returns `None`.
5670    ///
5671    /// # Errors
5672    ///
5673    /// Fails when the authority history or selected committed index is unavailable or invalid.
5674    pub async fn cluster_outcome_with_committed_checkpoint(
5675        &self,
5676        epoch: u64,
5677    ) -> Result<
5678        Option<(CheckpointOutcome, Option<CommittedCheckpointIndex>)>,
5679        ClusterCheckpointAuthorityError,
5680    > {
5681        let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5682        let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5683            return Ok(None);
5684        };
5685        let index = if outcome.is_commit() {
5686            let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
5687                DecisionError::Conflict(format!(
5688                    "cluster Commit epoch {} checkpoint {} has no committed checkpoint index",
5689                    outcome.epoch, outcome.checkpoint_id
5690                ))
5691            })?;
5692            let index = decisions.load_committed_checkpoint(reference).await?;
5693            if index.epoch != outcome.epoch
5694                || index.checkpoint_id != outcome.checkpoint_id
5695                || index.scope != outcome.scope
5696                || index.assignment_fence.as_ref() != outcome.assignment_fence.as_ref()
5697                || index.deployment_id != outcome.deployment_id
5698            {
5699                return Err(DecisionError::Conflict(format!(
5700                    "cluster Commit epoch {} checkpoint {} does not match committed checkpoint '{}'",
5701                    outcome.epoch, outcome.checkpoint_id, reference.sha256
5702                ))
5703                .into());
5704            }
5705            Some(index)
5706        } else {
5707            None
5708        };
5709        Ok(Some((outcome, index)))
5710    }
5711
5712    /// Audit and return every live cluster outcome in ascending epoch order.
5713    ///
5714    /// # Errors
5715    ///
5716    /// Fails when the durable authority history is unavailable or invalid.
5717    pub async fn cluster_outcome_inventory(
5718        &self,
5719    ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5720        let (head, outcomes) = self.audited_cluster_outcomes().await?;
5721        Ok(Self::cluster_outcome_inventory_from_audit(
5722            head.as_ref(),
5723            &outcomes,
5724        ))
5725    }
5726
5727    fn cluster_outcome_inventory_from_audit(
5728        head: Option<&LeaderAuthorityRecord>,
5729        outcomes: &[CheckpointOutcome],
5730    ) -> ClusterOutcomeInventory {
5731        let floor = head.and_then(|head| head.outcome_floor.as_ref());
5732        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5733        ClusterOutcomeInventory {
5734            outcomes: outcomes
5735                .iter()
5736                .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5737                .cloned()
5738                .collect(),
5739            retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5740        }
5741    }
5742
5743    /// Audit and return every live cluster outcome in ascending epoch order.
5744    ///
5745    /// # Errors
5746    ///
5747    /// Fails when the durable authority history is unavailable or invalid.
5748    pub async fn cluster_outcomes(
5749        &self,
5750    ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5751        Ok(self.cluster_outcome_inventory().await?.outcomes)
5752    }
5753
5754    /// Greatest live cluster commit recovery cut.
5755    ///
5756    /// # Errors
5757    ///
5758    /// Fails when the durable authority history is unavailable or invalid.
5759    pub async fn highest_cluster_committed_outcome(
5760        &self,
5761    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5762        Ok(self
5763            .cluster_outcomes()
5764            .await?
5765            .into_iter()
5766            .rev()
5767            .find(CheckpointOutcome::is_commit))
5768    }
5769
5770    /// Greatest terminal cluster outcome, including the compacted continuity anchor.
5771    ///
5772    /// # Errors
5773    ///
5774    /// Fails when the durable authority history is unavailable or invalid.
5775    pub async fn highest_cluster_terminal_outcome(
5776        &self,
5777    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5778        Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5779    }
5780
5781    fn cleanup_participant_ids(
5782        index: &CommittedCheckpointIndex,
5783    ) -> Result<Vec<u64>, ClusterCheckpointAuthorityError> {
5784        index.validate().map_err(DecisionError::Conflict)?;
5785        let participant_ids = index
5786            .participants
5787            .iter()
5788            .map(|participant| participant.participant_id)
5789            .collect::<Vec<_>>();
5790        if participant_ids.is_empty()
5791            || participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
5792            || participant_ids[0] == 0
5793            || !participant_ids.windows(2).all(|pair| pair[0] < pair[1])
5794        {
5795            return Err(DecisionError::Conflict(
5796                "committed checkpoint participants are not canonical".into(),
5797            )
5798            .into());
5799        }
5800        Ok(participant_ids)
5801    }
5802
5803    fn cleanup_stop_before(head: &LeaderAuthorityRecord) -> Option<CommittedCheckpointRef> {
5804        head.outcome_floor
5805            .as_ref()
5806            .filter(|floor| floor.artifact_before_epoch != 0)
5807            .and_then(|floor| floor.committed_anchor.as_ref())
5808            .and_then(|outcome| outcome.committed_checkpoint.clone())
5809            .into_iter()
5810            .chain(head.topology_operations.iter().filter_map(|operation| {
5811                operation.has_target_commit().then_some(
5812                    operation
5813                        .cut
5814                        .as_ref()?
5815                        .committed
5816                        .as_ref()?
5817                        .checkpoint
5818                        .clone(),
5819                )
5820            }))
5821            .max_by_key(|reference| (reference.epoch, reference.checkpoint_id))
5822    }
5823
5824    /// Read the exact cluster checkpoint artifact cleanup position.
5825    ///
5826    /// The returned cursor was admitted through the shared leader-authority sequence. `None`
5827    /// means no destructive checkpoint cleanup is currently authorized.
5828    ///
5829    /// # Errors
5830    /// Fails when the durable authority head is unavailable or invalid.
5831    pub async fn cluster_artifact_cleanup(
5832        &self,
5833    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
5834        Ok(self
5835            .load_record()
5836            .await?
5837            .and_then(|head| head.artifact_cleanup))
5838    }
5839
5840    /// Atomically advance the artifact floor and authorize its first expired checkpoint.
5841    ///
5842    /// `protected` must name a live cluster Commit. Its exact predecessor becomes the first
5843    /// cleanup target. An already covered horizon or a protected genesis cut returns `None`.
5844    /// An identical retry returns the active cursor; a different active segment is rejected.
5845    ///
5846    /// # Errors
5847    /// Fails for a stale proof, invalid committed-index chain, failed protected-artifact
5848    /// validation, concurrent cleanup segment, or object-store failure.
5849    pub async fn begin_cluster_artifact_cleanup<V, Fut>(
5850        &self,
5851        proof: &LeaderProof,
5852        protected: CommittedCheckpointRef,
5853        validate_artifacts: V,
5854    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError>
5855    where
5856        V: Fn(CheckpointOutcome) -> Fut + Send + Sync + 'static,
5857        Fut: std::future::Future<Output = Result<(), String>> + Send + 'static,
5858    {
5859        if !proof.is_canonical() {
5860            return Err(ClusterCheckpointAuthorityError::Fenced);
5861        }
5862        protected.validate().map_err(DecisionError::Conflict)?;
5863        loop {
5864            let published = self
5865                .load_published_authority_head()
5866                .await?
5867                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5868            let current = &published.record;
5869            if !current.lease.matches_proof(proof) {
5870                return Err(ClusterCheckpointAuthorityError::Fenced);
5871            }
5872            if let Some(active) = current.artifact_cleanup.as_ref() {
5873                if active.protected == protected {
5874                    return Ok(Some(active.clone()));
5875                }
5876                return Err(DecisionError::Conflict(format!(
5877                    "cluster artifact cleanup for protected checkpoint {} is still active",
5878                    active.protected.checkpoint_id
5879                ))
5880                .into());
5881            }
5882            if current.cleanup_is_pinned(&protected) {
5883                return Ok(None);
5884            }
5885
5886            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
5887            let protected_outcome = snapshot
5888                .outcomes
5889                .iter()
5890                .find(|outcome| {
5891                    outcome.is_commit() && outcome.committed_checkpoint.as_ref() == Some(&protected)
5892                })
5893                .cloned()
5894                .ok_or_else(|| {
5895                    DecisionError::Conflict(format!(
5896                        "protected checkpoint {} is not a live cluster Commit",
5897                        protected.checkpoint_id
5898                    ))
5899                })?;
5900            let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5901            let protected_index = decisions
5902                .validate_committed_checkpoint_for_outcome(&protected_outcome)
5903                .await?;
5904            validate_artifacts(protected_outcome.clone())
5905                .await
5906                .map_err(|error| {
5907                    DecisionError::Conflict(format!(
5908                        "protected cluster checkpoint {} failed durable artifact preflight: {error}",
5909                        protected.checkpoint_id
5910                    ))
5911                })?;
5912            let Some(expired) = protected_index.predecessor.clone() else {
5913                return Ok(None);
5914            };
5915            let stop_before = Self::cleanup_stop_before(current);
5916            if stop_before
5917                .as_ref()
5918                .is_some_and(|stop| expired.epoch <= stop.epoch)
5919            {
5920                return Ok(None);
5921            }
5922            // Retained roots remain metadata/state pins, even after ordinary target cuts expire.
5923            self.audit_topology_cleanup_roots().await?;
5924            let expired_index = decisions.load_committed_checkpoint(&expired).await?;
5925            protected_index
5926                .validate_predecessor_index(&expired_index)
5927                .map_err(DecisionError::Conflict)?;
5928
5929            let authority_before_epoch = current
5930                .outcome_floor
5931                .as_ref()
5932                .map_or(protected.epoch, |floor| {
5933                    floor.authority_before_epoch.max(protected.epoch)
5934                });
5935            let floor = self
5936                .build_cluster_outcome_floor(
5937                    current,
5938                    &snapshot,
5939                    protected.epoch,
5940                    authority_before_epoch,
5941                )
5942                .await?;
5943            if floor
5944                .committed_anchor
5945                .as_ref()
5946                .and_then(|outcome| outcome.committed_checkpoint.as_ref())
5947                != Some(&expired)
5948            {
5949                return Err(DecisionError::Conflict(
5950                    "protected checkpoint predecessor is not the artifact-floor Commit anchor"
5951                        .into(),
5952                )
5953                .into());
5954            }
5955            let cursor = ClusterArtifactCleanupCursor {
5956                protected: protected.clone(),
5957                current: expired,
5958                next: expired_index.predecessor.clone(),
5959                stop_before,
5960                participant_ids: Self::cleanup_participant_ids(&expired_index)?,
5961                phase: ClusterArtifactCleanupPhase::DeleteData,
5962            };
5963            cursor.validate()?;
5964
5965            // Artifact preflight and index reads may race harmless authority mutations. Publish
5966            // only from the exact outcome/floor/cursor snapshot used to derive this segment.
5967            let rechecked = self
5968                .load_published_authority_head()
5969                .await?
5970                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5971            if !rechecked.record.lease.matches_proof(proof) {
5972                return Err(ClusterCheckpointAuthorityError::Fenced);
5973            }
5974            if rechecked.record.outcome_head != current.outcome_head
5975                || rechecked.record.commit_head != current.commit_head
5976                || rechecked.record.outcome_floor != current.outcome_floor
5977                || rechecked.record.artifact_cleanup != current.artifact_cleanup
5978                || rechecked.record.assignment_handoff_pin != current.assignment_handoff_pin
5979                || rechecked.record.topology_operations != current.topology_operations
5980            {
5981                tokio::task::yield_now().await;
5982                continue;
5983            }
5984            let sequence =
5985                rechecked.record.lease.seq.checked_add(1).ok_or_else(|| {
5986                    LeaseError::Invalid("leader authority sequence exhausted".into())
5987                })?;
5988            let mut lease = rechecked.record.lease.clone();
5989            lease.seq = sequence;
5990            let mut next = rechecked.record.preserve_with_lease(lease);
5991            next.outcome_floor = Some(floor.clone());
5992            next.artifact_cleanup = Some(cursor.clone());
5993            next.validate()?;
5994            match self
5995                .create_authority_record(Some(&rechecked), &next)
5996                .await?
5997            {
5998                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5999                    self.install_cluster_outcome_audit(
6000                        Self::cluster_outcome_audit_key(&next),
6001                        next.lease.seq,
6002                        Self::outcomes_retained_by_floor(&floor, &snapshot),
6003                    );
6004                    return Ok(Some(cursor));
6005                }
6006                AuthorityCreateOutcome::Contended(winner) => {
6007                    if !winner.lease.matches_proof(proof) {
6008                        return Err(ClusterCheckpointAuthorityError::Fenced);
6009                    }
6010                    tokio::task::yield_now().await;
6011                }
6012            }
6013        }
6014    }
6015
6016    /// Durably authorize metadata deletion after the exact current data cleanup completes.
6017    ///
6018    /// # Errors
6019    /// Fails for a stale proof, stale or non-`DeleteData` cursor, or object-store failure.
6020    pub async fn mark_cluster_artifact_data_deleted(
6021        &self,
6022        proof: &LeaderProof,
6023        expected: &ClusterArtifactCleanupCursor,
6024    ) -> Result<ClusterArtifactCleanupCursor, ClusterCheckpointAuthorityError> {
6025        if !proof.is_canonical() {
6026            return Err(ClusterCheckpointAuthorityError::Fenced);
6027        }
6028        expected.validate()?;
6029        if expected.phase != ClusterArtifactCleanupPhase::DeleteData {
6030            return Err(DecisionError::Conflict(
6031                "cluster artifact data transition requires a DeleteData cursor".into(),
6032            )
6033            .into());
6034        }
6035        let mut replacement = expected.clone();
6036        replacement.phase = ClusterArtifactCleanupPhase::DeleteMetadata;
6037        loop {
6038            let published = self
6039                .load_published_authority_head()
6040                .await?
6041                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6042            let current = &published.record;
6043            if !current.lease.matches_proof(proof) {
6044                return Err(ClusterCheckpointAuthorityError::Fenced);
6045            }
6046            if current.artifact_cleanup.as_ref() == Some(&replacement) {
6047                return Ok(replacement);
6048            }
6049            if current.artifact_cleanup.as_ref() != Some(expected) {
6050                return Err(DecisionError::Conflict(
6051                    "cluster artifact cleanup cursor changed before data completion".into(),
6052                )
6053                .into());
6054            }
6055            let sequence =
6056                current.lease.seq.checked_add(1).ok_or_else(|| {
6057                    LeaseError::Invalid("leader authority sequence exhausted".into())
6058                })?;
6059            let mut lease = current.lease.clone();
6060            lease.seq = sequence;
6061            let mut next = current.preserve_with_lease(lease);
6062            next.artifact_cleanup = Some(replacement.clone());
6063            next.validate()?;
6064            match self
6065                .create_authority_record(Some(&published), &next)
6066                .await?
6067            {
6068                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6069                    return Ok(replacement);
6070                }
6071                AuthorityCreateOutcome::Contended(winner) => {
6072                    if !winner.lease.matches_proof(proof) {
6073                        return Err(ClusterCheckpointAuthorityError::Fenced);
6074                    }
6075                    tokio::task::yield_now().await;
6076                }
6077            }
6078        }
6079    }
6080
6081    /// Complete metadata deletion and authorize the exact next expired checkpoint, if any.
6082    ///
6083    /// The transition clears the journal at genesis or immediately before the previously
6084    /// reclaimed stop boundary. Otherwise it loads the immutable next index and records its exact
6085    /// predecessor and participants before any deletion of that target is allowed.
6086    ///
6087    /// # Errors
6088    /// Fails for a stale proof, stale or non-`DeleteMetadata` cursor, a broken predecessor chain,
6089    /// or object-store failure.
6090    pub async fn mark_cluster_artifact_metadata_deleted(
6091        &self,
6092        proof: &LeaderProof,
6093        expected: &ClusterArtifactCleanupCursor,
6094    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
6095        if !proof.is_canonical() {
6096            return Err(ClusterCheckpointAuthorityError::Fenced);
6097        }
6098        expected.validate()?;
6099        if expected.phase != ClusterArtifactCleanupPhase::DeleteMetadata {
6100            return Err(DecisionError::Conflict(
6101                "cluster artifact metadata transition requires a DeleteMetadata cursor".into(),
6102            )
6103            .into());
6104        }
6105        loop {
6106            let published = self
6107                .load_published_authority_head()
6108                .await?
6109                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6110            let current = &published.record;
6111            if !current.lease.matches_proof(proof) {
6112                return Err(ClusterCheckpointAuthorityError::Fenced);
6113            }
6114            if current.artifact_cleanup.as_ref() != Some(expected) {
6115                let completed = expected.next.is_none()
6116                    || expected.next.as_ref() == expected.stop_before.as_ref();
6117                if completed && current.artifact_cleanup.is_none() {
6118                    return Ok(None);
6119                }
6120                if let (Some(next_reference), Some(actual)) =
6121                    (expected.next.as_ref(), current.artifact_cleanup.as_ref())
6122                {
6123                    if actual.phase == ClusterArtifactCleanupPhase::DeleteData
6124                        && actual.protected == expected.protected
6125                        && &actual.current == next_reference
6126                        && actual.stop_before == expected.stop_before
6127                    {
6128                        return Ok(Some(actual.clone()));
6129                    }
6130                }
6131                return Err(DecisionError::Conflict(
6132                    "cluster artifact cleanup cursor changed before metadata completion".into(),
6133                )
6134                .into());
6135            }
6136
6137            let replacement = match expected.next.as_ref() {
6138                None => None,
6139                Some(next_reference) if Some(next_reference) == expected.stop_before.as_ref() => {
6140                    None
6141                }
6142                Some(next_reference) => {
6143                    let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6144                    let index = decisions.load_committed_checkpoint(next_reference).await?;
6145                    if index.scope != CheckpointScope::Cluster {
6146                        return Err(DecisionError::Conflict(
6147                            "cluster artifact cleanup next checkpoint is not cluster-scoped".into(),
6148                        )
6149                        .into());
6150                    }
6151                    let replacement = ClusterArtifactCleanupCursor {
6152                        protected: expected.protected.clone(),
6153                        current: next_reference.clone(),
6154                        next: index.predecessor.clone(),
6155                        stop_before: expected.stop_before.clone(),
6156                        participant_ids: Self::cleanup_participant_ids(&index)?,
6157                        phase: ClusterArtifactCleanupPhase::DeleteData,
6158                    };
6159                    replacement.validate()?;
6160                    Some(replacement)
6161                }
6162            };
6163
6164            let sequence =
6165                current.lease.seq.checked_add(1).ok_or_else(|| {
6166                    LeaseError::Invalid("leader authority sequence exhausted".into())
6167                })?;
6168            let mut lease = current.lease.clone();
6169            lease.seq = sequence;
6170            let mut next = current.preserve_with_lease(lease);
6171            next.artifact_cleanup = replacement.clone();
6172            next.validate()?;
6173            match self
6174                .create_authority_record(Some(&published), &next)
6175                .await?
6176            {
6177                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6178                    return Ok(replacement);
6179                }
6180                AuthorityCreateOutcome::Contended(winner) => {
6181                    if !winner.lease.matches_proof(proof) {
6182                        return Err(ClusterCheckpointAuthorityError::Fenced);
6183                    }
6184                    tokio::task::yield_now().await;
6185                }
6186            }
6187        }
6188    }
6189
6190    /// Exact continuity boundary for cluster outcomes compacted from the authority history.
6191    ///
6192    /// # Errors
6193    ///
6194    /// Fails when the durable authority history is unavailable or invalid.
6195    pub async fn cluster_outcome_retention_boundary(
6196        &self,
6197    ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
6198        let head = self.load_record().await?;
6199        Ok(ClusterOutcomeRetentionBoundary::from_floor(
6200            head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
6201        ))
6202    }
6203
6204    /// Acquire an empty authority as a new term, or rotate the fencing token when this exact
6205    /// process incarnation already owns the durable head. Rival wall clocks never authorize
6206    /// takeover.
6207    ///
6208    /// # Errors
6209    /// Fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
6210    pub async fn begin_new_term(
6211        &self,
6212        owner: &LeaderLeaseOwner,
6213        now_ms: i64,
6214    ) -> Result<LeaseOutcome, LeaseError> {
6215        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6216            owner,
6217            now_ms,
6218            SameOwnerToken::Rotate,
6219        ))
6220        .await
6221    }
6222
6223    /// Renew only the exact durable authority term identified by `token`.
6224    ///
6225    /// # Errors
6226    /// Returns [`LeaseError::Fenced`] if the durable head is absent or belongs to another owner or
6227    /// term. Also fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
6228    pub async fn renew_exact(
6229        &self,
6230        owner: &LeaderLeaseOwner,
6231        token: u64,
6232        now_ms: i64,
6233    ) -> Result<LeaseOutcome, LeaseError> {
6234        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6235            owner,
6236            now_ms,
6237            SameOwnerToken::Exact(token),
6238        ))
6239        .await
6240    }
6241
6242    #[cfg(test)]
6243    pub(crate) async fn acquire_or_renew_current_term_for_test(
6244        &self,
6245        owner: &LeaderLeaseOwner,
6246        now_ms: i64,
6247    ) -> Result<LeaseOutcome, LeaseError> {
6248        self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
6249            .await
6250    }
6251
6252    async fn acquire_or_renew_current_term_for_test_inner(
6253        &self,
6254        owner: &LeaderLeaseOwner,
6255        now_ms: i64,
6256        same_owner_token: SameOwnerToken,
6257    ) -> Result<LeaseOutcome, LeaseError> {
6258        owner.validate()?;
6259        self.ttl()?;
6260        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6261        loop {
6262            let published = self.load_published_authority_head().await?;
6263            let candidate = match published.as_ref() {
6264                None => {
6265                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6266                        return Err(LeaseError::Fenced(
6267                            "exact leader renewal lost the durable authority head".into(),
6268                        ));
6269                    }
6270                    LeaderAuthorityRecord::initial(LeaderLease {
6271                        seq: 1,
6272                        renewal_sequence: 1,
6273                        token: 1,
6274                        owner: owner.clone(),
6275                        expires_at_ms,
6276                        catalog_manifest: None,
6277                    })
6278                }
6279                Some(head) if head.record.lease.owner == *owner => {
6280                    let record = &head.record;
6281                    let token = match same_owner_token {
6282                        #[cfg(test)]
6283                        SameOwnerToken::Preserve => record.lease.token,
6284                        SameOwnerToken::Rotate => {
6285                            record.lease.token.checked_add(1).ok_or_else(|| {
6286                                LeaseError::Invalid("leader fencing token exhausted".into())
6287                            })?
6288                        }
6289                        SameOwnerToken::Exact(expected) if expected == record.lease.token => {
6290                            expected
6291                        }
6292                        SameOwnerToken::Exact(_) => {
6293                            return Err(LeaseError::Fenced(
6294                                "exact leader renewal was superseded by a newer local term".into(),
6295                            ));
6296                        }
6297                    };
6298                    let lease = LeaderLease {
6299                        seq: record.lease.seq.checked_add(1).ok_or_else(|| {
6300                            LeaseError::Invalid("lease sequence exhausted".into())
6301                        })?,
6302                        renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
6303                            || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6304                        )?,
6305                        token,
6306                        owner: owner.clone(),
6307                        expires_at_ms,
6308                        catalog_manifest: record.lease.catalog_manifest.clone(),
6309                    };
6310                    record.preserve_with_lease(lease)
6311                }
6312                Some(head) => {
6313                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6314                        return Err(LeaseError::Fenced(
6315                            "exact leader renewal was superseded by a rival owner".into(),
6316                        ));
6317                    }
6318                    return Ok(LeaseOutcome::Held(head.record.lease.clone()));
6319                }
6320            };
6321            match self
6322                .create_authority_record(published.as_ref(), &candidate)
6323                .await?
6324            {
6325                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6326                    return Ok(LeaseOutcome::Acquired(candidate.lease));
6327                }
6328                AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
6329                    if matches!(
6330                        same_owner_token,
6331                        SameOwnerToken::Exact(expected) if winner.lease.token != expected
6332                    ) {
6333                        return Err(LeaseError::Fenced(
6334                            "exact leader renewal lost a same-owner CAS race".into(),
6335                        ));
6336                    }
6337                    tokio::task::yield_now().await;
6338                }
6339                AuthorityCreateOutcome::Contended(winner) => {
6340                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6341                        return Err(LeaseError::Fenced(
6342                            "exact leader renewal lost a rival-owner CAS race".into(),
6343                        ));
6344                    }
6345                    return Ok(LeaseOutcome::Held(winner.lease));
6346                }
6347            }
6348        }
6349    }
6350
6351    /// Start a candidate-local observation of a rival durable liveness identity.
6352    ///
6353    /// # Errors
6354    /// Rejects malformed state or an observation of the candidate itself.
6355    pub fn observe_rival(
6356        &self,
6357        owner: &LeaderLeaseOwner,
6358        lease: &LeaderLease,
6359    ) -> Result<LeaderLeaseObservation, LeaseError> {
6360        owner.validate()?;
6361        lease.validate()?;
6362        self.ttl()?;
6363        if lease.owner == *owner {
6364            return Err(LeaseError::Invalid(
6365                "leader takeover observation must belong to a rival".into(),
6366            ));
6367        }
6368        Ok(LeaderLeaseObservation {
6369            lease: lease.clone(),
6370            started: Instant::now(),
6371        })
6372    }
6373
6374    /// Take over only after the rival's owner, fencing token, and renewal sequence remained
6375    /// current for a full TTL on the candidate's monotonic clock, or once the rival released its
6376    /// term.
6377    ///
6378    /// # Errors
6379    /// Fails closed on invalid state, I/O, or arithmetic exhaustion.
6380    pub async fn try_takeover(
6381        &self,
6382        owner: &LeaderLeaseOwner,
6383        observation: &LeaderLeaseObservation,
6384        now_ms: i64,
6385    ) -> Result<LeaseOutcome, LeaseError> {
6386        owner.validate()?;
6387        observation.lease.validate()?;
6388        let ttl = self.ttl()?;
6389        if observation.lease.owner == *owner {
6390            return Err(LeaseError::Invalid(
6391                "leader takeover observation must belong to a rival".into(),
6392            ));
6393        }
6394        let observed_for_ttl = observation.started.elapsed() >= ttl;
6395        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6396        loop {
6397            let published = self
6398                .load_published_authority_head()
6399                .await?
6400                .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
6401            let current = &published.record;
6402            let unrenewed =
6403                observed_for_ttl && current.lease.has_same_liveness_identity(&observation.lease);
6404            // A rival fences its local grant before it publishes a release, so a released term
6405            // needs no observation.
6406            if !unrenewed && !self.term_released(current).await? {
6407                return Ok(LeaseOutcome::Held(current.lease.clone()));
6408            }
6409            let candidate_lease = LeaderLease {
6410                seq: current
6411                    .lease
6412                    .seq
6413                    .checked_add(1)
6414                    .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
6415                renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
6416                    || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6417                )?,
6418                token: current
6419                    .lease
6420                    .token
6421                    .checked_add(1)
6422                    .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
6423                owner: owner.clone(),
6424                expires_at_ms,
6425                catalog_manifest: current.lease.catalog_manifest.clone(),
6426            };
6427            let candidate = current.preserve_with_lease(candidate_lease);
6428            match self
6429                .create_authority_record(Some(&published), &candidate)
6430                .await?
6431            {
6432                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6433                    return Ok(LeaseOutcome::Acquired(candidate.lease));
6434                }
6435                AuthorityCreateOutcome::Contended(winner)
6436                    if winner.lease.has_same_liveness_identity(&observation.lease) =>
6437                {
6438                    tokio::task::yield_now().await;
6439                }
6440                AuthorityCreateOutcome::Contended(winner) => {
6441                    return Ok(LeaseOutcome::Held(winner.lease));
6442                }
6443            }
6444        }
6445    }
6446}
6447
6448/// Renewal timings for the leader lease.
6449#[derive(Debug, Clone, Copy)]
6450pub struct LeaderLeaseConfig {
6451    /// Lease lifetime.
6452    pub ttl: Duration,
6453    /// Renewal cadence, strictly below the lifetime.
6454    pub renew_interval: Duration,
6455}
6456
6457impl Default for LeaderLeaseConfig {
6458    fn default() -> Self {
6459        Self {
6460            // INVARIANT: one 5s bounded control-I/O retry after the 2s tick still fits inside
6461            // the TTL, so a single transport tail cannot expire a live leader. Takeover of a
6462            // rival that did not release observes it for a full TTL, so this also bounds
6463            // failover latency.
6464            ttl: Duration::from_secs(10),
6465            renew_interval: Duration::from_secs(2),
6466        }
6467    }
6468}
6469
6470/// Whether the exact owner has a durable record and a live process-local deadline.
6471#[must_use]
6472pub fn lease_grants_leadership(
6473    lease: &Option<LeaderLease>,
6474    owner: &LeaderLeaseOwner,
6475    deadline: &LeaseDeadline,
6476) -> bool {
6477    deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
6478}
6479
6480/// Whether a captured proof still matches the current exact grant and local deadline.
6481#[must_use]
6482pub fn lease_grants_proof(
6483    lease: &Option<LeaderLease>,
6484    owner: &LeaderLeaseOwner,
6485    deadline: &LeaseDeadline,
6486    proof: &LeaderProof,
6487) -> bool {
6488    deadline.is_live()
6489        && proof.owner == owner.proof_owner()
6490        && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
6491}
6492
6493#[cfg(test)]
6494mod tests;