Skip to main content

laminar_core/cluster/control/
leader_lease.rs

1//! Durable, append-only leader fencing.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::sync::atomic::{AtomicBool, Ordering};
5use std::sync::{Arc, Weak};
6use std::time::{Duration, Instant};
7
8use bytes::Bytes;
9use object_store::path::Path as OsPath;
10use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
11use parking_lot::Mutex;
12use serde::{Deserialize, Serialize};
13use sha2::{Digest as _, Sha256};
14use tokio::sync::watch;
15use tokio_stream::StreamExt;
16use uuid::Uuid;
17
18use crate::checkpoint::{
19    AssignmentDrainId, AssignmentDrainTransition, CheckpointAssignmentFence,
20    ClusterRecoveryCapsule, LeaderProof, LeaderProofOwner, RecoveryCapsuleRef,
21    MAX_CHECKPOINT_PARTICIPANTS,
22};
23use crate::checkpoint_decision::{
24    CheckpointDecisionStore, CheckpointOutcome, CheckpointScope, CheckpointVerdict, DecisionError,
25    RecordOutcomeResult,
26};
27use crate::cluster::discovery::NodeId;
28
29use super::catalog_manifest::{
30    CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
31};
32use super::controller::{
33    RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
34    RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
35    MAX_RECOVERY_ANNOUNCEMENT_BYTES,
36};
37use super::lease_deadline::LeaseDeadline;
38use super::process_lease::{ProcessLease, ProcessLeaseFence};
39use super::snapshot::{
40    AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
41};
42
43const LEASE_PREFIX: &str = "control/leader-lease/";
44const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
45const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
46const STORE_CONTRACT_PROBE_V1: &[u8] = b"laminardb-control-store-contract-v1";
47const STORE_CONTRACT_PROBE_V2: &[u8] = b"laminardb-control-store-contract-v2";
48const STORE_CONTRACT_PROBE_STALE: &[u8] = b"laminardb-control-store-contract-stale";
49const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
50const AUTHORITY_RECORD_VERSION: u32 = 9;
51const AUTHORITY_HEAD_VERSION: u32 = 1;
52const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
53const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
54const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
55    + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
56    + 2;
57const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
58const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
59const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
60const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
61const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
62const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
63const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
64const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
65const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
66const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
67const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
68const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
69#[cfg(test)]
70const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
71
72fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
73    match error {
74        SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
75        error => LeaseError::Invalid(format!("{context}: {error}")),
76    }
77}
78
79fn lease_path(sequence: u64) -> OsPath {
80    OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
81}
82
83fn authority_head_path() -> OsPath {
84    OsPath::from(AUTHORITY_HEAD_PATH)
85}
86
87fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
88    OsPath::from(format!(
89        "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
90        reference.release.generation(),
91        reference.sha256
92    ))
93}
94
95fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
96    let raw = path
97        .as_ref()
98        .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
99        .and_then(|value| value.strip_prefix("generation="))
100        .ok_or_else(|| {
101            LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
102        })?;
103    let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
104        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
105    })?;
106    let digest = digest.strip_suffix(".json").ok_or_else(|| {
107        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
108    })?;
109    if generation.len() != 20
110        || !generation.bytes().all(|byte| byte.is_ascii_digit())
111        || digest.len() != 64
112        || !digest
113            .bytes()
114            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
115    {
116        return Err(LeaseError::Invalid(format!(
117            "invalid recovery release terminal path {path}"
118        )));
119    }
120    let generation = generation.parse::<u64>().map_err(|error| {
121        LeaseError::Invalid(format!(
122            "invalid recovery release terminal generation in {path}: {error}"
123        ))
124    })?;
125    if generation == 0 {
126        return Err(LeaseError::Invalid(format!(
127            "invalid recovery release terminal path {path}"
128        )));
129    }
130    Ok((generation, digest.to_owned()))
131}
132
133fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
134    let raw = path
135        .as_ref()
136        .strip_prefix(LEASE_PREFIX)
137        .and_then(|file| file.strip_prefix('v'))
138        .and_then(|file| file.strip_suffix(".json"))
139        .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
140    if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
141        return Err(LeaseError::Invalid(format!(
142            "invalid leader authority sequence in {path}"
143        )));
144    }
145    let sequence = raw.parse::<u64>().map_err(|error| {
146        LeaseError::Invalid(format!(
147            "invalid leader authority sequence in {path}: {error}"
148        ))
149    })?;
150    if sequence == 0 || lease_path(sequence) != *path {
151        return Err(LeaseError::Invalid(format!(
152            "noncanonical leader authority path {path}"
153        )));
154    }
155    Ok(sequence)
156}
157
158fn consume_live_authority_link(traversed: &mut usize) -> bool {
159    if *traversed == MAX_LIVE_AUTHORITY_LINKS {
160        return false;
161    }
162    *traversed += 1;
163    true
164}
165
166fn now_millis() -> i64 {
167    std::time::SystemTime::now()
168        .duration_since(std::time::UNIX_EPOCH)
169        .ok()
170        .and_then(|duration| i64::try_from(duration.as_millis()).ok())
171        .unwrap_or(i64::MAX)
172}
173
174async fn read_authority_record(
175    store: &dyn ObjectStore,
176    sequence: u64,
177) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
178    let result = match store.get(&lease_path(sequence)).await {
179        Ok(result) => result,
180        Err(object_store::Error::NotFound { .. }) => return Ok(None),
181        Err(error) => return Err(LeaseError::Io(error.to_string())),
182    };
183    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
184        return Err(LeaseError::Invalid(format!(
185            "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
186            result.meta.size
187        )));
188    }
189    let bytes = result
190        .bytes()
191        .await
192        .map_err(|error| LeaseError::Io(error.to_string()))?;
193    let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
194    record.validate()?;
195    if record.lease.seq != sequence {
196        return Err(LeaseError::Invalid(
197            "authority record sequence does not match its object name".into(),
198        ));
199    }
200    let canonical = serde_json::to_vec(&record)?;
201    if canonical.as_slice() != bytes.as_ref() {
202        return Err(LeaseError::Invalid(format!(
203            "leader authority record {sequence} does not use its canonical body"
204        )));
205    }
206    Ok(Some(record))
207}
208
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
210#[serde(deny_unknown_fields)]
211struct AuthorityHeadPointer {
212    version: u32,
213    sequence: u64,
214    nonce: Uuid,
215}
216
217impl AuthorityHeadPointer {
218    fn new(sequence: u64) -> Result<Self, LeaseError> {
219        let pointer = Self {
220            version: AUTHORITY_HEAD_VERSION,
221            sequence,
222            nonce: Uuid::new_v4(),
223        };
224        pointer.validate()?;
225        Ok(pointer)
226    }
227
228    fn validate(self) -> Result<(), LeaseError> {
229        if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
230            return Err(LeaseError::Invalid(
231                "leader authority head has an unsupported version, zero sequence, or nil nonce"
232                    .into(),
233            ));
234        }
235        Ok(())
236    }
237}
238
239#[derive(Clone)]
240struct VersionedAuthorityHeadPointer {
241    pointer: AuthorityHeadPointer,
242    update_version: UpdateVersion,
243}
244
245struct PublishedAuthorityHead {
246    record: LeaderAuthorityRecord,
247    pointer: VersionedAuthorityHeadPointer,
248}
249
250async fn read_authority_head_pointer(
251    store: &dyn ObjectStore,
252) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
253    let result = match store.get(&authority_head_path()).await {
254        Ok(result) => result,
255        Err(object_store::Error::NotFound { .. }) => return Ok(None),
256        Err(error) => return Err(LeaseError::Io(error.to_string())),
257    };
258    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
259        return Err(LeaseError::Invalid(format!(
260            "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
261            result.meta.size
262        )));
263    }
264    let update_version = UpdateVersion {
265        e_tag: result.meta.e_tag.clone(),
266        version: result.meta.version.clone(),
267    };
268    let bytes = result
269        .bytes()
270        .await
271        .map_err(|error| LeaseError::Io(error.to_string()))?;
272    let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
273    pointer.validate()?;
274    let canonical = serde_json::to_vec(&pointer)?;
275    if canonical.as_slice() != bytes.as_ref() {
276        return Err(LeaseError::Invalid(
277            "leader authority head does not use its canonical body".into(),
278        ));
279    }
280    Ok(Some(VersionedAuthorityHeadPointer {
281        pointer,
282        update_version,
283    }))
284}
285
286fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
287    let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
288    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
289    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
290        return Err(LeaseError::Invalid(format!(
291            "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
292            encoded.len()
293        )));
294    }
295    Ok(Bytes::from(encoded))
296}
297
298fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
299    record.validate()?;
300    let encoded = serde_json::to_vec(record)?;
301    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
302    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
303        return Err(LeaseError::Invalid(format!(
304            "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
305            encoded.len()
306        )));
307    }
308    if record.recovery_fault_slots.iter().any(|slot| slot.active) {
309        ensure_recovery_fault_authority_headroom(encoded_len)?;
310    }
311    Ok(Bytes::from(encoded))
312}
313
314fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
315    let limit = MAX_AUTHORITY_RECORD_BYTES
316        .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
317        .expect("recovery fault headroom must fit the authority record");
318    if encoded_len > limit {
319        return Err(LeaseError::Invalid(format!(
320            "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
321        )));
322    }
323    Ok(())
324}
325
326/// Exact process incarnation eligible to hold the leader lease.
327#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
328pub struct LeaderLeaseOwner {
329    /// Stable cluster node identity.
330    pub node: NodeId,
331    /// Boot-unique process identity.
332    pub boot: Uuid,
333    /// Durable stable-node process term.
334    pub process_term: u64,
335}
336
337impl LeaderLeaseOwner {
338    fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
339        process
340            .validate(process.node)
341            .map_err(|error| LeaseError::Invalid(error.to_string()))?;
342        Ok(Self {
343            node: process.node,
344            boot: process.owner,
345            process_term: process.term,
346        })
347    }
348
349    fn validate(&self) -> Result<(), LeaseError> {
350        if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
351            return Err(LeaseError::Invalid(
352                "leader owner node, boot identity, and process term must be nonzero".into(),
353            ));
354        }
355        Ok(())
356    }
357
358    fn proof_owner(&self) -> LeaderProofOwner {
359        LeaderProofOwner {
360            node_id: self.node.0,
361            boot_id: self.boot,
362            process_term: self.process_term,
363        }
364    }
365}
366
367/// Durable leader lease record.
368#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
369pub struct LeaderLease {
370    /// Append-only compare-and-set sequence.
371    pub seq: u64,
372    /// Monotonic liveness sequence, advanced only by an acquisition or renewal.
373    pub renewal_sequence: u64,
374    /// Fencing token, stable across uninterrupted renewals and advanced for each authority term.
375    pub token: u64,
376    /// Exact process incarnation holding the lease.
377    pub owner: LeaderLeaseOwner,
378    /// Owner-written wall-clock expiry for diagnostics only.
379    pub expires_at_ms: i64,
380    /// Immutable catalog content reference, once sealed for this control namespace.
381    pub catalog_manifest: Option<CatalogManifestRef>,
382}
383
384#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
385#[serde(deny_unknown_fields)]
386struct OutcomeLink {
387    sequence: u64,
388    epoch: u64,
389    checkpoint_id: u64,
390}
391
392impl OutcomeLink {
393    fn validate(self) -> Result<(), LeaseError> {
394        let attempt = crate::state::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
395        if self.sequence == 0 || !attempt.is_canonical() {
396            return Err(LeaseError::Invalid(
397                "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
398                    .into(),
399            ));
400        }
401        Ok(())
402    }
403}
404
405#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
406#[serde(deny_unknown_fields)]
407struct AssignmentDecisionLink {
408    sequence: u64,
409    target_version: u64,
410}
411
412#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
413#[serde(deny_unknown_fields)]
414struct AuthorityRecoveryFaultSlot {
415    publisher: RecoveryFaultPublisher,
416    request_sequence: u64,
417    fault_sequence: u64,
418    active: bool,
419}
420
421impl AuthorityRecoveryFaultSlot {
422    fn validate(&self) -> Result<(), LeaseError> {
423        self.publisher.validate().map_err(LeaseError::Invalid)?;
424        if self.request_sequence == 0 || self.fault_sequence == 0 {
425            return Err(LeaseError::Invalid(
426                "recovery fault request and authority sequence must be nonzero".into(),
427            ));
428        }
429        Ok(())
430    }
431
432    fn fault(&self) -> RecoveryFault {
433        RecoveryFault {
434            reporter: NodeId(self.publisher.participant.node_id),
435            sequence: self.fault_sequence,
436        }
437    }
438
439    fn matches_request(&self, publisher: RecoveryFaultPublisher, request_sequence: u64) -> bool {
440        self.publisher == publisher && self.request_sequence == request_sequence
441    }
442}
443
444#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
445#[serde(deny_unknown_fields)]
446pub(crate) struct RecoveryReleaseTerminalRef {
447    release: RecoveryReleaseId,
448    sha256: String,
449    encoded_len: u64,
450}
451
452impl RecoveryReleaseTerminalRef {
453    fn validate(&self) -> Result<(), LeaseError> {
454        if !self.release.is_canonical()
455            || self.sha256.len() != 64
456            || !self
457                .sha256
458                .bytes()
459                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
460            || self.encoded_len == 0
461            || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
462        {
463            return Err(LeaseError::Invalid(
464                "recovery release terminal reference is not canonical".into(),
465            ));
466        }
467        Ok(())
468    }
469
470    pub(crate) fn generation(&self) -> u64 {
471        self.release.generation()
472    }
473}
474
475#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
476#[serde(deny_unknown_fields)]
477struct AuthorityRecoveryReleaseCommit {
478    terminal: RecoveryReleaseTerminalRef,
479    leader_proof: LeaderProof,
480}
481
482impl AuthorityRecoveryReleaseCommit {
483    fn validate(&self) -> Result<(), LeaseError> {
484        self.terminal.validate()?;
485        if !self.leader_proof.is_canonical() {
486            return Err(LeaseError::Invalid(
487                "recovery release commit has no canonical leader proof".into(),
488            ));
489        }
490        Ok(())
491    }
492}
493
494#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
495#[serde(deny_unknown_fields)]
496pub(crate) struct RecoveryReleaseLink {
497    sequence: u64,
498    terminal: RecoveryReleaseTerminalRef,
499}
500
501impl RecoveryReleaseLink {
502    fn validate(&self) -> Result<(), LeaseError> {
503        if self.sequence == 0 {
504            return Err(LeaseError::Invalid(
505                "recovery release authority link has a zero sequence".into(),
506            ));
507        }
508        self.terminal.validate()
509    }
510}
511
512#[derive(Debug, Clone, PartialEq, Eq)]
513pub(crate) enum RecordRecoveryReleaseCommitResult {
514    Created(RecoveryReleaseTerminalRef),
515    Unchanged(RecoveryReleaseTerminalRef),
516    Conflict { winner: RecoveryReleaseTerminalRef },
517    FaultsChanged,
518}
519
520#[derive(Debug, Clone, Copy, PartialEq, Eq)]
521pub(crate) enum RecordRecoveryFaultResult {
522    Active,
523    AlreadyCleared,
524    CoveredByNewerRequest,
525    Superseded,
526}
527
528fn recovery_release_id_for_terminal(
529    terminal: &RecoveryAnnouncement,
530) -> Result<RecoveryReleaseId, LeaseError> {
531    terminal.validate().map_err(LeaseError::Invalid)?;
532    let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
533        return Err(LeaseError::Invalid(
534            "recovery release terminal must carry ReleaseCommitted".into(),
535        ));
536    };
537    RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
538        round: terminal.round.clone(),
539        phase: RecoverPhase::Release { epoch },
540    })
541    .map_err(LeaseError::Invalid)
542}
543
544fn encode_recovery_release_terminal(
545    terminal: &RecoveryAnnouncement,
546) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
547    let release = recovery_release_id_for_terminal(terminal)?;
548    let encoded = serde_json::to_vec(terminal)?;
549    let encoded_len = u64::try_from(encoded.len())
550        .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
551    if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
552        return Err(LeaseError::Invalid(format!(
553            "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
554        )));
555    }
556    let reference = RecoveryReleaseTerminalRef {
557        release,
558        sha256: format!("{:x}", Sha256::digest(&encoded)),
559        encoded_len,
560    };
561    reference.validate()?;
562    Ok((Bytes::from(encoded), reference))
563}
564
565/// Immutable settlement of one exact assignment-drain transition.
566#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
567#[serde(deny_unknown_fields)]
568pub struct AssignmentDrainDecision {
569    /// Exact predecessor-to-target transition being settled.
570    pub transition: AssignmentDrainTransition,
571    /// Durable leader term that decided the transition.
572    pub leader_proof: LeaderProof,
573    /// Whether to install the certified target or restore the predecessor map.
574    pub verdict: AssignmentDrainVerdict,
575}
576
577impl AssignmentDrainDecision {
578    /// Construct a decision for an exact canonical transition.
579    ///
580    /// # Errors
581    /// Rejects a malformed transition.
582    pub fn new(
583        transition: &AssignmentDrainTransition,
584        leader_proof: LeaderProof,
585        verdict: AssignmentDrainVerdict,
586    ) -> Result<Self, String> {
587        if !transition.is_canonical()
588            || !leader_proof.is_canonical()
589            || (verdict == AssignmentDrainVerdict::Commit && leader_proof != transition.leader)
590        {
591            return Err("assignment drain decision requires a canonical transition".into());
592        }
593        Ok(Self {
594            transition: transition.clone(),
595            leader_proof,
596            verdict,
597        })
598    }
599
600    /// Compact identity used by source-drain receipts and authority lookups.
601    #[must_use]
602    pub fn round(&self) -> AssignmentDrainId {
603        self.transition.id()
604    }
605
606    /// Target assignment version settled by this decision.
607    #[must_use]
608    pub fn target_version(&self) -> u64 {
609        self.transition.target.assignment_version
610    }
611
612    fn validate(&self) -> Result<(), LeaseError> {
613        if !self.transition.is_canonical()
614            || !self.leader_proof.is_canonical()
615            || (self.verdict == AssignmentDrainVerdict::Commit
616                && self.leader_proof != self.transition.leader)
617        {
618            return Err(LeaseError::Invalid(
619                "assignment drain decision is not canonical".into(),
620            ));
621        }
622        Ok(())
623    }
624}
625
626/// Terminal result for an assignment-drain transition.
627#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
628#[serde(rename_all = "snake_case")]
629pub enum AssignmentDrainVerdict {
630    /// Install the transition's certified target assignment.
631    Commit,
632    /// Restore the predecessor owner map at the target version.
633    Abort,
634}
635
636/// Result of admitting a drain decision through the shared authority sequence.
637#[derive(Debug, Clone, PartialEq, Eq)]
638pub enum RecordAssignmentDrainDecisionResult {
639    /// This call created the immutable decision.
640    Created(AssignmentDrainDecision),
641    /// The same decision was already durable.
642    Unchanged(AssignmentDrainDecision),
643    /// Another terminal decision already won for this transition version.
644    Conflict {
645        /// Immutable winner.
646        winner: AssignmentDrainDecision,
647    },
648}
649
650/// Immutable authorization to install one staged failure-recovery assignment.
651#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
652#[serde(deny_unknown_fields)]
653pub struct AssignmentRecoveryDecision {
654    /// Exact assignment being replaced.
655    pub predecessor: CheckpointAssignmentFence,
656    /// Exact successor assignment authorized for installation.
657    pub target: CheckpointAssignmentFence,
658    /// Content-addressed staged snapshot whose assignment must equal `target`.
659    pub proposal: AssignmentSnapshotRef,
660    /// Exact process-lease takeover proofs for every predecessor process absent from `target`.
661    pub removed_process_fences: Vec<ProcessLeaseFence>,
662    /// Durable leader term that authorized this recovery.
663    pub leader_proof: LeaderProof,
664}
665
666impl AssignmentRecoveryDecision {
667    /// Construct one exact failure-recovery authorization.
668    ///
669    /// # Errors
670    /// Rejects invalid fences, a non-successor proposal, or an incomplete process-removal proof.
671    pub fn new(
672        predecessor: CheckpointAssignmentFence,
673        target: CheckpointAssignmentFence,
674        proposal: AssignmentSnapshotRef,
675        removed_process_fences: Vec<ProcessLeaseFence>,
676        leader_proof: LeaderProof,
677    ) -> Result<Self, String> {
678        let decision = Self {
679            predecessor,
680            target,
681            proposal,
682            removed_process_fences,
683            leader_proof,
684        };
685        decision.validate().map_err(|error| error.to_string())?;
686        Ok(decision)
687    }
688
689    /// Target assignment version settled by this decision.
690    #[must_use]
691    pub fn target_version(&self) -> u64 {
692        self.target.assignment_version
693    }
694
695    fn validate(&self) -> Result<(), LeaseError> {
696        if !self.predecessor.is_canonical()
697            || !self.target.is_canonical()
698            || !self.leader_proof.is_canonical()
699            || self.predecessor.vnode_count != self.target.vnode_count
700            || self.predecessor.assignment_version.checked_add(1)
701                != Some(self.target.assignment_version)
702            || self.proposal.validate().is_err()
703            || self.proposal.version != self.target.assignment_version
704        {
705            return Err(LeaseError::Invalid(
706                "assignment recovery decision is not a canonical successor".into(),
707            ));
708        }
709
710        let removed: Vec<_> = self
711            .predecessor
712            .participants
713            .iter()
714            .filter(|participant| {
715                self.target.participant_incarnation(participant.node_id)
716                    != Some(participant.boot_incarnation)
717            })
718            .collect();
719        if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
720            return Err(LeaseError::Invalid(
721                "assignment recovery decision requires the exact removed predecessor process set"
722                    .into(),
723            ));
724        }
725        for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
726            if !fence.is_canonical()
727                || fence.predecessor.node.0 != participant.node_id
728                || fence.predecessor.owner != participant.boot_incarnation
729            {
730                return Err(LeaseError::Invalid(
731                    "assignment recovery process fences are incomplete, unsorted, or unrelated"
732                        .into(),
733                ));
734            }
735        }
736        Ok(())
737    }
738}
739
740/// Result of admitting a recovery decision through the shared authority sequence.
741#[derive(Debug, Clone, PartialEq, Eq)]
742pub enum RecordAssignmentRecoveryDecisionResult {
743    /// This call created the immutable decision.
744    Created(AssignmentRecoveryDecision),
745    /// The same decision was already durable.
746    Unchanged(AssignmentRecoveryDecision),
747    /// Another recovery decision already won for this target version.
748    Conflict {
749        /// Immutable winner.
750        winner: AssignmentRecoveryDecision,
751    },
752}
753
754#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
755#[serde(
756    tag = "kind",
757    content = "decision",
758    rename_all = "snake_case",
759    deny_unknown_fields
760)]
761enum AuthorityAssignmentDecision {
762    Drain(AssignmentDrainDecision),
763    Recovery(AssignmentRecoveryDecision),
764}
765
766impl AuthorityAssignmentDecision {
767    fn target_version(&self) -> u64 {
768        match self {
769            Self::Drain(decision) => decision.target_version(),
770            Self::Recovery(decision) => decision.target_version(),
771        }
772    }
773
774    fn leader_proof(&self) -> &LeaderProof {
775        match self {
776            Self::Drain(decision) => &decision.leader_proof,
777            Self::Recovery(decision) => &decision.leader_proof,
778        }
779    }
780
781    fn validate(&self) -> Result<(), LeaseError> {
782        match self {
783            Self::Drain(decision) => decision.validate(),
784            Self::Recovery(decision) => decision.validate(),
785        }
786    }
787}
788
789enum RecordAuthorityAssignmentDecisionResult {
790    Created(AuthorityAssignmentDecision),
791    Unchanged(AuthorityAssignmentDecision),
792    Conflict { winner: AuthorityAssignmentDecision },
793}
794
795#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
796#[serde(deny_unknown_fields)]
797struct AuthorityOutcomeFloor {
798    deployment_id: String,
799    artifact_before_epoch: u64,
800    authority_before_epoch: u64,
801    terminal_anchor: Option<CheckpointOutcome>,
802    terminal_anchor_link: Option<OutcomeLink>,
803    committed_anchor: Option<CheckpointOutcome>,
804    committed_anchor_link: Option<OutcomeLink>,
805}
806
807#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
808#[serde(deny_unknown_fields)]
809struct AuthorityAssignmentDecisionFloor {
810    before_target_version: u64,
811    terminal_anchor: Option<AuthorityAssignmentDecision>,
812    terminal_anchor_link: Option<AssignmentDecisionLink>,
813}
814
815/// One immutable entry in the cluster's shared leadership and checkpoint-decision sequence.
816#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
817#[serde(deny_unknown_fields)]
818struct LeaderAuthorityRecord {
819    version: u32,
820    lease: LeaderLease,
821    /// Present only on the sequence that admitted this terminal outcome.
822    checkpoint_outcome: Option<CheckpointOutcome>,
823    /// Link to the preceding admitted outcome, present only on an outcome-bearing record.
824    previous_outcome: Option<OutcomeLink>,
825    /// Latest admitted outcome. Renewals, takeovers, catalog seals, and floor advances preserve it.
826    outcome_head: Option<OutcomeLink>,
827    /// Link to the preceding admitted Commit, present only on a Commit-bearing record.
828    previous_commit: Option<OutcomeLink>,
829    /// Latest admitted Commit. Every authority mutation preserves it; Abort does not advance it.
830    commit_head: Option<OutcomeLink>,
831    /// Monotonic cluster outcome retention boundary and its continuity anchors.
832    outcome_floor: Option<AuthorityOutcomeFloor>,
833    /// Present only on the sequence that admitted an assignment decision.
834    assignment_decision: Option<AuthorityAssignmentDecision>,
835    /// Link to the preceding assignment decision, present only on a decision-bearing record.
836    previous_assignment_decision: Option<AssignmentDecisionLink>,
837    /// Latest admitted assignment decision. Every other authority mutation preserves it.
838    assignment_decision_head: Option<AssignmentDecisionLink>,
839    /// Monotonic assignment-decision retention boundary and its continuity anchor.
840    assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
841    /// Authority sequence that admitted the current recovery-fault slot state.
842    recovery_fault_revision: u64,
843    /// One active or tombstoned request identity per stable node.
844    recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
845    /// Present only on the sequence that admitted the latest recovery release.
846    recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
847    /// Latest admitted recovery release, preserved by every later authority mutation.
848    recovery_release_head: Option<RecoveryReleaseLink>,
849}
850
851impl LeaderLease {
852    fn validate(&self) -> Result<(), LeaseError> {
853        self.owner.validate()?;
854        if self.seq == 0
855            || self.renewal_sequence == 0
856            || self.renewal_sequence > self.seq
857            || self.token == 0
858        {
859            return Err(LeaseError::Invalid(
860                "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
861                    .into(),
862            ));
863        }
864        if let Some(reference) = &self.catalog_manifest {
865            reference
866                .validate()
867                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
868        }
869        Ok(())
870    }
871
872    /// Exact feature-neutral proof for this ownership term.
873    #[must_use]
874    pub fn proof(&self) -> LeaderProof {
875        LeaderProof {
876            owner: self.owner.proof_owner(),
877            fencing_token: self.token,
878        }
879    }
880
881    /// Whether `proof` names this exact owner and fencing token.
882    #[must_use]
883    pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
884        proof.is_canonical()
885            && proof.owner == self.owner.proof_owner()
886            && proof.fencing_token == self.token
887    }
888
889    fn has_same_liveness_identity(&self, other: &Self) -> bool {
890        self.owner == other.owner
891            && self.token == other.token
892            && self.renewal_sequence == other.renewal_sequence
893    }
894}
895
896impl AuthorityOutcomeFloor {
897    fn validate(&self) -> Result<(), LeaseError> {
898        let deployment = Uuid::parse_str(&self.deployment_id)
899            .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
900        if deployment.is_nil()
901            || deployment.to_string() != self.deployment_id
902            || self.authority_before_epoch == 0
903            || self.artifact_before_epoch > self.authority_before_epoch
904        {
905            return Err(LeaseError::Invalid(
906                "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
907                    .into(),
908            ));
909        }
910        for link in [self.terminal_anchor_link, self.committed_anchor_link]
911            .into_iter()
912            .flatten()
913        {
914            link.validate()?;
915        }
916        for (name, anchor) in [
917            ("terminal", self.terminal_anchor.as_ref()),
918            ("committed", self.committed_anchor.as_ref()),
919        ] {
920            let Some(anchor) = anchor else { continue };
921            anchor
922                .validate_shape(anchor.epoch)
923                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
924            if anchor.scope != CheckpointScope::Cluster
925                || anchor.deployment_id != self.deployment_id
926            {
927                return Err(LeaseError::Invalid(format!(
928                    "outcome floor has an invalid {name} anchor"
929                )));
930            }
931        }
932        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
933            (Some(anchor), Some(link))
934                if anchor.epoch < self.authority_before_epoch
935                    && link.sequence != 0
936                    && link.epoch == anchor.epoch
937                    && link.checkpoint_id == anchor.checkpoint_id => {}
938            (None, None) => {}
939            _ => {
940                return Err(LeaseError::Invalid(
941                    "outcome floor terminal anchor does not match its exact authority link".into(),
942                ));
943            }
944        }
945        match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
946            (Some(anchor), Some(link))
947                if self.artifact_before_epoch != 0
948                    && anchor.is_commit()
949                    && anchor.epoch < self.artifact_before_epoch
950                    && link.sequence != 0
951                    && link.epoch == anchor.epoch
952                    && link.checkpoint_id == anchor.checkpoint_id => {}
953            (None, None) => {}
954            _ => {
955                return Err(LeaseError::Invalid(
956                    "outcome floor committed anchor does not match its exact Commit link".into(),
957                ));
958            }
959        }
960        if let (Some(committed), Some(terminal)) = (
961            self.committed_anchor.as_ref(),
962            self.terminal_anchor.as_ref(),
963        ) {
964            let ordered = if committed.epoch == terminal.epoch {
965                committed == terminal
966            } else {
967                committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
968            };
969            if !ordered {
970                return Err(LeaseError::Invalid(
971                    "outcome floor anchors are not monotonically ordered".into(),
972                ));
973            }
974            let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
975                (Some(committed_link), Some(terminal_link)) if committed == terminal => {
976                    committed_link == terminal_link
977                }
978                (Some(committed_link), Some(terminal_link)) => {
979                    committed_link.sequence < terminal_link.sequence
980                }
981                _ => false,
982            };
983            if !links_ordered {
984                return Err(LeaseError::Invalid(
985                    "outcome floor anchor links are not ordered with their exact outcomes".into(),
986                ));
987            }
988        } else if self.committed_anchor.is_some() {
989            return Err(LeaseError::Invalid(
990                "outcome floor has a committed anchor without terminal continuity".into(),
991            ));
992        }
993        Ok(())
994    }
995}
996
997impl AuthorityAssignmentDecisionFloor {
998    fn validate(&self) -> Result<(), LeaseError> {
999        if self.before_target_version == 0 {
1000            return Err(LeaseError::Invalid(
1001                "assignment decision floor requires a nonzero target version".into(),
1002            ));
1003        }
1004        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1005            (Some(anchor), Some(link)) => {
1006                anchor.validate()?;
1007                if link.sequence == 0
1008                    || link.target_version != anchor.target_version()
1009                    || anchor.target_version() >= self.before_target_version
1010                {
1011                    return Err(LeaseError::Invalid(
1012                        "assignment decision floor anchor does not match its exact authority link"
1013                            .into(),
1014                    ));
1015                }
1016            }
1017            (None, None) => {}
1018            _ => {
1019                return Err(LeaseError::Invalid(
1020                    "assignment decision floor has an incomplete terminal anchor".into(),
1021                ));
1022            }
1023        }
1024        Ok(())
1025    }
1026}
1027
1028impl LeaderAuthorityRecord {
1029    fn initial(lease: LeaderLease) -> Self {
1030        Self {
1031            version: AUTHORITY_RECORD_VERSION,
1032            lease,
1033            checkpoint_outcome: None,
1034            previous_outcome: None,
1035            outcome_head: None,
1036            previous_commit: None,
1037            commit_head: None,
1038            outcome_floor: None,
1039            assignment_decision: None,
1040            previous_assignment_decision: None,
1041            assignment_decision_head: None,
1042            assignment_decision_floor: None,
1043            recovery_fault_revision: 0,
1044            recovery_fault_slots: Vec::new(),
1045            recovery_release_commit: None,
1046            recovery_release_head: None,
1047        }
1048    }
1049
1050    fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1051        Self {
1052            version: AUTHORITY_RECORD_VERSION,
1053            lease,
1054            checkpoint_outcome: None,
1055            previous_outcome: None,
1056            outcome_head: self.outcome_head,
1057            previous_commit: None,
1058            commit_head: self.commit_head,
1059            outcome_floor: self.outcome_floor.clone(),
1060            assignment_decision: None,
1061            previous_assignment_decision: None,
1062            assignment_decision_head: self.assignment_decision_head,
1063            assignment_decision_floor: self.assignment_decision_floor.clone(),
1064            recovery_fault_revision: self.recovery_fault_revision,
1065            recovery_fault_slots: self.recovery_fault_slots.clone(),
1066            recovery_release_commit: None,
1067            recovery_release_head: self.recovery_release_head.clone(),
1068        }
1069    }
1070
1071    fn validate(&self) -> Result<(), LeaseError> {
1072        if self.version != AUTHORITY_RECORD_VERSION {
1073            return Err(LeaseError::Invalid(format!(
1074                "authority record version {} is unsupported",
1075                self.version
1076            )));
1077        }
1078        self.lease.validate()?;
1079        for link in [
1080            self.previous_outcome,
1081            self.outcome_head,
1082            self.previous_commit,
1083            self.commit_head,
1084        ]
1085        .into_iter()
1086        .flatten()
1087        {
1088            link.validate()?;
1089        }
1090        if let Some(floor) = &self.outcome_floor {
1091            floor.validate()?;
1092        }
1093        if let Some(floor) = &self.assignment_decision_floor {
1094            floor.validate()?;
1095        }
1096        self.validate_recovery_fault_inventory()?;
1097        self.validate_checkpoint_outcome_chain()?;
1098        self.validate_outcome_floor()?;
1099        self.validate_assignment_decision_chain()?;
1100        self.validate_assignment_decision_floor()?;
1101        self.validate_recovery_release()?;
1102        Ok(())
1103    }
1104
1105    fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1106        if self.recovery_fault_slots.windows(2).any(|pair| {
1107            pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1108        }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1109            || (self.recovery_fault_revision != 0
1110                && self.recovery_fault_slots.is_empty()
1111                && self
1112                    .recovery_release_head
1113                    .as_ref()
1114                    .map(|head| head.sequence)
1115                    != Some(self.recovery_fault_revision))
1116            || self.recovery_fault_revision > self.lease.seq
1117            || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1118        {
1119            return Err(LeaseError::Invalid(
1120                "leader authority recovery-fault inventory is not canonical".into(),
1121            ));
1122        }
1123        let mut fault_sequences = BTreeSet::new();
1124        for slot in &self.recovery_fault_slots {
1125            slot.validate()?;
1126            if slot.fault_sequence > self.recovery_fault_revision
1127                || slot.fault_sequence > self.lease.seq
1128                || !fault_sequences.insert(slot.fault_sequence)
1129            {
1130                return Err(LeaseError::Invalid(
1131                    "leader authority recovery-fault sequences are not canonical".into(),
1132                ));
1133            }
1134        }
1135        if [
1136            self.checkpoint_outcome.is_some(),
1137            self.assignment_decision.is_some(),
1138            self.recovery_release_commit.is_some(),
1139        ]
1140        .into_iter()
1141        .filter(|present| *present)
1142        .count()
1143            > 1
1144        {
1145            return Err(LeaseError::Invalid(
1146                "one authority sequence cannot admit two terminal domains".into(),
1147            ));
1148        }
1149        Ok(())
1150    }
1151
1152    fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1153        if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1154            outcome
1155                .validate_shape(outcome.epoch)
1156                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1157            if outcome.scope != CheckpointScope::Cluster {
1158                return Err(LeaseError::Invalid(
1159                    "leader authority can only admit cluster checkpoint outcomes".into(),
1160                ));
1161            }
1162            let proof = outcome
1163                .leader_proof
1164                .as_ref()
1165                .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1166            let current_link = OutcomeLink {
1167                sequence: self.lease.seq,
1168                epoch: outcome.epoch,
1169                checkpoint_id: outcome.checkpoint_id,
1170            };
1171            if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1172                return Err(LeaseError::Invalid(
1173                    "cluster outcome is not bound to its exact authority sequence and term".into(),
1174                ));
1175            }
1176            if let Some(previous) = self.previous_outcome {
1177                if previous.sequence >= self.lease.seq
1178                    || previous.epoch >= outcome.epoch
1179                    || previous.checkpoint_id >= outcome.checkpoint_id
1180                {
1181                    return Err(LeaseError::Invalid(
1182                        "cluster outcome link does not move backward in sequence and epoch".into(),
1183                    ));
1184                }
1185            }
1186            if outcome.is_commit() {
1187                if self.commit_head != Some(current_link) {
1188                    return Err(LeaseError::Invalid(
1189                        "cluster Commit is not bound to its exact Commit-chain head".into(),
1190                    ));
1191                }
1192                if let Some(previous) = self.previous_commit {
1193                    if previous.sequence >= self.lease.seq
1194                        || previous.epoch >= outcome.epoch
1195                        || previous.checkpoint_id >= outcome.checkpoint_id
1196                    {
1197                        return Err(LeaseError::Invalid(
1198                            "cluster Commit link does not move backward in sequence and epoch"
1199                                .into(),
1200                        ));
1201                    }
1202                    if self.outcome_floor.as_ref().is_some_and(|floor| {
1203                        previous.epoch < floor.artifact_before_epoch
1204                            && Some(previous) != floor.committed_anchor_link
1205                    }) {
1206                        return Err(LeaseError::Invalid(
1207                            "cluster Commit link crosses the artifact floor without its exact anchor"
1208                                .into(),
1209                        ));
1210                    }
1211                }
1212            } else if self.previous_commit.is_some() {
1213                return Err(LeaseError::Invalid(
1214                    "cluster Abort carries a previous-Commit link".into(),
1215                ));
1216            }
1217            if let Some(floor) = &self.outcome_floor {
1218                if outcome.deployment_id != floor.deployment_id
1219                    || outcome.epoch < floor.authority_before_epoch
1220                {
1221                    return Err(LeaseError::Invalid(
1222                        "cluster outcome is below or outside its durable authority floor".into(),
1223                    ));
1224                }
1225            }
1226        } else {
1227            if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1228                return Err(LeaseError::Invalid(
1229                    "non-outcome authority record carries an outcome-chain link".into(),
1230                ));
1231            }
1232            if self.outcome_head.is_some_and(|head| {
1233                head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1234            }) {
1235                return Err(LeaseError::Invalid(
1236                    "authority outcome head is outside the durable sequence".into(),
1237                ));
1238            }
1239        }
1240        if self.commit_head.is_some_and(|head| {
1241            head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1242        }) {
1243            return Err(LeaseError::Invalid(
1244                "authority Commit head is outside the durable sequence".into(),
1245            ));
1246        }
1247        match (self.commit_head, self.outcome_head) {
1248            (Some(commit), Some(terminal)) => {
1249                let equal = commit == terminal;
1250                let strictly_older = commit.sequence < terminal.sequence
1251                    && commit.epoch < terminal.epoch
1252                    && commit.checkpoint_id < terminal.checkpoint_id;
1253                if !equal && !strictly_older {
1254                    return Err(LeaseError::Invalid(
1255                        "authority Commit head is not ordered behind the terminal head".into(),
1256                    ));
1257                }
1258                if self
1259                    .checkpoint_outcome
1260                    .as_ref()
1261                    .is_some_and(|outcome| !outcome.is_commit())
1262                    && !strictly_older
1263                {
1264                    return Err(LeaseError::Invalid(
1265                        "cluster Abort did not preserve a strictly older Commit head".into(),
1266                    ));
1267                }
1268            }
1269            (Some(_), None) => {
1270                return Err(LeaseError::Invalid(
1271                    "authority Commit head exists without terminal continuity".into(),
1272                ));
1273            }
1274            (None, _) => {}
1275        }
1276        Ok(())
1277    }
1278
1279    fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1280        if let Some(floor) = self.outcome_floor.as_ref() {
1281            if floor
1282                .terminal_anchor_link
1283                .is_some_and(|link| link.sequence >= self.lease.seq)
1284            {
1285                return Err(LeaseError::Invalid(
1286                    "outcome floor anchor is outside the authority sequence".into(),
1287                ));
1288            }
1289            if floor
1290                .committed_anchor_link
1291                .is_some_and(|link| link.sequence >= self.lease.seq)
1292            {
1293                return Err(LeaseError::Invalid(
1294                    "outcome floor Commit anchor is outside the authority sequence".into(),
1295                ));
1296            }
1297            if let Some(head) = self.outcome_head {
1298                if head.epoch < floor.authority_before_epoch
1299                    && Some(head) != floor.terminal_anchor_link
1300                {
1301                    return Err(LeaseError::Invalid(
1302                        "authority outcome head does not meet its durable compaction floor".into(),
1303                    ));
1304                }
1305            } else if floor.terminal_anchor_link.is_some() {
1306                return Err(LeaseError::Invalid(
1307                    "outcome floor anchor is disconnected from the authority head".into(),
1308                ));
1309            }
1310            if let Some(head) = self.commit_head {
1311                if head.epoch < floor.artifact_before_epoch
1312                    && Some(head) != floor.committed_anchor_link
1313                {
1314                    return Err(LeaseError::Invalid(
1315                        "authority Commit head does not meet its durable artifact floor".into(),
1316                    ));
1317                }
1318            } else if floor.committed_anchor_link.is_some() {
1319                return Err(LeaseError::Invalid(
1320                    "outcome floor Commit anchor is disconnected from the Commit head".into(),
1321                ));
1322            }
1323        }
1324        Ok(())
1325    }
1326
1327    fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1328        if let Some(decision) = self.assignment_decision.as_ref() {
1329            decision.validate()?;
1330            if self
1331                .assignment_decision_floor
1332                .as_ref()
1333                .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1334            {
1335                return Err(LeaseError::Invalid(
1336                    "assignment decision is below its durable authority floor".into(),
1337                ));
1338            }
1339            if !self.lease.matches_proof(decision.leader_proof())
1340                || self.assignment_decision_head
1341                    != Some(AssignmentDecisionLink {
1342                        sequence: self.lease.seq,
1343                        target_version: decision.target_version(),
1344                    })
1345            {
1346                return Err(LeaseError::Invalid(
1347                    "assignment decision is not bound to its exact authority sequence and term"
1348                        .into(),
1349                ));
1350            }
1351            if let Some(previous) = self.previous_assignment_decision {
1352                if previous.sequence >= self.lease.seq
1353                    || previous.target_version >= decision.target_version()
1354                {
1355                    return Err(LeaseError::Invalid(
1356                        "assignment decision link does not move backward".into(),
1357                    ));
1358                }
1359            }
1360        } else {
1361            if self.previous_assignment_decision.is_some() {
1362                return Err(LeaseError::Invalid(
1363                    "non-decision authority record carries a previous assignment-decision link"
1364                        .into(),
1365                ));
1366            }
1367            if self
1368                .assignment_decision_head
1369                .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1370            {
1371                return Err(LeaseError::Invalid(
1372                    "authority assignment-decision head is outside the durable sequence".into(),
1373                ));
1374            }
1375        }
1376        Ok(())
1377    }
1378
1379    fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1380        if let Some(floor) = self.assignment_decision_floor.as_ref() {
1381            if floor
1382                .terminal_anchor_link
1383                .is_some_and(|link| link.sequence >= self.lease.seq)
1384            {
1385                return Err(LeaseError::Invalid(
1386                    "assignment decision floor anchor is outside the authority sequence".into(),
1387                ));
1388            }
1389            if let Some(head) = self.assignment_decision_head {
1390                if head.target_version < floor.before_target_version
1391                    && Some(head) != floor.terminal_anchor_link
1392                {
1393                    return Err(LeaseError::Invalid(
1394                        "authority assignment-decision head does not meet its durable floor".into(),
1395                    ));
1396                }
1397            } else if floor.terminal_anchor_link.is_some() {
1398                return Err(LeaseError::Invalid(
1399                    "assignment decision floor anchor is disconnected from the authority head"
1400                        .into(),
1401                ));
1402            }
1403        }
1404        Ok(())
1405    }
1406
1407    fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1408        match self.recovery_release_commit.as_ref() {
1409            Some(commit) => {
1410                commit.validate()?;
1411                let expected = RecoveryReleaseLink {
1412                    sequence: self.lease.seq,
1413                    terminal: commit.terminal.clone(),
1414                };
1415                if !self.lease.matches_proof(&commit.leader_proof)
1416                    || self.recovery_release_head.as_ref() != Some(&expected)
1417                    || self.recovery_fault_revision != self.lease.seq
1418                    || self.recovery_fault_slots.iter().any(|slot| slot.active)
1419                {
1420                    return Err(LeaseError::Invalid(
1421                        "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1422                            .into(),
1423                    ));
1424                }
1425            }
1426            None => {
1427                if let Some(head) = self.recovery_release_head.as_ref() {
1428                    head.validate()?;
1429                    if head.sequence >= self.lease.seq {
1430                        return Err(LeaseError::Invalid(
1431                            "recovery release head is outside the durable authority sequence"
1432                                .into(),
1433                        ));
1434                    }
1435                    if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1436                        && self.recovery_fault_revision != head.sequence
1437                    {
1438                        return Err(LeaseError::Invalid(
1439                            "settled recovery fault inventory is not bound to its release head"
1440                                .into(),
1441                        ));
1442                    }
1443                }
1444            }
1445        }
1446        Ok(())
1447    }
1448}
1449
1450/// Result of an acquisition or renewal attempt.
1451#[derive(Debug, Clone, PartialEq, Eq)]
1452pub enum LeaseOutcome {
1453    /// The caller's exact process incarnation owns the returned record.
1454    Acquired(LeaderLease),
1455    /// A rival exact process incarnation owns the returned record.
1456    Held(LeaderLease),
1457}
1458
1459enum AuthorityCreateOutcome {
1460    Created,
1461    ExistingIdentical,
1462    Contended(Box<LeaderAuthorityRecord>),
1463}
1464
1465#[derive(Clone, Copy)]
1466enum SameOwnerToken {
1467    #[cfg(test)]
1468    Preserve,
1469    Rotate,
1470    Exact(u64),
1471}
1472
1473/// Candidate-local proof that one rival liveness identity remained current for a full TTL.
1474#[derive(Debug)]
1475pub struct LeaderLeaseObservation {
1476    lease: LeaderLease,
1477    started: Instant,
1478}
1479
1480/// Coalescing-safe local candidacy state. A generation change means a published grant was lost
1481/// and must never reuse its fencing token, even when the current state is eligible again.
1482#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1483pub struct LeaderCandidacy {
1484    eligible: bool,
1485    generation: u64,
1486}
1487
1488impl LeaderCandidacy {
1489    pub(crate) const fn initial(eligible: bool) -> Self {
1490        Self {
1491            eligible,
1492            generation: 1,
1493        }
1494    }
1495
1496    pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1497        let generation = if self.eligible && !eligible {
1498            self.generation.checked_add(1)?
1499        } else {
1500            self.generation
1501        };
1502        Some(Self {
1503            eligible,
1504            generation,
1505        })
1506    }
1507
1508    pub(crate) const fn terminal() -> Self {
1509        Self {
1510            eligible: false,
1511            generation: u64::MAX,
1512        }
1513    }
1514
1515    /// Whether this process is currently eligible to contend for the leader lease.
1516    #[must_use]
1517    pub const fn is_eligible(self) -> bool {
1518        self.eligible
1519    }
1520}
1521
1522/// Leader lease storage or validation failure.
1523#[derive(Debug, thiserror::Error)]
1524pub enum LeaseError {
1525    /// Underlying object-store failure.
1526    #[error("object store I/O: {0}")]
1527    Io(String),
1528    /// Malformed configuration, owner, or durable record.
1529    #[error("invalid leader lease: {0}")]
1530    Invalid(String),
1531    /// A newer durable authority term superseded an exact renewal.
1532    #[error("leader lease fenced: {0}")]
1533    Fenced(String),
1534    /// JSON encoding or decoding failure.
1535    #[error("JSON: {0}")]
1536    Json(#[from] serde_json::Error),
1537}
1538
1539/// Failure while using the leader sequence as the cluster checkpoint-decision authority.
1540#[derive(Debug, thiserror::Error)]
1541pub enum ClusterCheckpointAuthorityError {
1542    /// Cluster runtime did not wire the durable leader authority.
1543    #[error("cluster checkpoint authority is not installed")]
1544    NotConfigured,
1545    /// Shared append-only authority failed.
1546    #[error("leader authority: {0}")]
1547    Authority(#[from] LeaseError),
1548    /// Checkpoint metadata or content-addressed recovery state was invalid.
1549    #[error("checkpoint decision: {0}")]
1550    Decision(#[from] DecisionError),
1551    /// The supplied proof no longer names the exact durable leader term.
1552    #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1553    Fenced,
1554}
1555
1556/// Exact continuity retained when old cluster checkpoint outcomes are compacted.
1557#[derive(Debug, Clone, PartialEq, Eq)]
1558pub struct ClusterOutcomeRetentionBoundary {
1559    /// Checkpoint artifacts below this epoch are eligible for garbage collection.
1560    pub artifact_before_epoch: u64,
1561    /// Terminal outcome links below this epoch have been compacted.
1562    pub terminal_before_epoch: u64,
1563    /// Greatest committed outcome below the artifact-retention horizon.
1564    pub committed_anchor: Option<CheckpointOutcome>,
1565    /// Greatest terminal outcome compacted from the authority chain, including aborts.
1566    pub terminal_anchor: Option<CheckpointOutcome>,
1567}
1568
1569impl ClusterOutcomeRetentionBoundary {
1570    fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1571        floor.map_or(
1572            Self {
1573                artifact_before_epoch: 0,
1574                terminal_before_epoch: 0,
1575                committed_anchor: None,
1576                terminal_anchor: None,
1577            },
1578            |floor| Self {
1579                artifact_before_epoch: floor.artifact_before_epoch,
1580                terminal_before_epoch: floor.authority_before_epoch,
1581                committed_anchor: floor.committed_anchor.clone(),
1582                terminal_anchor: floor.terminal_anchor.clone(),
1583            },
1584        )
1585    }
1586}
1587
1588/// Live cluster outcomes and their retention horizons from one audited authority head.
1589#[derive(Debug, Clone, PartialEq, Eq)]
1590pub struct ClusterOutcomeInventory {
1591    /// Outcomes at or above the artifact-retention horizon, in ascending epoch order.
1592    pub outcomes: Vec<CheckpointOutcome>,
1593    /// Artifact and terminal-history horizons paired with `outcomes`.
1594    pub retention_boundary: ClusterOutcomeRetentionBoundary,
1595}
1596
1597/// Append-only object-store authority for the cluster leader.
1598pub struct LeaderLeaseStore {
1599    store: Arc<dyn ObjectStore>,
1600    ttl_ms: i64,
1601    prune_running: Arc<AtomicBool>,
1602    outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
1603    outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
1604}
1605
1606#[derive(Clone, PartialEq, Eq)]
1607struct ClusterOutcomeAuditKey {
1608    terminal_head: Option<OutcomeLink>,
1609    commit_head: Option<OutcomeLink>,
1610    floor: Option<AuthorityOutcomeFloor>,
1611}
1612
1613struct ClusterOutcomeAuditCache {
1614    key: ClusterOutcomeAuditKey,
1615    authority_sequence: u64,
1616    snapshot: ClusterOutcomeAuditSnapshot,
1617}
1618
1619#[derive(Clone)]
1620struct ClusterOutcomeAuditSnapshot {
1621    outcomes: Arc<[CheckpointOutcome]>,
1622    terminal_links: Arc<[OutcomeLink]>,
1623    commit_links: Arc<[OutcomeLink]>,
1624}
1625
1626struct ClusterOutcomeAuditFlight {
1627    key: ClusterOutcomeAuditKey,
1628    gate: Weak<tokio::sync::Mutex<()>>,
1629}
1630
1631struct PruneLatchGuard(Arc<AtomicBool>);
1632
1633impl Drop for PruneLatchGuard {
1634    fn drop(&mut self) {
1635        self.0.store(false, Ordering::Release);
1636    }
1637}
1638
1639impl std::fmt::Debug for LeaderLeaseStore {
1640    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1641        formatter
1642            .debug_struct("LeaderLeaseStore")
1643            .field("ttl_ms", &self.ttl_ms)
1644            .finish_non_exhaustive()
1645    }
1646}
1647
1648impl LeaderLeaseStore {
1649    /// Create a leader lease authority.
1650    ///
1651    /// The store must provide linearizable `PutMode::Create`/`Update` and GET `ETag` or version
1652    /// metadata; unsupported conditional updates fail closed.
1653    #[must_use]
1654    pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
1655        Self {
1656            store,
1657            ttl_ms,
1658            prune_running: Arc::new(AtomicBool::new(false)),
1659            outcome_audit_cache: Mutex::new(None),
1660            outcome_audit_flights: Mutex::new(Vec::new()),
1661        }
1662    }
1663
1664    /// Verify the conditional-write contract required by the mutable authority head.
1665    ///
1666    /// Validation and cleanup each receive the supplied timeout. The probe uses a unique path and
1667    /// cleanup is attempted after every validation outcome.
1668    ///
1669    /// # Errors
1670    /// Returns an error when the store does not enforce native conditional writes, omits update
1671    /// metadata, times out, or cannot remove the probe.
1672    pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
1673        if timeout.is_zero() {
1674            return Err(LeaseError::Invalid(
1675                "object-store contract probe timeout must be nonzero".into(),
1676            ));
1677        }
1678
1679        let path = OsPath::from(format!(
1680            "{STORE_CONTRACT_PROBE_PREFIX}{}.probe",
1681            Uuid::new_v4()
1682        ));
1683        let validation = tokio::time::timeout(timeout, self.validate_store_contract_at(&path))
1684            .await
1685            .unwrap_or_else(|_| {
1686                Err(LeaseError::Io(format!(
1687                    "object-store contract probe exceeded {timeout:?}"
1688                )))
1689            });
1690        let cleanup = tokio::time::timeout(timeout, self.cleanup_store_contract_probe(&path))
1691            .await
1692            .unwrap_or_else(|_| {
1693                Err(LeaseError::Io(format!(
1694                    "object-store contract probe cleanup exceeded {timeout:?}"
1695                )))
1696            });
1697
1698        match (validation, cleanup) {
1699            (Ok(()), Ok(())) => Ok(()),
1700            (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
1701            (Err(validation), Err(cleanup)) => Err(LeaseError::Io(format!(
1702                "object-store contract validation failed ({validation}); cleanup failed ({cleanup})"
1703            ))),
1704        }
1705    }
1706
1707    async fn validate_store_contract_at(&self, path: &OsPath) -> Result<(), LeaseError> {
1708        self.store
1709            .put_opts(
1710                path,
1711                PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_V1)),
1712                PutOptions {
1713                    mode: PutMode::Create,
1714                    ..PutOptions::default()
1715                },
1716            )
1717            .await
1718            .map_err(|error| {
1719                LeaseError::Io(format!(
1720                    "object-store contract PutMode::Create failed: {error}"
1721                ))
1722            })?;
1723
1724        self.require_store_contract_create_conflict(path).await?;
1725
1726        let version_v1 = self
1727            .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V1)
1728            .await?;
1729        self.store
1730            .put_opts(
1731                path,
1732                PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_V2)),
1733                PutOptions {
1734                    mode: PutMode::Update(version_v1.clone()),
1735                    ..PutOptions::default()
1736                },
1737            )
1738            .await
1739            .map_err(|error| {
1740                LeaseError::Io(format!(
1741                    "object-store contract PutMode::Update failed: {error}"
1742                ))
1743            })?;
1744
1745        let version_v2 = self
1746            .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V2)
1747            .await?;
1748        if version_v2 == version_v1 {
1749            return Err(LeaseError::Invalid(
1750                "object-store conditional-update version did not change after PutMode::Update"
1751                    .into(),
1752            ));
1753        }
1754
1755        match self
1756            .store
1757            .put_opts(
1758                path,
1759                PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_STALE)),
1760                PutOptions {
1761                    mode: PutMode::Update(version_v1),
1762                    ..PutOptions::default()
1763                },
1764            )
1765            .await
1766        {
1767            Err(object_store::Error::Precondition { .. }) => {}
1768            Ok(_) => {
1769                return Err(LeaseError::Invalid(
1770                    "object-store accepted a stale PutMode::Update token".into(),
1771                ));
1772            }
1773            Err(error) => {
1774                return Err(LeaseError::Io(format!(
1775                    "object-store stale PutMode::Update failed unexpectedly: {error}"
1776                )));
1777            }
1778        }
1779
1780        let durable_version = self
1781            .read_store_contract_probe(path, STORE_CONTRACT_PROBE_V2)
1782            .await?;
1783        if durable_version != version_v2 {
1784            return Err(LeaseError::Invalid(
1785                "object-store version changed after rejecting stale PutMode::Update".into(),
1786            ));
1787        }
1788        Ok(())
1789    }
1790
1791    async fn require_store_contract_create_conflict(
1792        &self,
1793        path: &OsPath,
1794    ) -> Result<(), LeaseError> {
1795        match self
1796            .store
1797            .put_opts(
1798                path,
1799                PutPayload::from(Bytes::from_static(STORE_CONTRACT_PROBE_STALE)),
1800                PutOptions {
1801                    mode: PutMode::Create,
1802                    ..PutOptions::default()
1803                },
1804            )
1805            .await
1806        {
1807            Err(
1808                object_store::Error::AlreadyExists { .. }
1809                | object_store::Error::Precondition { .. },
1810            ) => Ok(()),
1811            Ok(_) => Err(LeaseError::Invalid(
1812                "object-store contract PutMode::Create overwrote an existing object".into(),
1813            )),
1814            Err(error) => Err(LeaseError::Io(format!(
1815                "object-store contract duplicate PutMode::Create failed unexpectedly: {error}"
1816            ))),
1817        }
1818    }
1819
1820    async fn read_store_contract_probe(
1821        &self,
1822        path: &OsPath,
1823        expected: &[u8],
1824    ) -> Result<UpdateVersion, LeaseError> {
1825        let result = self.store.get(path).await.map_err(|error| {
1826            LeaseError::Io(format!("object-store contract probe GET failed: {error}"))
1827        })?;
1828        let version = UpdateVersion {
1829            e_tag: result.meta.e_tag.clone(),
1830            version: result.meta.version.clone(),
1831        };
1832        if version.e_tag.is_none() && version.version.is_none() {
1833            return Err(LeaseError::Invalid(
1834                "object-store contract probe GET returned neither ETag nor version".into(),
1835            ));
1836        }
1837        if result.meta.size != u64::try_from(expected.len()).unwrap_or(u64::MAX) {
1838            return Err(LeaseError::Invalid(format!(
1839                "object-store contract probe is {} bytes; expected {}",
1840                result.meta.size,
1841                expected.len()
1842            )));
1843        }
1844        let bytes = result.bytes().await.map_err(|error| {
1845            LeaseError::Io(format!(
1846                "object-store contract probe payload read failed: {error}"
1847            ))
1848        })?;
1849        if bytes.as_ref() != expected {
1850            return Err(LeaseError::Invalid(
1851                "object-store contract probe content changed unexpectedly".into(),
1852            ));
1853        }
1854        Ok(version)
1855    }
1856
1857    async fn cleanup_store_contract_probe(&self, path: &OsPath) -> Result<(), LeaseError> {
1858        let delete_error = match self.store.delete(path).await {
1859            Ok(()) | Err(object_store::Error::NotFound { .. }) => None,
1860            Err(error) => Some(error.to_string()),
1861        };
1862        match self.store.get(path).await {
1863            Err(object_store::Error::NotFound { .. }) => Ok(()),
1864            Ok(_) => Err(LeaseError::Io(delete_error.map_or_else(
1865                || "object-store contract probe remained visible after deletion".into(),
1866                |error| {
1867                    format!(
1868                        "object-store contract probe deletion failed ({error}) and the object remained visible"
1869                    )
1870                },
1871            ))),
1872            Err(error) => Err(LeaseError::Io(delete_error.map_or_else(
1873                || format!("verify object-store contract probe deletion: {error}"),
1874                |delete| {
1875                    format!(
1876                        "object-store contract probe deletion failed ({delete}); verification failed ({error})"
1877                    )
1878                },
1879            ))),
1880        }
1881    }
1882
1883    fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
1884        RecoveryFaultInventory {
1885            revision: record.recovery_fault_revision,
1886            faults: record
1887                .recovery_fault_slots
1888                .iter()
1889                .filter(|slot| slot.active)
1890                .map(AuthorityRecoveryFaultSlot::fault)
1891                .collect(),
1892        }
1893    }
1894
1895    pub(crate) async fn recovery_fault_inventory(
1896        &self,
1897    ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
1898        let record = self
1899            .load_record()
1900            .await?
1901            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
1902        Ok(Self::recovery_fault_inventory_from(&record))
1903    }
1904
1905    pub(crate) async fn record_recovery_fault(
1906        &self,
1907        publisher: RecoveryFaultPublisher,
1908        request_sequence: u64,
1909    ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
1910        publisher.validate().map_err(LeaseError::Invalid)?;
1911        if request_sequence == 0 {
1912            return Err(LeaseError::Invalid(
1913                "recovery fault request sequence must be nonzero".into(),
1914            )
1915            .into());
1916        }
1917
1918        loop {
1919            let head = self
1920                .load_published_authority_head()
1921                .await?
1922                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
1923            let current = &head.record;
1924            let node_id = publisher.participant.node_id;
1925            let slot_index = current
1926                .recovery_fault_slots
1927                .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
1928            let (insert_at, replace) = match slot_index {
1929                Ok(index) => {
1930                    let slot = &current.recovery_fault_slots[index];
1931                    if slot.matches_request(publisher, request_sequence) {
1932                        return Ok(if slot.active {
1933                            RecordRecoveryFaultResult::Active
1934                        } else {
1935                            RecordRecoveryFaultResult::AlreadyCleared
1936                        });
1937                    }
1938                    if publisher.process_term < slot.publisher.process_term {
1939                        return Ok(RecordRecoveryFaultResult::Superseded);
1940                    }
1941                    if publisher.process_term == slot.publisher.process_term
1942                        && publisher.participant.boot_incarnation
1943                            != slot.publisher.participant.boot_incarnation
1944                    {
1945                        return Err(LeaseError::Invalid(format!(
1946                            "stable node {node_id} has two recovery fault publishers for process term {}",
1947                            publisher.process_term
1948                        ))
1949                        .into());
1950                    }
1951                    if publisher.process_term == slot.publisher.process_term
1952                        && request_sequence < slot.request_sequence
1953                    {
1954                        return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
1955                    }
1956                    (index, true)
1957                }
1958                Err(index) => (index, false),
1959            };
1960
1961            let sequence =
1962                current.lease.seq.checked_add(1).ok_or_else(|| {
1963                    LeaseError::Invalid("leader authority sequence exhausted".into())
1964                })?;
1965            let mut lease = current.lease.clone();
1966            lease.seq = sequence;
1967            let mut candidate = current.preserve_with_lease(lease);
1968            let slot = AuthorityRecoveryFaultSlot {
1969                publisher,
1970                request_sequence,
1971                // Authority sequence is globally monotonic even after an obsolete tombstone is
1972                // compacted. A delayed retry can therefore become a conservative fresh fault,
1973                // but can never alias a sequence already remembered by a recovery monitor.
1974                fault_sequence: sequence,
1975                active: true,
1976            };
1977            if replace {
1978                candidate.recovery_fault_slots[insert_at] = slot;
1979            } else {
1980                candidate.recovery_fault_slots.insert(insert_at, slot);
1981            }
1982            candidate.recovery_fault_revision = sequence;
1983            candidate.validate()?;
1984
1985            match self
1986                .create_authority_record(Some(&head), &candidate)
1987                .await?
1988            {
1989                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
1990                    return Ok(RecordRecoveryFaultResult::Active);
1991                }
1992                AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
1993            }
1994        }
1995    }
1996
1997    pub(crate) async fn authorize_recovery_release(
1998        &self,
1999        clearer: RecoveryFaultPublisher,
2000        terminal: &RecoveryAnnouncement,
2001    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2002        clearer.validate().map_err(LeaseError::Invalid)?;
2003        let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2004        let reporter = NodeId(clearer.participant.node_id);
2005        let expected = terminal.round.fault_sequence(reporter);
2006        let current = self
2007            .load_record()
2008            .await?
2009            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2010        let Some(release_head) = current.recovery_release_head.as_ref() else {
2011            return Ok(false);
2012        };
2013        if release_head.terminal != terminal_reference
2014            || current.recovery_fault_revision != release_head.sequence
2015        {
2016            return Ok(false);
2017        }
2018        if current.recovery_fault_slots.iter().any(|slot| slot.active) {
2019            return Ok(false);
2020        }
2021        let slot_index = current
2022            .recovery_fault_slots
2023            .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2024        let Some(expected_sequence) = expected else {
2025            return Ok(match slot_index {
2026                Ok(index) => !current.recovery_fault_slots[index].active,
2027                Err(_) => true,
2028            });
2029        };
2030        let Ok(index) = slot_index else {
2031            return Ok(false);
2032        };
2033        let slot = &current.recovery_fault_slots[index];
2034        Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2035    }
2036
2037    pub(crate) async fn stage_recovery_release_terminal(
2038        &self,
2039        terminal: &RecoveryAnnouncement,
2040    ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2041        let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2042        let path = recovery_release_terminal_path(&reference);
2043        let put_error = match self
2044            .store
2045            .put_opts(
2046                &path,
2047                PutPayload::from(encoded),
2048                PutOptions {
2049                    mode: PutMode::Create,
2050                    ..PutOptions::default()
2051                },
2052            )
2053            .await
2054        {
2055            Ok(_)
2056            | Err(
2057                object_store::Error::AlreadyExists { .. }
2058                | object_store::Error::Precondition { .. },
2059            ) => None,
2060            Err(error) => Some(error),
2061        };
2062        match self.load_recovery_release_terminal(&reference).await {
2063            Ok(stored) if stored == *terminal => Ok(reference),
2064            Ok(_) => Err(LeaseError::Invalid(format!(
2065                "recovery release terminal '{}' differs from the proposed content",
2066                reference.sha256
2067            ))
2068            .into()),
2069            Err(reconcile_error) => {
2070                if let Some(put_error) = put_error {
2071                    Err(LeaseError::Io(format!(
2072                        "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2073                    ))
2074                    .into())
2075                } else {
2076                    Err(reconcile_error)
2077                }
2078            }
2079        }
2080    }
2081
2082    pub(crate) async fn load_recovery_release_terminal(
2083        &self,
2084        reference: &RecoveryReleaseTerminalRef,
2085    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2086        reference.validate()?;
2087        let result = match self
2088            .store
2089            .get(&recovery_release_terminal_path(reference))
2090            .await
2091        {
2092            Ok(result) => result,
2093            Err(object_store::Error::NotFound { .. }) => {
2094                return Err(LeaseError::Invalid(format!(
2095                    "recovery release terminal '{}' is missing",
2096                    reference.sha256
2097                ))
2098                .into());
2099            }
2100            Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2101        };
2102        if result.meta.size != reference.encoded_len {
2103            return Err(LeaseError::Invalid(format!(
2104                "recovery release terminal '{}' is {} bytes, expected {}",
2105                reference.sha256, result.meta.size, reference.encoded_len
2106            ))
2107            .into());
2108        }
2109        let bytes = result
2110            .bytes()
2111            .await
2112            .map_err(|error| LeaseError::Io(error.to_string()))?;
2113        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2114            return Err(LeaseError::Invalid(format!(
2115                "recovery release terminal '{}' payload length changed while reading",
2116                reference.sha256
2117            ))
2118            .into());
2119        }
2120        let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2121            LeaseError::Invalid(format!(
2122                "recovery release terminal '{}': {error}",
2123                reference.sha256
2124            ))
2125        })?;
2126        let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2127        if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2128            return Err(LeaseError::Invalid(format!(
2129                "recovery release terminal '{}' does not match its content-addressed reference",
2130                reference.sha256
2131            ))
2132            .into());
2133        }
2134        Ok(terminal)
2135    }
2136
2137    pub(crate) async fn record_recovery_release_commit(
2138        &self,
2139        proof: &LeaderProof,
2140        reference: RecoveryReleaseTerminalRef,
2141    ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2142        reference.validate()?;
2143        let terminal = self.load_recovery_release_terminal(&reference).await?;
2144        if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2145            return Err(ClusterCheckpointAuthorityError::Fenced);
2146        }
2147
2148        loop {
2149            let published = self
2150                .load_published_authority_head()
2151                .await?
2152                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2153            let current = &published.record;
2154            if let Some(winner) = current.recovery_release_head.as_ref() {
2155                if winner.terminal == reference
2156                    || winner.terminal.generation() >= reference.generation()
2157                {
2158                    self.load_recovery_release_terminal(&winner.terminal)
2159                        .await?;
2160                    return if winner.terminal == reference {
2161                        Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2162                    } else {
2163                        Ok(RecordRecoveryReleaseCommitResult::Conflict {
2164                            winner: winner.terminal.clone(),
2165                        })
2166                    };
2167                }
2168            }
2169            if !current.lease.matches_proof(proof) {
2170                return Err(ClusterCheckpointAuthorityError::Fenced);
2171            }
2172            let fault_inventory = Self::recovery_fault_inventory_from(current);
2173            if fault_inventory.revision != terminal.round.fault_revision()
2174                || fault_inventory.faults != terminal.round.faults
2175            {
2176                return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2177            }
2178
2179            let base_sequence = current.lease.seq;
2180            let sequence = base_sequence
2181                .checked_add(1)
2182                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2183            let mut candidate = current.preserve_with_lease(LeaderLease {
2184                seq: sequence,
2185                renewal_sequence: current.lease.renewal_sequence,
2186                token: current.lease.token,
2187                owner: current.lease.owner.clone(),
2188                expires_at_ms: current.lease.expires_at_ms,
2189                catalog_manifest: current.lease.catalog_manifest.clone(),
2190            });
2191            // Only a process frozen into the stopped roster may consume this terminal. Covered
2192            // unavailable publishers remain fenced and conservatively republish if they return.
2193            candidate.recovery_fault_slots.retain(|slot| {
2194                slot.active
2195                    && terminal
2196                        .round
2197                        .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2198                        == Some(slot.publisher.participant.boot_incarnation)
2199            });
2200            for slot in &mut candidate.recovery_fault_slots {
2201                slot.active = false;
2202            }
2203            candidate.recovery_fault_revision = sequence;
2204            candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2205                terminal: reference.clone(),
2206                leader_proof: proof.clone(),
2207            });
2208            candidate.recovery_release_head = Some(RecoveryReleaseLink {
2209                sequence,
2210                terminal: reference.clone(),
2211            });
2212            candidate.validate()?;
2213
2214            match self
2215                .create_authority_record(Some(&published), &candidate)
2216                .await?
2217            {
2218                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2219                    return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2220                }
2221                AuthorityCreateOutcome::Contended(winner_head) => {
2222                    if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2223                        if winner.terminal == reference
2224                            || winner.terminal.generation() >= reference.generation()
2225                        {
2226                            self.load_recovery_release_terminal(&winner.terminal)
2227                                .await?;
2228                            return if winner.terminal == reference {
2229                                Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2230                            } else {
2231                                Ok(RecordRecoveryReleaseCommitResult::Conflict {
2232                                    winner: winner.terminal.clone(),
2233                                })
2234                            };
2235                        }
2236                    }
2237                    if !winner_head.lease.matches_proof(proof) {
2238                        return Err(ClusterCheckpointAuthorityError::Fenced);
2239                    }
2240                    if winner_head.lease.seq <= base_sequence {
2241                        return Err(LeaseError::Invalid(
2242                            "recovery release authority contention did not advance the sequence"
2243                                .into(),
2244                        )
2245                        .into());
2246                    }
2247                    tokio::task::yield_now().await;
2248                }
2249            }
2250        }
2251    }
2252
2253    async fn recovery_release_terminal_from(
2254        &self,
2255        head: &LeaderAuthorityRecord,
2256        link: &RecoveryReleaseLink,
2257    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2258        let admission = if link.sequence == head.lease.seq {
2259            head.clone()
2260        } else {
2261            read_authority_record(self.store.as_ref(), link.sequence)
2262                .await?
2263                .ok_or_else(|| {
2264                    LeaseError::Invalid(
2265                        "recovery release admission is missing from retained authority history"
2266                            .into(),
2267                    )
2268                })?
2269        };
2270        let commit = admission
2271            .recovery_release_commit
2272            .as_ref()
2273            .filter(|commit| {
2274                commit.terminal == link.terminal
2275                    && admission.recovery_release_head.as_ref() == Some(link)
2276            })
2277            .ok_or_else(|| {
2278                LeaseError::Invalid(
2279                    "recovery release head does not match its admitting authority record".into(),
2280                )
2281            })?;
2282        let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2283        if terminal.round.leader_proof != commit.leader_proof {
2284            return Err(LeaseError::Invalid(
2285                "recovery release terminal does not match its admitting leader proof".into(),
2286            )
2287            .into());
2288        }
2289        Ok(terminal)
2290    }
2291
2292    pub(crate) async fn recovery_admission_snapshot(
2293        &self,
2294    ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2295        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2296            let head = self
2297                .load_record()
2298                .await?
2299                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2300            let Some(link) = head.recovery_release_head.clone() else {
2301                return Ok(RecoveryAdmissionSnapshot {
2302                    committed_release: None,
2303                    fault_inventory: Self::recovery_fault_inventory_from(&head),
2304                    authority_sequence: head.lease.seq,
2305                    release_head: None,
2306                });
2307            };
2308            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2309            let rechecked = self
2310                .load_record()
2311                .await?
2312                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2313            if rechecked.lease.seq >= head.lease.seq
2314                && rechecked.recovery_release_head == head.recovery_release_head
2315                && rechecked.recovery_fault_revision == head.recovery_fault_revision
2316                && rechecked.recovery_fault_slots == head.recovery_fault_slots
2317            {
2318                return Ok(RecoveryAdmissionSnapshot {
2319                    committed_release: Some(terminal?),
2320                    fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2321                    authority_sequence: rechecked.lease.seq,
2322                    release_head: rechecked.recovery_release_head,
2323                });
2324            }
2325            tokio::task::yield_now().await;
2326        }
2327        Err(LeaseError::Io(format!(
2328            "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2329        ))
2330        .into())
2331    }
2332
2333    pub(crate) async fn recovery_admission_is_current(
2334        &self,
2335        snapshot: &RecoveryAdmissionSnapshot,
2336        leader_proof: &LeaderProof,
2337    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2338        let current = self
2339            .load_record()
2340            .await?
2341            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2342        let inventory = Self::recovery_fault_inventory_from(&current);
2343        Ok(current.lease.matches_proof(leader_proof)
2344            && current.lease.seq >= snapshot.authority_sequence
2345            && current.recovery_release_head == snapshot.release_head
2346            && inventory == snapshot.fault_inventory
2347            && inventory.faults().is_empty())
2348    }
2349
2350    pub(crate) async fn latest_recovery_release_terminal(
2351        &self,
2352    ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2353        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2354            let Some(head) = self.load_record().await? else {
2355                return Ok(None);
2356            };
2357            let Some(link) = head.recovery_release_head.clone() else {
2358                return Ok(None);
2359            };
2360            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2361            let rechecked = self
2362                .load_record()
2363                .await?
2364                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2365            if rechecked.recovery_release_head.as_ref() == Some(&link) {
2366                return Ok(Some(terminal?));
2367            }
2368            tokio::task::yield_now().await;
2369        }
2370        Err(LeaseError::Io(format!(
2371            "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2372        ))
2373        .into())
2374    }
2375
2376    fn ttl(&self) -> Result<Duration, LeaseError> {
2377        let ttl = u64::try_from(self.ttl_ms)
2378            .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2379        if ttl == 0 {
2380            return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2381        }
2382        Ok(Duration::from_millis(ttl))
2383    }
2384
2385    fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2386        now_ms
2387            .checked_add(self.ttl_ms)
2388            .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2389    }
2390
2391    #[cfg(test)]
2392    async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2393        let prefix = OsPath::from(LEASE_PREFIX);
2394        let mut entries = self.store.list(Some(&prefix));
2395        let mut sequences = Vec::new();
2396        while let Some(entry) = entries.next().await {
2397            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2398            if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2399                return Err(LeaseError::Invalid(format!(
2400                    "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2401                )));
2402            }
2403            sequences.push(lease_sequence_from_path(&entry.location)?);
2404        }
2405        sequences.sort_unstable();
2406        sequences.dedup();
2407        Ok(sequences)
2408    }
2409
2410    fn schedule_history_prune(&self) {
2411        if self
2412            .prune_running
2413            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2414            .is_err()
2415        {
2416            return;
2417        }
2418        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2419            self.prune_running.store(false, Ordering::Release);
2420            return;
2421        };
2422        let store = Arc::clone(&self.store);
2423        let prune_running = Arc::clone(&self.prune_running);
2424        let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2425        runtime.spawn(async move {
2426            let _latch = PruneLatchGuard(prune_running);
2427            match tokio::time::timeout(
2428                LEADER_LEASE_PRUNE_TIMEOUT,
2429                Self::prune_history(&store, grace_ms),
2430            )
2431            .await
2432            {
2433                Ok(Ok(())) => {}
2434                Ok(Err(error)) => {
2435                    tracing::warn!(%error, "leader lease history prune failed");
2436                }
2437                Err(_) => {
2438                    tracing::warn!(
2439                        timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2440                        "leader lease history prune timed out"
2441                    );
2442                }
2443            }
2444        });
2445    }
2446
2447    async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2448        let authority = Self::new(Arc::clone(store), 1);
2449        let Some(head) = authority.load_record().await? else {
2450            return Ok(());
2451        };
2452        let head_sequence = head.lease.seq;
2453        let mut retained = BTreeSet::from([head_sequence]);
2454        if let Some(previous) = head_sequence
2455            .checked_sub(1)
2456            .filter(|sequence| *sequence != 0)
2457        {
2458            retained.insert(previous);
2459        }
2460        let floor = head
2461            .outcome_floor
2462            .as_ref()
2463            .map_or(0, |floor| floor.authority_before_epoch);
2464        let artifact_floor = head
2465            .outcome_floor
2466            .as_ref()
2467            .map_or(0, |floor| floor.artifact_before_epoch);
2468        let mut link = head.outcome_head;
2469        let mut outcome_links = 0;
2470        let mut terminal_commit_links = BTreeSet::new();
2471        let mut expected_commit_head = head.commit_head;
2472        while let Some(current) = link {
2473            if current.epoch < floor {
2474                if head
2475                    .outcome_floor
2476                    .as_ref()
2477                    .and_then(|floor| floor.terminal_anchor_link)
2478                    != Some(current)
2479                {
2480                    return Err(LeaseError::Invalid(
2481                        "outcome floor does not anchor the retained authority chain".into(),
2482                    ));
2483                }
2484                break;
2485            }
2486            if !consume_live_authority_link(&mut outcome_links) {
2487                return Err(LeaseError::Invalid(format!(
2488                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2489                )));
2490            }
2491            let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2492                .await?
2493                .filter(|record| {
2494                    record.outcome_head == Some(current)
2495                        && record
2496                            .checkpoint_outcome
2497                            .as_ref()
2498                            .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2499                            == Some((current.epoch, current.checkpoint_id))
2500                })
2501                .ok_or_else(|| {
2502                    LeaseError::Invalid("retained outcome authority chain is broken".into())
2503                })?;
2504            if outcome_record.commit_head != expected_commit_head {
2505                return Err(LeaseError::Invalid(
2506                    "retained outcome chain did not preserve the exact Commit head".into(),
2507                ));
2508            }
2509            retained.insert(current.sequence);
2510            if outcome_record
2511                .checkpoint_outcome
2512                .as_ref()
2513                .is_some_and(CheckpointOutcome::is_commit)
2514            {
2515                terminal_commit_links.insert(current);
2516                expected_commit_head = outcome_record.previous_commit;
2517            } else {
2518                expected_commit_head = outcome_record.commit_head;
2519            }
2520            link = outcome_record.previous_outcome;
2521        }
2522        let mut commit_link = head.commit_head;
2523        let mut commit_links = 0;
2524        let mut retained_commit_links = BTreeSet::new();
2525        while let Some(current) = commit_link {
2526            if current.epoch < artifact_floor {
2527                break;
2528            }
2529            if !consume_live_authority_link(&mut commit_links) {
2530                return Err(LeaseError::Invalid(format!(
2531                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2532                )));
2533            }
2534            let commit_record = read_authority_record(store.as_ref(), current.sequence)
2535                .await?
2536                .filter(|record| {
2537                    record.commit_head == Some(current)
2538                        && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2539                            outcome.is_commit()
2540                                && outcome.epoch == current.epoch
2541                                && outcome.checkpoint_id == current.checkpoint_id
2542                        })
2543                })
2544                .ok_or_else(|| {
2545                    LeaseError::Invalid("retained Commit authority chain is broken".into())
2546                })?;
2547            retained.insert(current.sequence);
2548            retained_commit_links.insert(current);
2549            commit_link = commit_record.previous_commit;
2550        }
2551        match (
2552            commit_link,
2553            head.outcome_floor
2554                .as_ref()
2555                .and_then(|floor| floor.committed_anchor_link),
2556            head.outcome_floor
2557                .as_ref()
2558                .and_then(|floor| floor.committed_anchor.as_ref()),
2559        ) {
2560            (Some(link), Some(anchor_link), Some(anchor))
2561                if link == anchor_link
2562                    && link.epoch == anchor.epoch
2563                    && link.checkpoint_id == anchor.checkpoint_id => {}
2564            (None, None, None) => {}
2565            _ => {
2566                return Err(LeaseError::Invalid(
2567                    "Commit chain does not meet the retained artifact-floor anchor".into(),
2568                ));
2569            }
2570        }
2571        let boundary_commit_head = head
2572            .outcome_floor
2573            .as_ref()
2574            .and_then(|floor| floor.terminal_anchor.as_ref())
2575            .and_then(|anchor| {
2576                retained_commit_links
2577                    .iter()
2578                    .rev()
2579                    .find(|link| link.epoch <= anchor.epoch)
2580                    .copied()
2581                    .or_else(|| {
2582                        head.outcome_floor
2583                            .as_ref()
2584                            .and_then(|floor| floor.committed_anchor_link)
2585                    })
2586            });
2587        if expected_commit_head != boundary_commit_head {
2588            return Err(LeaseError::Invalid(
2589                "retained terminal chain lost Commit continuity at its durable floor".into(),
2590            ));
2591        }
2592        if !terminal_commit_links.is_subset(&retained_commit_links) {
2593            return Err(LeaseError::Invalid(
2594                "terminal Commit records are not linked from the retained Commit chain".into(),
2595            ));
2596        }
2597        if let Some((anchor, anchor_link)) = head.outcome_floor.as_ref().and_then(|floor| {
2598            floor
2599                .terminal_anchor
2600                .as_ref()
2601                .zip(floor.terminal_anchor_link)
2602        }) {
2603            if anchor.is_commit() {
2604                let linked = if anchor.epoch >= artifact_floor {
2605                    retained_commit_links.contains(&anchor_link)
2606                } else {
2607                    head.outcome_floor.as_ref().is_some_and(|floor| {
2608                        floor.committed_anchor.as_ref() == Some(anchor)
2609                            && floor.committed_anchor_link == Some(anchor_link)
2610                    })
2611                };
2612                if !linked {
2613                    return Err(LeaseError::Invalid(
2614                        "terminal Commit anchor is not linked from the retained Commit chain"
2615                            .into(),
2616                    ));
2617                }
2618            }
2619        }
2620        let mut assignment_link = head.assignment_decision_head;
2621        let mut assignment_links = 0;
2622        while let Some(current) = assignment_link {
2623            if !consume_live_authority_link(&mut assignment_links) {
2624                return Err(LeaseError::Invalid(format!(
2625                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2626                )));
2627            }
2628            if head
2629                .assignment_decision_floor
2630                .as_ref()
2631                .is_some_and(|floor| current.target_version < floor.before_target_version)
2632            {
2633                if head
2634                    .assignment_decision_floor
2635                    .as_ref()
2636                    .and_then(|floor| floor.terminal_anchor_link)
2637                    != Some(current)
2638                {
2639                    return Err(LeaseError::Invalid(
2640                        "assignment-decision floor does not anchor the retained authority chain"
2641                            .into(),
2642                    ));
2643                }
2644                break;
2645            }
2646            let decision_record = read_authority_record(store.as_ref(), current.sequence)
2647                .await?
2648                .filter(|record| {
2649                    record.assignment_decision_head == Some(current)
2650                        && record
2651                            .assignment_decision
2652                            .as_ref()
2653                            .map(AuthorityAssignmentDecision::target_version)
2654                            == Some(current.target_version)
2655                })
2656                .ok_or_else(|| {
2657                    LeaseError::Invalid(
2658                        "retained assignment-decision authority chain is broken".into(),
2659                    )
2660                })?;
2661            retained.insert(current.sequence);
2662            assignment_link = decision_record.previous_assignment_decision;
2663        }
2664        let retained_release = match head.recovery_release_head.as_ref() {
2665            Some(link) => {
2666                let admission = read_authority_record(store.as_ref(), link.sequence)
2667                    .await?
2668                    .filter(|record| {
2669                        record.recovery_release_head.as_ref() == Some(link)
2670                            && record
2671                                .recovery_release_commit
2672                                .as_ref()
2673                                .is_some_and(|commit| commit.terminal == link.terminal)
2674                    })
2675                    .ok_or_else(|| {
2676                        LeaseError::Invalid(
2677                            "retained recovery release authority link is broken".into(),
2678                        )
2679                    })?;
2680                if admission.lease.seq != link.sequence {
2681                    return Err(LeaseError::Invalid(
2682                        "recovery release admission sequence does not match its retained link"
2683                            .into(),
2684                    ));
2685                }
2686                let terminal = authority
2687                    .load_recovery_release_terminal(&link.terminal)
2688                    .await
2689                    .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2690                if admission
2691                    .recovery_release_commit
2692                    .as_ref()
2693                    .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2694                {
2695                    return Err(LeaseError::Invalid(
2696                        "retained recovery release terminal does not match its admission".into(),
2697                    ));
2698                }
2699                retained.insert(link.sequence);
2700                Some(link.terminal.clone())
2701            }
2702            None => None,
2703        };
2704
2705        let mut history_exhausted = false;
2706        for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2707            let (candidates, exhausted) =
2708                Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2709            if candidates.is_empty() {
2710                history_exhausted = true;
2711                break;
2712            }
2713            let deletions =
2714                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2715            let mut results = store.delete_stream(Box::pin(deletions));
2716            while let Some(result) = results.next().await {
2717                if let Err(error) = result {
2718                    if !matches!(error, object_store::Error::NotFound { .. }) {
2719                        return Err(LeaseError::Io(error.to_string()));
2720                    }
2721                }
2722            }
2723            if exhausted {
2724                history_exhausted = true;
2725                break;
2726            }
2727            tokio::task::yield_now().await;
2728        }
2729        if !history_exhausted {
2730            return Err(LeaseError::Io(
2731                "leader lease history still exceeds the bounded prune budget".into(),
2732            ));
2733        }
2734        Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await
2735    }
2736
2737    async fn prune_recovery_release_terminals(
2738        store: &Arc<dyn ObjectStore>,
2739        retained: Option<&RecoveryReleaseTerminalRef>,
2740        grace_ms: i64,
2741    ) -> Result<(), LeaseError> {
2742        let Some(retained) = retained else {
2743            return Ok(());
2744        };
2745        retained.validate()?;
2746        for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
2747            let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
2748            let mut listed = store.list(Some(&prefix));
2749            let now = now_millis();
2750            let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
2751            let mut exhausted = true;
2752            while let Some(entry) = listed.next().await {
2753                let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2754                let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
2755                let canonical = OsPath::from(format!(
2756                    "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
2757                ));
2758                if canonical != entry.location {
2759                    return Err(LeaseError::Invalid(format!(
2760                        "noncanonical recovery release terminal path {}",
2761                        entry.location
2762                    )));
2763                }
2764                if entry.location == recovery_release_terminal_path(retained)
2765                    || generation > retained.generation()
2766                    || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2767                {
2768                    continue;
2769                }
2770                candidates.push(entry.location);
2771                if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
2772                    exhausted = false;
2773                    break;
2774                }
2775            }
2776            if candidates.is_empty() {
2777                return Ok(());
2778            }
2779            let deletions =
2780                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2781            let mut results = store.delete_stream(Box::pin(deletions));
2782            while let Some(result) = results.next().await {
2783                if let Err(error) = result {
2784                    if !matches!(error, object_store::Error::NotFound { .. }) {
2785                        return Err(LeaseError::Io(error.to_string()));
2786                    }
2787                }
2788            }
2789            if exhausted {
2790                return Ok(());
2791            }
2792            tokio::task::yield_now().await;
2793        }
2794        Ok(())
2795    }
2796
2797    async fn prune_candidates(
2798        store: &Arc<dyn ObjectStore>,
2799        retained: &BTreeSet<u64>,
2800        snapshot_head_sequence: u64,
2801        grace_ms: i64,
2802    ) -> Result<(Vec<OsPath>, bool), LeaseError> {
2803        let prefix = OsPath::from(LEASE_PREFIX);
2804        let mut listed = store.list(Some(&prefix));
2805        let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
2806        let now = now_millis();
2807        let mut exhausted = true;
2808        while let Some(entry) = listed.next().await {
2809            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2810            let sequence = lease_sequence_from_path(&entry.location)?;
2811            if sequence >= snapshot_head_sequence
2812                || retained.contains(&sequence)
2813                || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2814            {
2815                continue;
2816            }
2817            candidates.push(entry.location);
2818            if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
2819                exhausted = false;
2820                break;
2821            }
2822        }
2823        Ok((candidates, exhausted))
2824    }
2825
2826    /// Load the highest durable sequence.
2827    ///
2828    /// # Errors
2829    /// Fails closed on object-store I/O or malformed durable state.
2830    pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
2831        Ok(self.load_record().await?.map(|record| record.lease))
2832    }
2833
2834    async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
2835        Ok(self
2836            .load_published_authority_head()
2837            .await?
2838            .map(|head| head.record))
2839    }
2840
2841    async fn load_published_authority_head(
2842        &self,
2843    ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
2844        for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2845            let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
2846                let Some(discovered) = self.discover_authority_head().await? else {
2847                    if read_authority_head_pointer(self.store.as_ref())
2848                        .await?
2849                        .is_none()
2850                    {
2851                        return Ok(None);
2852                    }
2853                    if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2854                        tokio::task::yield_now().await;
2855                        continue;
2856                    }
2857                    break;
2858                };
2859                self.publish_authority_head(discovered.lease.seq, None)
2860                    .await?;
2861                if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2862                    tokio::task::yield_now().await;
2863                    continue;
2864                }
2865                break;
2866            };
2867
2868            let sequence = pointer.pointer.sequence;
2869            let successor_sequence = sequence.checked_add(1);
2870            let (record, successor) = if let Some(successor_sequence) = successor_sequence {
2871                tokio::try_join!(
2872                    read_authority_record(self.store.as_ref(), sequence),
2873                    read_authority_record(self.store.as_ref(), successor_sequence)
2874                )?
2875            } else {
2876                (
2877                    read_authority_record(self.store.as_ref(), sequence).await?,
2878                    None,
2879                )
2880            };
2881            let Some(record) = record else {
2882                let rechecked = read_authority_head_pointer(self.store.as_ref())
2883                    .await?
2884                    .ok_or_else(|| {
2885                        LeaseError::Invalid(
2886                            "leader authority head disappeared while reading its target".into(),
2887                        )
2888                    })?;
2889                if rechecked.pointer.sequence > sequence {
2890                    if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2891                        tokio::task::yield_now().await;
2892                        continue;
2893                    }
2894                    break;
2895                }
2896                if rechecked.pointer.sequence < sequence {
2897                    return Err(LeaseError::Invalid(format!(
2898                        "leader authority head regressed from sequence {sequence} to {}",
2899                        rechecked.pointer.sequence
2900                    )));
2901                }
2902                return Err(LeaseError::Invalid(format!(
2903                    "leader authority head points ahead to missing sequence {sequence}"
2904                )));
2905            };
2906            let Some(successor_sequence) = successor_sequence else {
2907                return Ok(Some(PublishedAuthorityHead { record, pointer }));
2908            };
2909            if successor.is_none() {
2910                return Ok(Some(PublishedAuthorityHead { record, pointer }));
2911            }
2912
2913            if let Some(after_successor) = successor_sequence.checked_add(1) {
2914                if read_authority_record(self.store.as_ref(), after_successor)
2915                    .await?
2916                    .is_some()
2917                {
2918                    let rechecked = read_authority_head_pointer(self.store.as_ref())
2919                        .await?
2920                        .ok_or_else(|| {
2921                            LeaseError::Invalid(
2922                                "leader authority head disappeared while checking pointer lag"
2923                                    .into(),
2924                            )
2925                        })?;
2926                    if rechecked.pointer.sequence == sequence {
2927                        return Err(LeaseError::Invalid(format!(
2928                            "leader authority head at sequence {sequence} lags by more than one record"
2929                        )));
2930                    }
2931                    if rechecked.pointer.sequence < sequence {
2932                        return Err(LeaseError::Invalid(format!(
2933                            "leader authority head regressed from sequence {sequence} to {}",
2934                            rechecked.pointer.sequence
2935                        )));
2936                    }
2937                    if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2938                        tokio::task::yield_now().await;
2939                        continue;
2940                    }
2941                    break;
2942                }
2943            }
2944
2945            self.publish_authority_head(successor_sequence, Some(&pointer))
2946                .await?;
2947            if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
2948                tokio::task::yield_now().await;
2949                continue;
2950            }
2951            break;
2952        }
2953        Err(LeaseError::Io(format!(
2954            "leader authority head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2955        )))
2956    }
2957
2958    async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
2959        let prefix = OsPath::from(LEASE_PREFIX);
2960        let mut listed = self.store.list(Some(&prefix));
2961        let mut discovered = 0usize;
2962        let mut maximum = None;
2963        while let Some(entry) = listed.next().await {
2964            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2965            discovered = discovered.checked_add(1).ok_or_else(|| {
2966                LeaseError::Invalid("leader authority discovery count overflowed".into())
2967            })?;
2968            if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
2969                return Err(LeaseError::Invalid(format!(
2970                    "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
2971                )));
2972            }
2973            let sequence = lease_sequence_from_path(&entry.location)?;
2974            maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
2975        }
2976        let Some(sequence) = maximum else {
2977            return Ok(None);
2978        };
2979        read_authority_record(self.store.as_ref(), sequence)
2980            .await?
2981            .map(Some)
2982            .ok_or_else(|| {
2983                LeaseError::Io(format!(
2984                    "discovered leader authority sequence {sequence} vanished before publication"
2985                ))
2986            })
2987    }
2988
2989    async fn publish_authority_head(
2990        &self,
2991        sequence: u64,
2992        expected: Option<&VersionedAuthorityHeadPointer>,
2993    ) -> Result<u64, LeaseError> {
2994        if let Some(expected) = expected {
2995            if expected.pointer.sequence.checked_add(1) != Some(sequence) {
2996                return Err(LeaseError::Invalid(
2997                    "leader authority head update is not the exact successor".into(),
2998                ));
2999            }
3000            if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3001            {
3002                return Err(LeaseError::Invalid(
3003                    "leader authority store did not provide a native conditional update version"
3004                        .into(),
3005                ));
3006            }
3007        }
3008
3009        let options = PutOptions {
3010            mode: expected.map_or(PutMode::Create, |expected| {
3011                PutMode::Update(expected.update_version.clone())
3012            }),
3013            ..PutOptions::default()
3014        };
3015        let result = self
3016            .store
3017            .put_opts(
3018                &authority_head_path(),
3019                PutPayload::from(encode_authority_head_pointer(sequence)?),
3020                options,
3021            )
3022            .await;
3023        if result.is_ok() {
3024            return Ok(sequence);
3025        }
3026        let write_error = result
3027            .expect_err("successful authority head writes return above")
3028            .to_string();
3029        let current = read_authority_head_pointer(self.store.as_ref())
3030            .await?
3031            .ok_or_else(|| {
3032                LeaseError::Io(format!(
3033                    "leader authority head write failed ({write_error}) and no pointer was durable"
3034                ))
3035            })?;
3036        if let Some(expected) = expected {
3037            if current.pointer.sequence < expected.pointer.sequence {
3038                return Err(LeaseError::Invalid(format!(
3039                    "leader authority head regressed from sequence {} to {}",
3040                    expected.pointer.sequence, current.pointer.sequence
3041                )));
3042            }
3043        }
3044        if current.pointer.sequence >= sequence {
3045            if current.pointer.sequence > sequence {
3046                read_authority_record(self.store.as_ref(), current.pointer.sequence)
3047                    .await?
3048                    .ok_or_else(|| {
3049                        LeaseError::Invalid(format!(
3050                            "leader authority head points ahead to missing sequence {}",
3051                            current.pointer.sequence
3052                        ))
3053                    })?;
3054            }
3055            return Ok(current.pointer.sequence);
3056        }
3057        Err(LeaseError::Io(format!(
3058            "leader authority head write failed ({write_error}) and remained at sequence {}",
3059            current.pointer.sequence
3060        )))
3061    }
3062
3063    async fn read_catalog_manifest_blob(
3064        &self,
3065        reference: &CatalogManifestRef,
3066    ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3067        reference.validate()?;
3068        let path = reference.object_path();
3069        let result = match self.store.get(&path).await {
3070            Ok(result) => result,
3071            Err(object_store::Error::NotFound { .. }) => {
3072                return Err(CatalogManifestError::Invalid(format!(
3073                    "catalog manifest blob '{}' is missing",
3074                    reference.sha256
3075                )));
3076            }
3077            Err(error) => {
3078                return Err(CatalogManifestError::Authority(LeaseError::Io(
3079                    error.to_string(),
3080                )));
3081            }
3082        };
3083        if result.meta.size != reference.encoded_len {
3084            return Err(CatalogManifestError::Invalid(format!(
3085                "catalog manifest blob '{}' is {} bytes, expected {}",
3086                reference.sha256, result.meta.size, reference.encoded_len
3087            )));
3088        }
3089        let bytes = result
3090            .bytes()
3091            .await
3092            .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3093        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3094            return Err(CatalogManifestError::Invalid(format!(
3095                "catalog manifest blob '{}' payload length changed while reading",
3096                reference.sha256
3097            )));
3098        }
3099        let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3100        let (canonical, actual_reference) = manifest.encode_and_reference()?;
3101        if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3102            return Err(CatalogManifestError::Invalid(format!(
3103                "catalog manifest blob '{}' does not match its sealed reference",
3104                reference.sha256
3105            )));
3106        }
3107        Ok((manifest, bytes))
3108    }
3109
3110    pub(super) async fn load_catalog_manifest(
3111        &self,
3112        reference: &CatalogManifestRef,
3113    ) -> Result<CatalogManifest, CatalogManifestError> {
3114        self.read_catalog_manifest_blob(reference)
3115            .await
3116            .map(|(manifest, _)| manifest)
3117    }
3118
3119    async fn ensure_catalog_manifest_blob(
3120        &self,
3121        encoded: &[u8],
3122        reference: &CatalogManifestRef,
3123    ) -> Result<(), CatalogManifestError> {
3124        reference.validate()?;
3125        let options = PutOptions {
3126            mode: PutMode::Create,
3127            ..PutOptions::default()
3128        };
3129        let path = reference.object_path();
3130        let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3131        let put_error = match self.store.put_opts(&path, payload, options).await {
3132            Ok(_)
3133            | Err(
3134                object_store::Error::AlreadyExists { .. }
3135                | object_store::Error::Precondition { .. },
3136            ) => None,
3137            Err(error) => Some(error),
3138        };
3139
3140        match self.read_catalog_manifest_blob(reference).await {
3141            Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3142            Ok(_) => Err(CatalogManifestError::Invalid(format!(
3143                "catalog manifest blob '{}' differs from the proposed content",
3144                reference.sha256
3145            ))),
3146            Err(error) => {
3147                if let Some(put_error) = put_error {
3148                    Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3149                        "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3150                    ))))
3151                } else {
3152                    Err(error)
3153                }
3154            }
3155        }
3156    }
3157
3158    async fn create_authority_record(
3159        &self,
3160        expected: Option<&PublishedAuthorityHead>,
3161        candidate: &LeaderAuthorityRecord,
3162    ) -> Result<AuthorityCreateOutcome, LeaseError> {
3163        let encoded = encode_authority_record(candidate)?;
3164        let expected_pointer = match expected {
3165            None => {
3166                if candidate.lease.seq != 1 {
3167                    return Err(LeaseError::Invalid(format!(
3168                        "cannot append authority sequence {} to an empty published namespace",
3169                        candidate.lease.seq
3170                    )));
3171                }
3172                None
3173            }
3174            Some(head) if head.record.lease.seq == candidate.lease.seq => {
3175                return if head.record == *candidate {
3176                    Ok(AuthorityCreateOutcome::ExistingIdentical)
3177                } else {
3178                    Ok(AuthorityCreateOutcome::Contended(Box::new(
3179                        head.record.clone(),
3180                    )))
3181                };
3182            }
3183            Some(head) if head.record.lease.seq > candidate.lease.seq => {
3184                return Ok(AuthorityCreateOutcome::Contended(Box::new(
3185                    head.record.clone(),
3186                )));
3187            }
3188            Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3189                Some(&head.pointer)
3190            }
3191            Some(head) => {
3192                return Err(LeaseError::Invalid(format!(
3193                    "cannot append authority sequence {} after published sequence {}",
3194                    candidate.lease.seq, head.record.lease.seq
3195                )));
3196            }
3197        };
3198        let options = PutOptions {
3199            mode: PutMode::Create,
3200            ..PutOptions::default()
3201        };
3202        let result = self
3203            .store
3204            .put_opts(
3205                &lease_path(candidate.lease.seq),
3206                PutPayload::from(encoded),
3207                options,
3208            )
3209            .await;
3210        let created = result.is_ok();
3211        if let Err(error) = result {
3212            let Some(at_sequence) =
3213                read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3214            else {
3215                return Err(LeaseError::Io(format!(
3216                    "authority record {} write failed ({error}) and no record was durable",
3217                    candidate.lease.seq
3218                )));
3219            };
3220            if at_sequence != *candidate {
3221                let winner = self.load_record().await?.ok_or_else(|| {
3222                    LeaseError::Io("authority contender was not published or readable".into())
3223                })?;
3224                return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3225            }
3226        }
3227
3228        let published_sequence = self
3229            .publish_authority_head(candidate.lease.seq, expected_pointer)
3230            .await?;
3231        if published_sequence > candidate.lease.seq {
3232            if created {
3233                self.schedule_history_prune();
3234            }
3235            let winner = self
3236                .load_record()
3237                .await?
3238                .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3239            return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3240        }
3241        self.schedule_history_prune();
3242        if created {
3243            Ok(AuthorityCreateOutcome::Created)
3244        } else {
3245            Ok(AuthorityCreateOutcome::ExistingIdentical)
3246        }
3247    }
3248
3249    /// Seal the complete catalog by appending it to the exact durable leader term.
3250    ///
3251    /// Renewals that race this operation are retried under the same proof. A takeover either
3252    /// observes and preserves the seal or wins the next sequence and fences this writer.
3253    ///
3254    /// # Errors
3255    /// Rejects malformed inventories, divergent existing inventories, stale proofs, and durable
3256    /// storage failures.
3257    pub(super) async fn seal_catalog(
3258        &self,
3259        proof: &LeaderProof,
3260        manifest: &CatalogManifest,
3261    ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3262        if !proof.is_canonical() {
3263            return Err(CatalogManifestError::Fenced);
3264        }
3265        let (encoded, reference) = manifest.encode_and_reference()?;
3266
3267        let current = self
3268            .load_record()
3269            .await?
3270            .ok_or(CatalogManifestError::Fenced)?;
3271        if let Some(sealed) = &current.lease.catalog_manifest {
3272            let durable = self.load_catalog_manifest(sealed).await?;
3273            return if durable == *manifest {
3274                Ok(CatalogSealOutcome::ExistingIdentical)
3275            } else {
3276                Err(CatalogManifestError::Conflict)
3277            };
3278        }
3279        if !current.lease.matches_proof(proof) {
3280            return Err(CatalogManifestError::Fenced);
3281        }
3282        self.ensure_catalog_manifest_blob(&encoded, &reference)
3283            .await?;
3284
3285        loop {
3286            let published = self
3287                .load_published_authority_head()
3288                .await?
3289                .ok_or(CatalogManifestError::Fenced)?;
3290            let current = &published.record;
3291            if let Some(sealed) = &current.lease.catalog_manifest {
3292                let durable = self.load_catalog_manifest(sealed).await?;
3293                return if durable == *manifest {
3294                    Ok(CatalogSealOutcome::ExistingIdentical)
3295                } else {
3296                    Err(CatalogManifestError::Conflict)
3297                };
3298            }
3299            if !current.lease.matches_proof(proof) {
3300                return Err(CatalogManifestError::Fenced);
3301            }
3302
3303            let base_sequence = current.lease.seq;
3304            let candidate_lease = LeaderLease {
3305                seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3306                    CatalogManifestError::Authority(LeaseError::Invalid(
3307                        "lease sequence exhausted".into(),
3308                    ))
3309                })?,
3310                renewal_sequence: current.lease.renewal_sequence,
3311                token: current.lease.token,
3312                owner: current.lease.owner.clone(),
3313                expires_at_ms: current.lease.expires_at_ms,
3314                catalog_manifest: Some(reference.clone()),
3315            };
3316            let candidate = current.preserve_with_lease(candidate_lease);
3317            match self
3318                .create_authority_record(Some(&published), &candidate)
3319                .await?
3320            {
3321                AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3322                AuthorityCreateOutcome::ExistingIdentical => {
3323                    return Ok(CatalogSealOutcome::ExistingIdentical);
3324                }
3325                AuthorityCreateOutcome::Contended(winner) => {
3326                    if let Some(sealed) = &winner.lease.catalog_manifest {
3327                        let durable = self.load_catalog_manifest(sealed).await?;
3328                        return if durable == *manifest {
3329                            Ok(CatalogSealOutcome::ExistingIdentical)
3330                        } else {
3331                            Err(CatalogManifestError::Conflict)
3332                        };
3333                    }
3334                    if !winner.lease.matches_proof(proof) {
3335                        return Err(CatalogManifestError::Fenced);
3336                    }
3337                    if winner.lease.seq <= base_sequence {
3338                        return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3339                            "catalog seal contention did not advance the authority sequence".into(),
3340                        )));
3341                    }
3342                    tokio::task::yield_now().await;
3343                }
3344            }
3345        }
3346    }
3347
3348    async fn audited_cluster_outcomes_from(
3349        &self,
3350        head: &LeaderAuthorityRecord,
3351    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3352        let floor = head.outcome_floor.as_ref();
3353        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3354        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3355        let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3356        let mut terminal_newest_first = Vec::new();
3357        let mut terminal_links = Vec::new();
3358        let mut link = head.outcome_head;
3359        let mut expected_commit_head = head.commit_head;
3360        let mut traversed = 0;
3361        while let Some(current) = link {
3362            if current.epoch < authority_before_epoch {
3363                break;
3364            }
3365            if !consume_live_authority_link(&mut traversed) {
3366                return Err(DecisionError::Conflict(format!(
3367                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3368                ))
3369                .into());
3370            }
3371            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3372                record.clone()
3373            } else {
3374                let record = read_authority_record(self.store.as_ref(), current.sequence)
3375                    .await?
3376                    .ok_or_else(|| {
3377                        DecisionError::InventoryChanged(format!(
3378                            "cluster outcome authority record {} disappeared during audit",
3379                            current.sequence
3380                        ))
3381                    })?;
3382                terminal_records.insert(current.sequence, record.clone());
3383                record
3384            };
3385            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3386                DecisionError::Conflict(format!(
3387                    "cluster outcome head epoch {} points to non-outcome authority record {}",
3388                    current.epoch, current.sequence
3389                ))
3390            })?;
3391            if record.outcome_head != Some(current)
3392                || outcome.epoch != current.epoch
3393                || outcome.checkpoint_id != current.checkpoint_id
3394            {
3395                return Err(DecisionError::Conflict(format!(
3396                    "cluster outcome link epoch {} sequence {} does not match its authority record",
3397                    current.epoch, current.sequence
3398                ))
3399                .into());
3400            }
3401            if record.commit_head != expected_commit_head {
3402                return Err(DecisionError::Conflict(format!(
3403                    "cluster outcome epoch {} did not preserve the exact Commit head",
3404                    outcome.epoch
3405                ))
3406                .into());
3407            }
3408            expected_commit_head = if outcome.is_commit() {
3409                record.previous_commit
3410            } else {
3411                record.commit_head
3412            };
3413            terminal_newest_first.push(outcome);
3414            terminal_links.push(current);
3415            link = record.previous_outcome;
3416        }
3417
3418        if let Some(floor) = floor {
3419            match (
3420                link,
3421                floor.terminal_anchor_link,
3422                floor.terminal_anchor.as_ref(),
3423            ) {
3424                (Some(link), Some(anchor_link), Some(anchor))
3425                    if link == anchor_link
3426                        && link.epoch == anchor.epoch
3427                        && link.checkpoint_id == anchor.checkpoint_id => {}
3428                (None, None, None) => {}
3429                _ => {
3430                    return Err(DecisionError::Conflict(format!(
3431                        "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3432                        floor.authority_before_epoch
3433                    ))
3434                    .into());
3435                }
3436            }
3437        } else if link.is_some() {
3438            return Err(DecisionError::Conflict(
3439                "cluster outcome chain stopped without a durable retention floor".into(),
3440            )
3441            .into());
3442        }
3443
3444        let mut commit_newest_first = Vec::new();
3445        let mut commit_links = Vec::new();
3446        link = head.commit_head;
3447        traversed = 0;
3448        while let Some(current) = link {
3449            if current.epoch < artifact_before_epoch {
3450                break;
3451            }
3452            if !consume_live_authority_link(&mut traversed) {
3453                return Err(DecisionError::Conflict(format!(
3454                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3455                ))
3456                .into());
3457            }
3458            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3459                record.clone()
3460            } else {
3461                read_authority_record(self.store.as_ref(), current.sequence)
3462                    .await?
3463                    .ok_or_else(|| {
3464                        DecisionError::InventoryChanged(format!(
3465                            "cluster Commit authority record {} disappeared during audit",
3466                            current.sequence
3467                        ))
3468                    })?
3469            };
3470            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3471                DecisionError::Conflict(format!(
3472                    "cluster Commit head epoch {} points to non-outcome authority record {}",
3473                    current.epoch, current.sequence
3474                ))
3475            })?;
3476            if !outcome.is_commit()
3477                || record.commit_head != Some(current)
3478                || outcome.epoch != current.epoch
3479                || outcome.checkpoint_id != current.checkpoint_id
3480            {
3481                return Err(DecisionError::Conflict(format!(
3482                    "cluster Commit link epoch {} sequence {} does not match its authority record",
3483                    current.epoch, current.sequence
3484                ))
3485                .into());
3486            }
3487            commit_newest_first.push(outcome);
3488            commit_links.push(current);
3489            link = record.previous_commit;
3490        }
3491
3492        if let Some(floor) = floor {
3493            match (
3494                link,
3495                floor.committed_anchor_link,
3496                floor.committed_anchor.as_ref(),
3497            ) {
3498                (Some(link), Some(anchor_link), Some(anchor))
3499                    if link == anchor_link
3500                        && link.epoch == anchor.epoch
3501                        && link.checkpoint_id == anchor.checkpoint_id => {}
3502                (None, None, None) => {}
3503                _ => {
3504                    return Err(DecisionError::Conflict(format!(
3505                        "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3506                        floor.artifact_before_epoch
3507                    ))
3508                    .into());
3509                }
3510            }
3511        } else if link.is_some() {
3512            return Err(DecisionError::Conflict(
3513                "cluster Commit chain stopped without a durable retention floor".into(),
3514            )
3515            .into());
3516        }
3517
3518        terminal_newest_first.reverse();
3519        terminal_links.reverse();
3520        commit_newest_first.reverse();
3521        commit_links.reverse();
3522        let boundary_commit_head = floor
3523            .and_then(|floor| floor.terminal_anchor.as_ref())
3524            .and_then(|anchor| {
3525                commit_links
3526                    .iter()
3527                    .rev()
3528                    .find(|link| link.epoch <= anchor.epoch)
3529                    .copied()
3530                    .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3531            });
3532        if expected_commit_head != boundary_commit_head {
3533            return Err(DecisionError::Conflict(
3534                "cluster terminal chain does not preserve Commit continuity at its durable floor"
3535                    .into(),
3536            )
3537            .into());
3538        }
3539        for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3540            if !outcome.is_commit() {
3541                continue;
3542            }
3543            let commit_link = commit_links
3544                .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3545                .ok()
3546                .map(|index| commit_links[index]);
3547            if commit_link != Some(*terminal_link) {
3548                return Err(DecisionError::Conflict(format!(
3549                    "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3550                    outcome.epoch
3551                ))
3552                .into());
3553            }
3554        }
3555        if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3556            floor
3557                .terminal_anchor
3558                .as_ref()
3559                .zip(floor.terminal_anchor_link)
3560        }) {
3561            if terminal_anchor.is_commit() {
3562                let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3563                    commit_links
3564                        .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3565                        .ok()
3566                        .map(|index| commit_links[index])
3567                } else {
3568                    floor.and_then(|floor| {
3569                        (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3570                            .then_some(floor.committed_anchor_link)
3571                            .flatten()
3572                    })
3573                };
3574                if commit_link != Some(terminal_anchor_link) {
3575                    return Err(DecisionError::Conflict(format!(
3576                        "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3577                        terminal_anchor.epoch
3578                    ))
3579                    .into());
3580                }
3581            }
3582        }
3583        let mut outcomes = Vec::with_capacity(
3584            terminal_newest_first
3585                .len()
3586                .saturating_add(commit_newest_first.len())
3587                .saturating_add(2),
3588        );
3589        outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3590        outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3591        outcomes.extend(terminal_newest_first);
3592        outcomes.extend(commit_newest_first);
3593        outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3594        let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3595        for outcome in outcomes {
3596            if let Some(previous) = merged.last() {
3597                if previous.epoch == outcome.epoch {
3598                    if previous != &outcome {
3599                        return Err(DecisionError::Conflict(format!(
3600                            "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3601                            outcome.epoch
3602                        ))
3603                        .into());
3604                    }
3605                    continue;
3606                }
3607            }
3608            merged.push(outcome);
3609        }
3610
3611        let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3612            .load_or_create_deployment_id()
3613            .await?;
3614        for outcome in &merged {
3615            if outcome.deployment_id != expected_deployment {
3616                return Err(DecisionError::Conflict(format!(
3617                    "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3618                    outcome.epoch, outcome.deployment_id, expected_deployment
3619                ))
3620                .into());
3621            }
3622        }
3623        for pair in merged.windows(2) {
3624            let previous = &pair[0];
3625            let current = &pair[1];
3626            if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3627                return Err(DecisionError::Conflict(format!(
3628                    "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3629                    previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3630                ))
3631                .into());
3632            }
3633        }
3634        Ok(ClusterOutcomeAuditSnapshot {
3635            outcomes: Arc::from(merged),
3636            terminal_links: Arc::from(terminal_links),
3637            commit_links: Arc::from(commit_links),
3638        })
3639    }
3640
3641    fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3642        ClusterOutcomeAuditKey {
3643            terminal_head: head.outcome_head,
3644            commit_head: head.commit_head,
3645            floor: head.outcome_floor.clone(),
3646        }
3647    }
3648
3649    fn cached_cluster_outcome_audit(
3650        &self,
3651        key: &ClusterOutcomeAuditKey,
3652        authority_sequence: u64,
3653    ) -> Option<ClusterOutcomeAuditSnapshot> {
3654        let mut cache = self.outcome_audit_cache.lock();
3655        let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3656        cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3657        Some(cached.snapshot.clone())
3658    }
3659
3660    fn install_cluster_outcome_audit(
3661        &self,
3662        key: ClusterOutcomeAuditKey,
3663        authority_sequence: u64,
3664        snapshot: ClusterOutcomeAuditSnapshot,
3665    ) {
3666        let mut cache = self.outcome_audit_cache.lock();
3667        if cache.as_ref().is_none_or(|cached| {
3668            cached.authority_sequence < authority_sequence
3669                || (cached.authority_sequence == authority_sequence && cached.key == key)
3670        }) {
3671            *cache = Some(ClusterOutcomeAuditCache {
3672                key,
3673                authority_sequence,
3674                snapshot,
3675            });
3676        }
3677    }
3678
3679    fn cluster_outcome_audit_gate(
3680        &self,
3681        key: &ClusterOutcomeAuditKey,
3682    ) -> Arc<tokio::sync::Mutex<()>> {
3683        let mut flights = self.outcome_audit_flights.lock();
3684        flights.retain(|flight| flight.gate.strong_count() != 0);
3685        if let Some(gate) = flights
3686            .iter()
3687            .find(|flight| flight.key == *key)
3688            .and_then(|flight| flight.gate.upgrade())
3689        {
3690            return gate;
3691        }
3692        let gate = Arc::new(tokio::sync::Mutex::new(()));
3693        flights.push(ClusterOutcomeAuditFlight {
3694            key: key.clone(),
3695            gate: Arc::downgrade(&gate),
3696        });
3697        gate
3698    }
3699
3700    async fn cached_audited_cluster_outcomes_from(
3701        &self,
3702        head: &LeaderAuthorityRecord,
3703    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3704        let key = Self::cluster_outcome_audit_key(head);
3705        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3706            return Ok(snapshot);
3707        }
3708
3709        let gate = self.cluster_outcome_audit_gate(&key);
3710        let _guard = gate.lock().await;
3711        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3712            return Ok(snapshot);
3713        }
3714
3715        let snapshot = self.audited_cluster_outcomes_from(head).await?;
3716        self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
3717        Ok(snapshot)
3718    }
3719
3720    fn outcomes_retained_by_floor(
3721        floor: &AuthorityOutcomeFloor,
3722        snapshot: &ClusterOutcomeAuditSnapshot,
3723    ) -> ClusterOutcomeAuditSnapshot {
3724        let outcomes = &snapshot.outcomes;
3725        let mut retained = Vec::with_capacity(outcomes.len());
3726        if let Some(anchor) = floor.committed_anchor.as_ref() {
3727            retained.push(anchor.clone());
3728        }
3729        if let Some(anchor) = floor.terminal_anchor.as_ref() {
3730            if retained.last() != Some(anchor) {
3731                retained.push(anchor.clone());
3732            }
3733        }
3734        retained.extend(
3735            outcomes
3736                .iter()
3737                .filter(|outcome| {
3738                    outcome.epoch >= floor.authority_before_epoch
3739                        || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
3740                })
3741                .cloned(),
3742        );
3743        retained.sort_unstable_by_key(|outcome| outcome.epoch);
3744        retained.dedup();
3745        ClusterOutcomeAuditSnapshot {
3746            outcomes: Arc::from(retained),
3747            terminal_links: Arc::from(
3748                snapshot
3749                    .terminal_links
3750                    .iter()
3751                    .filter(|link| link.epoch >= floor.authority_before_epoch)
3752                    .copied()
3753                    .collect::<Vec<_>>(),
3754            ),
3755            commit_links: Arc::from(
3756                snapshot
3757                    .commit_links
3758                    .iter()
3759                    .filter(|link| link.epoch >= floor.artifact_before_epoch)
3760                    .copied()
3761                    .collect::<Vec<_>>(),
3762            ),
3763        }
3764    }
3765
3766    async fn build_cluster_outcome_floor(
3767        &self,
3768        current: &LeaderAuthorityRecord,
3769        snapshot: &ClusterOutcomeAuditSnapshot,
3770        artifact_before_epoch: u64,
3771        authority_before_epoch: u64,
3772    ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
3773        let terminal_anchor = snapshot
3774            .outcomes
3775            .iter()
3776            .rev()
3777            .find(|outcome| outcome.epoch < authority_before_epoch)
3778            .cloned();
3779        let terminal_anchor_link = match terminal_anchor.as_ref() {
3780            Some(anchor)
3781                if current
3782                    .outcome_floor
3783                    .as_ref()
3784                    .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
3785            {
3786                current
3787                    .outcome_floor
3788                    .as_ref()
3789                    .and_then(|floor| floor.terminal_anchor_link)
3790            }
3791            Some(anchor) => {
3792                let index = snapshot
3793                    .terminal_links
3794                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
3795                    .map_err(|_| {
3796                        DecisionError::Conflict(format!(
3797                            "cluster outcome epoch {} has no audited authority link",
3798                            anchor.epoch
3799                        ))
3800                    })?;
3801                let link = snapshot.terminal_links[index];
3802                if link.checkpoint_id != anchor.checkpoint_id {
3803                    return Err(DecisionError::Conflict(format!(
3804                        "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
3805                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
3806                    ))
3807                    .into());
3808                }
3809                Some(link)
3810            }
3811            None => None,
3812        };
3813        let committed_anchor = snapshot
3814            .outcomes
3815            .iter()
3816            .rev()
3817            .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
3818            .cloned();
3819        let committed_anchor_link = match committed_anchor.as_ref() {
3820            Some(anchor)
3821                if current
3822                    .outcome_floor
3823                    .as_ref()
3824                    .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
3825            {
3826                current
3827                    .outcome_floor
3828                    .as_ref()
3829                    .and_then(|floor| floor.committed_anchor_link)
3830            }
3831            Some(anchor) => {
3832                let index = snapshot
3833                    .commit_links
3834                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
3835                    .map_err(|_| {
3836                        DecisionError::Conflict(format!(
3837                            "cluster Commit epoch {} has no audited Commit link",
3838                            anchor.epoch
3839                        ))
3840                    })?;
3841                let link = snapshot.commit_links[index];
3842                if link.checkpoint_id != anchor.checkpoint_id {
3843                    return Err(DecisionError::Conflict(format!(
3844                        "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
3845                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
3846                    ))
3847                    .into());
3848                }
3849                Some(link)
3850            }
3851            None => None,
3852        };
3853        let floor = AuthorityOutcomeFloor {
3854            deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
3855                .load_or_create_deployment_id()
3856                .await?,
3857            artifact_before_epoch,
3858            authority_before_epoch,
3859            terminal_anchor,
3860            terminal_anchor_link,
3861            committed_anchor,
3862            committed_anchor_link,
3863        };
3864        floor.validate()?;
3865        Ok(floor)
3866    }
3867
3868    async fn compact_cluster_outcome_history_before_append(
3869        &self,
3870        proof: &LeaderProof,
3871        current: &LeaderAuthorityRecord,
3872        snapshot: &ClusterOutcomeAuditSnapshot,
3873    ) -> Result<bool, ClusterCheckpointAuthorityError> {
3874        if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
3875            return Ok(false);
3876        }
3877        let authority_before_epoch = snapshot.terminal_links
3878            [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
3879            .epoch;
3880        let artifact_before_epoch = current
3881            .outcome_floor
3882            .as_ref()
3883            .map_or(0, |floor| floor.artifact_before_epoch);
3884        let floor = self
3885            .build_cluster_outcome_floor(
3886                current,
3887                snapshot,
3888                artifact_before_epoch,
3889                authority_before_epoch,
3890            )
3891            .await?;
3892        // Floor construction may perform remote reads. Publish only if the audited chain and both
3893        // horizons are still the exact authority snapshot used to select the anchors.
3894        let published = self
3895            .load_published_authority_head()
3896            .await?
3897            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
3898        let rechecked = &published.record;
3899        if !rechecked.lease.matches_proof(proof) {
3900            return Err(ClusterCheckpointAuthorityError::Fenced);
3901        }
3902        if rechecked.outcome_head != current.outcome_head
3903            || rechecked.commit_head != current.commit_head
3904            || rechecked.outcome_floor != current.outcome_floor
3905        {
3906            return Ok(true);
3907        }
3908
3909        let sequence = rechecked
3910            .lease
3911            .seq
3912            .checked_add(1)
3913            .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
3914        let mut lease = rechecked.lease.clone();
3915        lease.seq = sequence;
3916        let mut next = rechecked.preserve_with_lease(lease);
3917        next.outcome_floor = Some(floor.clone());
3918        next.validate()?;
3919        match self
3920            .create_authority_record(Some(&published), &next)
3921            .await?
3922        {
3923            AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
3924                self.install_cluster_outcome_audit(
3925                    Self::cluster_outcome_audit_key(&next),
3926                    next.lease.seq,
3927                    Self::outcomes_retained_by_floor(&floor, snapshot),
3928                );
3929                Ok(true)
3930            }
3931            AuthorityCreateOutcome::Contended(winner) => {
3932                if !winner.lease.matches_proof(proof) {
3933                    return Err(ClusterCheckpointAuthorityError::Fenced);
3934                }
3935                if winner.lease.seq <= current.lease.seq {
3936                    return Err(LeaseError::Invalid(
3937                        "outcome history compaction contention did not advance the authority sequence"
3938                            .into(),
3939                    )
3940                    .into());
3941                }
3942                Ok(true)
3943            }
3944        }
3945    }
3946
3947    async fn audited_cluster_outcomes(
3948        &self,
3949    ) -> Result<
3950        (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
3951        ClusterCheckpointAuthorityError,
3952    > {
3953        const AUDIT_RETRIES: usize = 3;
3954        for attempt in 0..AUDIT_RETRIES {
3955            let Some(head) = self.load_record().await? else {
3956                *self.outcome_audit_cache.lock() = None;
3957                return Ok((None, Arc::from([])));
3958            };
3959            match self.cached_audited_cluster_outcomes_from(&head).await {
3960                Err(ClusterCheckpointAuthorityError::Decision(
3961                    DecisionError::InventoryChanged(_),
3962                )) if attempt + 1 < AUDIT_RETRIES => {
3963                    tokio::task::yield_now().await;
3964                }
3965                Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
3966                Err(error) => return Err(error),
3967            }
3968        }
3969        Err(DecisionError::InventoryChanged(
3970            "cluster outcome audit exhausted stability retries".into(),
3971        )
3972        .into())
3973    }
3974
3975    async fn audited_assignment_decisions_from(
3976        &self,
3977        head: &LeaderAuthorityRecord,
3978    ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
3979        let mut newest_first = Vec::new();
3980        let floor = head.assignment_decision_floor.as_ref();
3981        let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
3982        let mut stopped_at_anchor = false;
3983        let mut link = head.assignment_decision_head;
3984        let mut traversed = 0;
3985        while let Some(current) = link {
3986            if !consume_live_authority_link(&mut traversed) {
3987                return Err(DecisionError::Conflict(format!(
3988                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3989                ))
3990                .into());
3991            }
3992            if current.target_version < before_target_version {
3993                if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
3994                    return Err(DecisionError::Conflict(format!(
3995                        "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
3996                    ))
3997                    .into());
3998                }
3999                stopped_at_anchor = true;
4000                break;
4001            }
4002            let record = read_authority_record(self.store.as_ref(), current.sequence)
4003                .await?
4004                .ok_or_else(|| {
4005                    DecisionError::InventoryChanged(format!(
4006                        "assignment decision authority record {} disappeared during audit",
4007                        current.sequence
4008                    ))
4009                })?;
4010            let decision = record.assignment_decision.clone().ok_or_else(|| {
4011                DecisionError::Conflict(format!(
4012                    "assignment decision head version {} points to non-decision authority record {}",
4013                    current.target_version, current.sequence
4014                ))
4015            })?;
4016            if record.assignment_decision_head != Some(current)
4017                || decision.target_version() != current.target_version
4018            {
4019                return Err(DecisionError::Conflict(format!(
4020                    "assignment decision link version {} sequence {} does not match its authority record",
4021                    current.target_version, current.sequence
4022                ))
4023                .into());
4024            }
4025            newest_first.push(decision);
4026            link = record.previous_assignment_decision;
4027        }
4028        if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4029            return Err(DecisionError::Conflict(
4030                "assignment decision chain stopped without its durable retention anchor".into(),
4031            )
4032            .into());
4033        }
4034        newest_first.reverse();
4035        if let Some(pair) = newest_first
4036            .windows(2)
4037            .find(|pair| pair[0].target_version() >= pair[1].target_version())
4038        {
4039            return Err(DecisionError::Conflict(format!(
4040                "assignment decisions regress from version {} to {}",
4041                pair[0].target_version(),
4042                pair[1].target_version()
4043            ))
4044            .into());
4045        }
4046        Ok(newest_first)
4047    }
4048
4049    async fn exact_assignment_decision_link(
4050        &self,
4051        head: &LeaderAuthorityRecord,
4052        decision: &AuthorityAssignmentDecision,
4053    ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4054        if let Some(floor) = head.assignment_decision_floor.as_ref() {
4055            if floor.terminal_anchor.as_ref() == Some(decision) {
4056                return floor.terminal_anchor_link.ok_or_else(|| {
4057                    DecisionError::Conflict(
4058                        "assignment decision floor lost its terminal authority link".into(),
4059                    )
4060                    .into()
4061                });
4062            }
4063        }
4064        let before_target_version = head
4065            .assignment_decision_floor
4066            .as_ref()
4067            .map_or(0, |floor| floor.before_target_version);
4068        let mut link = head.assignment_decision_head;
4069        let mut traversed = 0;
4070        while let Some(current) = link {
4071            if !consume_live_authority_link(&mut traversed) {
4072                return Err(DecisionError::Conflict(format!(
4073                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4074                ))
4075                .into());
4076            }
4077            if current.target_version < before_target_version {
4078                break;
4079            }
4080            let record = read_authority_record(self.store.as_ref(), current.sequence)
4081                .await?
4082                .ok_or_else(|| {
4083                    DecisionError::InventoryChanged(format!(
4084                        "assignment decision authority record {} disappeared during exact lookup",
4085                        current.sequence
4086                    ))
4087                })?;
4088            if record.assignment_decision.as_ref() == Some(decision) {
4089                return Ok(current);
4090            }
4091            link = record.previous_assignment_decision;
4092        }
4093        Err(DecisionError::Conflict(format!(
4094            "assignment decision version {} is not linked from the durable authority head",
4095            decision.target_version()
4096        ))
4097        .into())
4098    }
4099
4100    /// Admit one cluster terminal outcome through the exact next leader-authority sequence.
4101    ///
4102    /// Renewals, takeovers, catalog seals, floor advances, and other decisions all contend on the
4103    /// same create-only object. An identical retry converges on the durable winner.
4104    ///
4105    /// # Errors
4106    /// Fails closed for a stale proof, non-monotonic or conflicting outcome, malformed recovery
4107    /// capsule, or object-store failure.
4108    pub async fn record_cluster_outcome(
4109        &self,
4110        proof: &LeaderProof,
4111        epoch: u64,
4112        checkpoint_id: u64,
4113        assignment_fence: CheckpointAssignmentFence,
4114        verdict: CheckpointVerdict,
4115        recovery_capsule: Option<RecoveryCapsuleRef>,
4116    ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4117        if !proof.is_canonical() {
4118            return Err(ClusterCheckpointAuthorityError::Fenced);
4119        }
4120        let attempt = crate::state::CheckpointAttempt::new(epoch, checkpoint_id);
4121        if !attempt.is_canonical() {
4122            return Err(DecisionError::Conflict(
4123                "cluster checkpoint outcomes require one nonzero canonical checkpoint ID".into(),
4124            )
4125            .into());
4126        }
4127        let initial = self
4128            .load_record()
4129            .await?
4130            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4131        if !initial.lease.matches_proof(proof) {
4132            return Err(ClusterCheckpointAuthorityError::Fenced);
4133        }
4134        let candidate = CheckpointDecisionStore::new(Arc::clone(&self.store))
4135            .canonical_outcome(
4136                epoch,
4137                checkpoint_id,
4138                CheckpointScope::Cluster,
4139                Some(assignment_fence),
4140                Some(proof.clone()),
4141                verdict,
4142                recovery_capsule,
4143            )
4144            .await?;
4145
4146        loop {
4147            let published = self
4148                .load_published_authority_head()
4149                .await?
4150                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4151            let current = &published.record;
4152            if !current.lease.matches_proof(proof) {
4153                return Err(ClusterCheckpointAuthorityError::Fenced);
4154            }
4155            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4156            let outcomes = &snapshot.outcomes;
4157            if let Some(winner) = outcomes
4158                .iter()
4159                .find(|outcome| outcome.epoch == candidate.epoch)
4160            {
4161                return if winner == &candidate {
4162                    Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4163                } else {
4164                    Ok(RecordOutcomeResult::Conflict {
4165                        winner: winner.clone(),
4166                    })
4167                };
4168            }
4169            if let Some(last) = outcomes.last() {
4170                if candidate.checkpoint_id <= last.checkpoint_id {
4171                    return Err(DecisionError::Conflict(format!(
4172                        "cluster checkpoint {} does not advance durable checkpoint {}",
4173                        candidate.checkpoint_id, last.checkpoint_id
4174                    ))
4175                    .into());
4176                }
4177            }
4178            if candidate.is_commit() && snapshot.commit_links.len() >= MAX_LIVE_AUTHORITY_LINKS {
4179                return Err(DecisionError::Conflict(format!(
4180                    "live Commit retention reached the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound; advance the artifact-retention horizon before admitting another Commit"
4181                ))
4182                .into());
4183            }
4184            if let Some(floor) = current.outcome_floor.as_ref() {
4185                if candidate.deployment_id != floor.deployment_id
4186                    || candidate.epoch < floor.authority_before_epoch
4187                {
4188                    return Err(DecisionError::Conflict(format!(
4189                        "cluster outcome epoch {} is below or outside authority floor {}",
4190                        candidate.epoch, floor.authority_before_epoch
4191                    ))
4192                    .into());
4193                }
4194            }
4195            if Box::pin(
4196                self.compact_cluster_outcome_history_before_append(proof, current, &snapshot),
4197            )
4198            .await?
4199            {
4200                tokio::task::yield_now().await;
4201                continue;
4202            }
4203
4204            let base_sequence = current.lease.seq;
4205            let sequence = base_sequence
4206                .checked_add(1)
4207                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4208            let mut next = current.preserve_with_lease(LeaderLease {
4209                seq: sequence,
4210                renewal_sequence: current.lease.renewal_sequence,
4211                token: current.lease.token,
4212                owner: current.lease.owner.clone(),
4213                expires_at_ms: current.lease.expires_at_ms,
4214                catalog_manifest: current.lease.catalog_manifest.clone(),
4215            });
4216            next.checkpoint_outcome = Some(candidate.clone());
4217            next.previous_outcome = current.outcome_head;
4218            let new_link = OutcomeLink {
4219                sequence,
4220                epoch: candidate.epoch,
4221                checkpoint_id: candidate.checkpoint_id,
4222            };
4223            next.outcome_head = Some(new_link);
4224            if candidate.is_commit() {
4225                next.previous_commit = current.commit_head;
4226                next.commit_head = Some(new_link);
4227            }
4228            next.validate()?;
4229            match self
4230                .create_authority_record(Some(&published), &next)
4231                .await?
4232            {
4233                AuthorityCreateOutcome::Created => {
4234                    let mut appended = outcomes.to_vec();
4235                    appended.push(candidate.clone());
4236                    let mut terminal_links = snapshot.terminal_links.to_vec();
4237                    terminal_links.push(new_link);
4238                    let mut commit_links = snapshot.commit_links.to_vec();
4239                    if candidate.is_commit() {
4240                        commit_links.push(new_link);
4241                    }
4242                    self.install_cluster_outcome_audit(
4243                        Self::cluster_outcome_audit_key(&next),
4244                        next.lease.seq,
4245                        ClusterOutcomeAuditSnapshot {
4246                            outcomes: Arc::from(appended),
4247                            terminal_links: Arc::from(terminal_links),
4248                            commit_links: Arc::from(commit_links),
4249                        },
4250                    );
4251                    return Ok(RecordOutcomeResult::Created(candidate));
4252                }
4253                AuthorityCreateOutcome::ExistingIdentical => {
4254                    return Ok(RecordOutcomeResult::Unchanged(candidate));
4255                }
4256                AuthorityCreateOutcome::Contended(winner_head) => {
4257                    let winners = self
4258                        .cached_audited_cluster_outcomes_from(&winner_head)
4259                        .await?;
4260                    if let Some(winner) = winners
4261                        .outcomes
4262                        .iter()
4263                        .find(|outcome| outcome.epoch == candidate.epoch)
4264                    {
4265                        return if winner == &candidate {
4266                            Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4267                        } else {
4268                            Ok(RecordOutcomeResult::Conflict {
4269                                winner: winner.clone(),
4270                            })
4271                        };
4272                    }
4273                    if !winner_head.lease.matches_proof(proof) {
4274                        return Err(ClusterCheckpointAuthorityError::Fenced);
4275                    }
4276                    if winner_head.lease.seq <= base_sequence {
4277                        return Err(LeaseError::Invalid(
4278                            "cluster outcome contention did not advance the authority sequence"
4279                                .into(),
4280                        )
4281                        .into());
4282                    }
4283                    tokio::task::yield_now().await;
4284                }
4285            }
4286        }
4287    }
4288
4289    async fn validate_assignment_recovery_snapshot(
4290        &self,
4291        decision: &AssignmentRecoveryDecision,
4292    ) -> Result<(), ClusterCheckpointAuthorityError> {
4293        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4294        let predecessor = snapshots
4295            .load()
4296            .await
4297            .map_err(|error| {
4298                assignment_snapshot_error("load assignment recovery predecessor", error)
4299            })?
4300            .ok_or_else(|| {
4301                LeaseError::Invalid(
4302                    "assignment recovery requires a durable predecessor snapshot".into(),
4303                )
4304            })?;
4305        let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4306            LeaseError::Invalid(format!(
4307                "assignment recovery predecessor has no valid fence: {error}"
4308            ))
4309        })?;
4310        if predecessor.draining || predecessor_fence != decision.predecessor {
4311            return Err(LeaseError::Invalid(
4312                "assignment recovery predecessor is not the exact committed assignment head".into(),
4313            )
4314            .into());
4315        }
4316        let proposal = snapshots
4317            .load_recovery_proposal(&decision.proposal)
4318            .await
4319            .map_err(|error| {
4320                assignment_snapshot_error("load assignment recovery proposal", error)
4321            })?;
4322        let proposal_fence = proposal.assignment_fence().map_err(|error| {
4323            LeaseError::Invalid(format!(
4324                "assignment recovery proposal has no valid fence: {error}"
4325            ))
4326        })?;
4327        if proposal_fence != decision.target {
4328            return Err(LeaseError::Invalid(
4329                "assignment recovery proposal does not match its authorized target fence".into(),
4330            )
4331            .into());
4332        }
4333        Ok(())
4334    }
4335
4336    async fn record_assignment_decision(
4337        &self,
4338        proof: &LeaderProof,
4339        decision: AuthorityAssignmentDecision,
4340    ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
4341        decision.validate()?;
4342        if decision.leader_proof() != proof || !proof.is_canonical() {
4343            return Err(ClusterCheckpointAuthorityError::Fenced);
4344        }
4345
4346        loop {
4347            let published = self
4348                .load_published_authority_head()
4349                .await?
4350                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4351            let current = &published.record;
4352            if !current.lease.matches_proof(proof) {
4353                return Err(ClusterCheckpointAuthorityError::Fenced);
4354            }
4355            if let Some(floor) = current.assignment_decision_floor.as_ref() {
4356                if decision.target_version() < floor.before_target_version {
4357                    return Err(DecisionError::Conflict(format!(
4358                        "assignment decision version {} is below durable retention floor {}",
4359                        decision.target_version(),
4360                        floor.before_target_version
4361                    ))
4362                    .into());
4363                }
4364            }
4365            let decisions = self.audited_assignment_decisions_from(current).await?;
4366            if let Some(winner) = decisions
4367                .iter()
4368                .find(|winner| winner.target_version() == decision.target_version())
4369            {
4370                return if winner == &decision {
4371                    Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
4372                        winner.clone(),
4373                    ))
4374                } else {
4375                    Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
4376                        winner: winner.clone(),
4377                    })
4378                };
4379            }
4380            if let Some(last) = decisions.last() {
4381                if decision.target_version() <= last.target_version() {
4382                    return Err(DecisionError::Conflict(format!(
4383                        "assignment decision version {} does not advance durable version {}",
4384                        decision.target_version(),
4385                        last.target_version()
4386                    ))
4387                    .into());
4388                }
4389            }
4390            if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
4391                self.validate_assignment_recovery_snapshot(recovery).await?;
4392            }
4393
4394            let base_sequence = current.lease.seq;
4395            let sequence = base_sequence
4396                .checked_add(1)
4397                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4398            let mut candidate = current.preserve_with_lease(LeaderLease {
4399                seq: sequence,
4400                renewal_sequence: current.lease.renewal_sequence,
4401                token: current.lease.token,
4402                owner: current.lease.owner.clone(),
4403                expires_at_ms: current.lease.expires_at_ms,
4404                catalog_manifest: current.lease.catalog_manifest.clone(),
4405            });
4406            candidate.assignment_decision = Some(decision.clone());
4407            candidate.previous_assignment_decision = current.assignment_decision_head;
4408            candidate.assignment_decision_head = Some(AssignmentDecisionLink {
4409                sequence,
4410                target_version: decision.target_version(),
4411            });
4412            candidate.validate()?;
4413
4414            match self
4415                .create_authority_record(Some(&published), &candidate)
4416                .await?
4417            {
4418                AuthorityCreateOutcome::Created => {
4419                    return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
4420                }
4421                AuthorityCreateOutcome::ExistingIdentical => {
4422                    return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
4423                }
4424                AuthorityCreateOutcome::Contended(winner_head) => {
4425                    let winners = self.audited_assignment_decisions_from(&winner_head).await?;
4426                    if let Some(winner) = winners
4427                        .iter()
4428                        .find(|winner| winner.target_version() == decision.target_version())
4429                    {
4430                        return if winner == &decision {
4431                            Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
4432                                winner.clone(),
4433                            ))
4434                        } else {
4435                            Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
4436                                winner: winner.clone(),
4437                            })
4438                        };
4439                    }
4440                    if !winner_head.lease.matches_proof(proof) {
4441                        return Err(ClusterCheckpointAuthorityError::Fenced);
4442                    }
4443                    if winner_head.lease.seq > base_sequence {
4444                        tokio::task::yield_now().await;
4445                        continue;
4446                    }
4447                    return Err(LeaseError::Invalid(
4448                        "assignment authority contention did not advance the sequence".into(),
4449                    )
4450                    .into());
4451                }
4452            }
4453        }
4454    }
4455
4456    /// Admit one assignment-drain settlement through the exact next authority sequence.
4457    ///
4458    /// Lease renewals, takeovers, checkpoint outcomes, and other decisions contend on that same
4459    /// create-only sequence. An identical retry converges on the durable winner.
4460    ///
4461    /// # Errors
4462    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
4463    pub async fn record_assignment_drain_decision(
4464        &self,
4465        proof: &LeaderProof,
4466        decision: AssignmentDrainDecision,
4467    ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
4468        match Box::pin(
4469            self.record_assignment_decision(proof, AuthorityAssignmentDecision::Drain(decision)),
4470        )
4471        .await?
4472        {
4473            RecordAuthorityAssignmentDecisionResult::Created(
4474                AuthorityAssignmentDecision::Drain(decision),
4475            ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
4476            RecordAuthorityAssignmentDecisionResult::Unchanged(
4477                AuthorityAssignmentDecision::Drain(decision),
4478            ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
4479            RecordAuthorityAssignmentDecisionResult::Conflict {
4480                winner: AuthorityAssignmentDecision::Drain(winner),
4481            } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
4482            RecordAuthorityAssignmentDecisionResult::Conflict {
4483                winner: AuthorityAssignmentDecision::Recovery(winner),
4484            } => Err(DecisionError::Conflict(format!(
4485                "assignment recovery decision already settled target version {}",
4486                winner.target_version()
4487            ))
4488            .into()),
4489            RecordAuthorityAssignmentDecisionResult::Created(_)
4490            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
4491                "assignment decision recorder returned the wrong decision kind".into(),
4492            )
4493            .into()),
4494        }
4495    }
4496
4497    /// Admit one failure-recovery assignment through the exact next authority sequence.
4498    ///
4499    /// # Errors
4500    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
4501    pub(crate) async fn record_assignment_recovery_decision(
4502        &self,
4503        proof: &LeaderProof,
4504        decision: AssignmentRecoveryDecision,
4505    ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
4506        decision.validate()?;
4507        if &decision.leader_proof != proof || !proof.is_canonical() {
4508            return Err(ClusterCheckpointAuthorityError::Fenced);
4509        }
4510        let current = self
4511            .load_record()
4512            .await?
4513            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4514        if !current.lease.matches_proof(proof) {
4515            return Err(ClusterCheckpointAuthorityError::Fenced);
4516        }
4517        if let Some(floor) = current.assignment_decision_floor.as_ref() {
4518            if decision.target_version() < floor.before_target_version {
4519                return Err(DecisionError::Conflict(format!(
4520                    "assignment decision version {} is below durable retention floor {}",
4521                    decision.target_version(),
4522                    floor.before_target_version
4523                ))
4524                .into());
4525            }
4526        }
4527        if let Some(winner) = self
4528            .audited_assignment_decisions_from(&current)
4529            .await?
4530            .into_iter()
4531            .find(|winner| winner.target_version() == decision.target_version())
4532        {
4533            return match winner {
4534                AuthorityAssignmentDecision::Recovery(winner) if winner == decision => {
4535                    Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(winner))
4536                }
4537                AuthorityAssignmentDecision::Recovery(winner) => {
4538                    Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner })
4539                }
4540                AuthorityAssignmentDecision::Drain(winner) => {
4541                    Err(DecisionError::Conflict(format!(
4542                        "assignment drain decision already settled target version {}",
4543                        winner.target_version()
4544                    ))
4545                    .into())
4546                }
4547            };
4548        }
4549        match Box::pin(
4550            self.record_assignment_decision(proof, AuthorityAssignmentDecision::Recovery(decision)),
4551        )
4552        .await?
4553        {
4554            RecordAuthorityAssignmentDecisionResult::Created(
4555                AuthorityAssignmentDecision::Recovery(decision),
4556            ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
4557            RecordAuthorityAssignmentDecisionResult::Unchanged(
4558                AuthorityAssignmentDecision::Recovery(decision),
4559            ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
4560            RecordAuthorityAssignmentDecisionResult::Conflict {
4561                winner: AuthorityAssignmentDecision::Recovery(winner),
4562            } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
4563            RecordAuthorityAssignmentDecisionResult::Conflict {
4564                winner: AuthorityAssignmentDecision::Drain(winner),
4565            } => Err(DecisionError::Conflict(format!(
4566                "assignment drain decision already settled target version {}",
4567                winner.target_version()
4568            ))
4569            .into()),
4570            RecordAuthorityAssignmentDecisionResult::Created(_)
4571            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
4572                "assignment decision recorder returned the wrong decision kind".into(),
4573            )
4574            .into()),
4575        }
4576    }
4577
4578    async fn assignment_decision(
4579        &self,
4580        target_version: u64,
4581    ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4582        if target_version == 0 {
4583            return Err(LeaseError::Invalid(
4584                "assignment decision target version must be nonzero".into(),
4585            )
4586            .into());
4587        }
4588        let Some(head) = self.load_record().await? else {
4589            return Ok(None);
4590        };
4591        if let Some(floor) = head.assignment_decision_floor.as_ref() {
4592            if target_version < floor.before_target_version {
4593                return Err(DecisionError::Conflict(format!(
4594                    "assignment decision version {target_version} is below durable retention floor {}",
4595                    floor.before_target_version
4596                ))
4597                .into());
4598            }
4599        }
4600        Ok(self
4601            .audited_assignment_decisions_from(&head)
4602            .await?
4603            .into_iter()
4604            .find(|decision| decision.target_version() == target_version))
4605    }
4606
4607    /// Read the immutable settlement for one exact target assignment version.
4608    ///
4609    /// # Errors
4610    /// Fails closed on malformed or incomplete authority history.
4611    pub async fn assignment_drain_decision(
4612        &self,
4613        target_version: u64,
4614    ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
4615        match self.assignment_decision(target_version).await? {
4616            Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
4617            Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
4618                format!("target version {target_version} was settled by assignment recovery"),
4619            )
4620            .into()),
4621            None => Ok(None),
4622        }
4623    }
4624
4625    /// Read the immutable recovery authorization for one exact target assignment version.
4626    ///
4627    /// # Errors
4628    /// Fails closed on malformed, incomplete, or cross-kind authority history.
4629    pub async fn assignment_recovery_decision(
4630        &self,
4631        target_version: u64,
4632    ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
4633        match self.assignment_decision(target_version).await? {
4634            Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
4635            Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
4636                "target version {target_version} was settled by assignment drain"
4637            ))
4638            .into()),
4639            None => Ok(None),
4640        }
4641    }
4642
4643    /// Materialize the immutable authority winner for one recovery target version.
4644    ///
4645    /// The caller supplies only the version; an arbitrary staged proposal can never bypass the
4646    /// shared leader-decision chain.
4647    ///
4648    /// # Errors
4649    /// Fails closed when no recovery decision exists, its proposal is invalid, or storage fails.
4650    pub async fn materialize_assignment_recovery(
4651        &self,
4652        target_version: u64,
4653    ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
4654        let decision = self
4655            .assignment_recovery_decision(target_version)
4656            .await?
4657            .ok_or_else(|| {
4658                DecisionError::Conflict(format!(
4659                    "assignment recovery target {target_version} has no authority decision"
4660                ))
4661            })?;
4662        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4663        let proposal = snapshots
4664            .load_recovery_proposal(&decision.proposal)
4665            .await
4666            .map_err(|error| {
4667                assignment_snapshot_error("load authorized assignment recovery proposal", error)
4668            })?;
4669        if proposal.assignment_fence().map_err(|error| {
4670            LeaseError::Invalid(format!(
4671                "authorized assignment recovery proposal has no valid fence: {error}"
4672            ))
4673        })? != decision.target
4674        {
4675            return Err(LeaseError::Invalid(
4676                "authorized assignment recovery proposal does not match its target".into(),
4677            )
4678            .into());
4679        }
4680        snapshots
4681            .materialize_recovery(&decision.proposal)
4682            .await
4683            .map_err(|error| {
4684                assignment_snapshot_error("materialize authorized assignment recovery", error)
4685                    .into()
4686            })
4687    }
4688
4689    /// Advance the shared assignment-decision floor through the exact next authority sequence.
4690    ///
4691    /// The compatibility-shaped API retains both drain and recovery decisions on one ordered
4692    /// chain. The caller must first durably prune assignment snapshots below the same
4693    /// target-version horizon. Decision-bearing authority records below the durable floor then
4694    /// become eligible for best-effort deletion while the exact terminal anchor preserves chain
4695    /// continuity.
4696    ///
4697    /// # Errors
4698    ///
4699    /// Fails for a stale proof, invalid horizon, corrupt history, or storage failure.
4700    pub async fn prune_assignment_drain_decisions_before(
4701        &self,
4702        proof: &LeaderProof,
4703        before_target_version: u64,
4704    ) -> Result<u64, ClusterCheckpointAuthorityError> {
4705        if before_target_version == 0 {
4706            return Ok(self
4707                .load_record()
4708                .await?
4709                .and_then(|head| head.assignment_decision_floor)
4710                .map_or(0, |floor| floor.before_target_version));
4711        }
4712        if !proof.is_canonical() {
4713            return Err(ClusterCheckpointAuthorityError::Fenced);
4714        }
4715        loop {
4716            let published = self
4717                .load_published_authority_head()
4718                .await?
4719                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4720            let current = &published.record;
4721            if !current.lease.matches_proof(proof) {
4722                return Err(ClusterCheckpointAuthorityError::Fenced);
4723            }
4724            if let Some(floor) = current.assignment_decision_floor.as_ref() {
4725                if floor.before_target_version >= before_target_version {
4726                    self.schedule_history_prune();
4727                    return Ok(floor.before_target_version);
4728                }
4729            }
4730
4731            let decisions = self.audited_assignment_decisions_from(current).await?;
4732            let terminal_anchor = decisions
4733                .iter()
4734                .rev()
4735                .find(|decision| decision.target_version() < before_target_version)
4736                .cloned()
4737                .or_else(|| {
4738                    current
4739                        .assignment_decision_floor
4740                        .as_ref()
4741                        .and_then(|floor| floor.terminal_anchor.clone())
4742                });
4743            let terminal_anchor_link = match terminal_anchor.as_ref() {
4744                Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
4745                None => None,
4746            };
4747            let floor = AuthorityAssignmentDecisionFloor {
4748                before_target_version,
4749                terminal_anchor,
4750                terminal_anchor_link,
4751            };
4752            floor.validate()?;
4753
4754            let base_sequence = current.lease.seq;
4755            let sequence = base_sequence
4756                .checked_add(1)
4757                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4758            let mut next = current.preserve_with_lease(LeaderLease {
4759                seq: sequence,
4760                renewal_sequence: current.lease.renewal_sequence,
4761                token: current.lease.token,
4762                owner: current.lease.owner.clone(),
4763                expires_at_ms: current.lease.expires_at_ms,
4764                catalog_manifest: current.lease.catalog_manifest.clone(),
4765            });
4766            next.assignment_decision_floor = Some(floor);
4767            next.validate()?;
4768
4769            match self
4770                .create_authority_record(Some(&published), &next)
4771                .await?
4772            {
4773                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4774                    return Ok(before_target_version);
4775                }
4776                AuthorityCreateOutcome::Contended(winner) => {
4777                    if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
4778                        if winner_floor.before_target_version >= before_target_version {
4779                            self.schedule_history_prune();
4780                            return Ok(winner_floor.before_target_version);
4781                        }
4782                    }
4783                    if !winner.lease.matches_proof(proof) {
4784                        return Err(ClusterCheckpointAuthorityError::Fenced);
4785                    }
4786                    if winner.lease.seq > base_sequence {
4787                        tokio::task::yield_now().await;
4788                        continue;
4789                    }
4790                    return Err(LeaseError::Invalid(
4791                        "assignment decision floor contention did not advance the sequence".into(),
4792                    )
4793                    .into());
4794                }
4795            }
4796        }
4797    }
4798
4799    async fn cluster_outcome_from_snapshot(
4800        &self,
4801        head: &LeaderAuthorityRecord,
4802        epoch: u64,
4803    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4804        let floor = head.outcome_floor.as_ref();
4805        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
4806        if epoch < artifact_before_epoch {
4807            return Ok(None);
4808        }
4809        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
4810        let cache_key = Self::cluster_outcome_audit_key(head);
4811        if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
4812            return Ok(snapshot
4813                .outcomes
4814                .binary_search_by_key(&epoch, |outcome| outcome.epoch)
4815                .ok()
4816                .map(|index| snapshot.outcomes[index].clone())
4817                .filter(|outcome| {
4818                    outcome.epoch >= authority_before_epoch
4819                        || outcome.is_commit()
4820                        || floor
4821                            .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
4822                }));
4823        }
4824        if let Some(anchor) = floor
4825            .and_then(|floor| floor.terminal_anchor.as_ref())
4826            .filter(|anchor| anchor.epoch == epoch)
4827        {
4828            let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
4829                .load_or_create_deployment_id()
4830                .await?;
4831            if anchor.deployment_id != expected_deployment {
4832                return Err(DecisionError::Conflict(format!(
4833                    "cluster terminal anchor epoch {} does not belong to current deployment {}",
4834                    anchor.epoch, expected_deployment
4835                ))
4836                .into());
4837            }
4838            return Ok(Some(anchor.clone()));
4839        }
4840
4841        let commit_only = epoch < authority_before_epoch;
4842        let before_epoch = if commit_only {
4843            artifact_before_epoch
4844        } else {
4845            authority_before_epoch
4846        };
4847        let mut current = if commit_only {
4848            head.commit_head
4849        } else {
4850            head.outcome_head
4851        };
4852        let expected_anchor = if commit_only {
4853            floor.and_then(|floor| floor.committed_anchor_link)
4854        } else {
4855            floor.and_then(|floor| floor.terminal_anchor_link)
4856        };
4857        let expected_anchor_outcome = if commit_only {
4858            floor.and_then(|floor| floor.committed_anchor.as_ref())
4859        } else {
4860            floor.and_then(|floor| floor.terminal_anchor.as_ref())
4861        };
4862        let mut traversed = 0;
4863        while let Some(link) = current {
4864            if link.epoch < before_epoch {
4865                match (expected_anchor, expected_anchor_outcome) {
4866                    (Some(anchor_link), Some(anchor))
4867                        if link == anchor_link
4868                            && link.epoch == anchor.epoch
4869                            && link.checkpoint_id == anchor.checkpoint_id =>
4870                    {
4871                        return Ok(None)
4872                    }
4873                    (None, None) => return Ok(None),
4874                    _ => {
4875                        return Err(DecisionError::Conflict(format!(
4876                            "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
4877                            if commit_only { "Commit" } else { "terminal outcome" }
4878                        ))
4879                        .into());
4880                    }
4881                }
4882            }
4883            if link.epoch < epoch {
4884                return Ok(None);
4885            }
4886            if !consume_live_authority_link(&mut traversed) {
4887                return Err(DecisionError::Conflict(format!(
4888                    "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
4889                    if commit_only { "Commit" } else { "outcome" }
4890                ))
4891                .into());
4892            }
4893
4894            let admission;
4895            let record = if link.sequence == head.lease.seq {
4896                head
4897            } else {
4898                admission = read_authority_record(self.store.as_ref(), link.sequence)
4899                    .await?
4900                    .ok_or_else(|| {
4901                        DecisionError::InventoryChanged(format!(
4902                            "cluster {} authority record {} disappeared during exact lookup",
4903                            if commit_only { "Commit" } else { "outcome" },
4904                            link.sequence
4905                        ))
4906                    })?;
4907                &admission
4908            };
4909            let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
4910                DecisionError::Conflict(format!(
4911                    "cluster {} head epoch {} points to non-outcome authority record {}",
4912                    if commit_only { "Commit" } else { "outcome" },
4913                    link.epoch,
4914                    link.sequence
4915                ))
4916            })?;
4917            let record_head = if commit_only {
4918                record.commit_head
4919            } else {
4920                record.outcome_head
4921            };
4922            if record_head != Some(link)
4923                || (commit_only && !outcome.is_commit())
4924                || outcome.epoch != link.epoch
4925                || outcome.checkpoint_id != link.checkpoint_id
4926            {
4927                return Err(DecisionError::Conflict(format!(
4928                    "cluster {} link epoch {} sequence {} does not match its authority record",
4929                    if commit_only { "Commit" } else { "outcome" },
4930                    link.epoch,
4931                    link.sequence
4932                ))
4933                .into());
4934            }
4935            if outcome.epoch == epoch {
4936                let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
4937                    .load_or_create_deployment_id()
4938                    .await?;
4939                if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
4940                    || outcome.deployment_id != expected_deployment
4941                {
4942                    return Err(DecisionError::Conflict(format!(
4943                        "cluster outcome epoch {} does not belong to current deployment {}",
4944                        outcome.epoch, expected_deployment
4945                    ))
4946                    .into());
4947                }
4948                return Ok(Some(outcome.clone()));
4949            }
4950            current = if commit_only {
4951                record.previous_commit
4952            } else {
4953                record.previous_outcome
4954            };
4955        }
4956        if expected_anchor.is_some() {
4957            return Err(DecisionError::Conflict(format!(
4958                "cluster {} chain stopped without its durable floor anchor",
4959                if commit_only { "Commit" } else { "outcome" }
4960            ))
4961            .into());
4962        }
4963        Ok(None)
4964    }
4965
4966    async fn stable_cluster_outcome(
4967        &self,
4968        epoch: u64,
4969    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4970        const LOOKUP_RETRIES: usize = 3;
4971        for attempt in 0..LOOKUP_RETRIES {
4972            let Some(head) = self.load_record().await? else {
4973                return Ok(None);
4974            };
4975            match self.cluster_outcome_from_snapshot(&head, epoch).await {
4976                Err(ClusterCheckpointAuthorityError::Decision(
4977                    DecisionError::InventoryChanged(_),
4978                )) if attempt + 1 < LOOKUP_RETRIES => {
4979                    tokio::task::yield_now().await;
4980                }
4981                result => return result,
4982            }
4983        }
4984        Err(DecisionError::InventoryChanged(
4985            "cluster outcome exact lookup exhausted stability retries".into(),
4986        )
4987        .into())
4988    }
4989
4990    /// Read one live cluster outcome from the shared authority.
4991    ///
4992    /// # Errors
4993    ///
4994    /// Fails when the durable authority history is unavailable or invalid.
4995    pub async fn cluster_outcome(
4996        &self,
4997        epoch: u64,
4998    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
4999        self.stable_cluster_outcome(epoch).await
5000    }
5001
5002    /// Read one live cluster outcome together with its content-addressed recovery capsule.
5003    /// Commit always returns a validated capsule; Abort returns `None` for the capsule.
5004    ///
5005    /// # Errors
5006    ///
5007    /// Fails when the authority history or selected recovery capsule is unavailable or invalid.
5008    pub async fn cluster_outcome_with_recovery_capsule(
5009        &self,
5010        epoch: u64,
5011    ) -> Result<
5012        Option<(CheckpointOutcome, Option<ClusterRecoveryCapsule>)>,
5013        ClusterCheckpointAuthorityError,
5014    > {
5015        let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5016        let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5017            return Ok(None);
5018        };
5019        let capsule = if outcome.is_commit() {
5020            let reference = outcome.recovery_capsule.as_ref().ok_or_else(|| {
5021                DecisionError::Conflict(format!(
5022                    "cluster Commit epoch {} checkpoint {} has no recovery capsule",
5023                    outcome.epoch, outcome.checkpoint_id
5024                ))
5025            })?;
5026            let capsule = decisions.load_recovery_capsule(reference).await?;
5027            if capsule.attempt.epoch != outcome.epoch
5028                || capsule.attempt.checkpoint_id != outcome.checkpoint_id
5029                || Some(&capsule.assignment_fence) != outcome.assignment_fence.as_ref()
5030                || capsule.deployment_id != outcome.deployment_id
5031            {
5032                return Err(DecisionError::Conflict(format!(
5033                    "cluster Commit epoch {} checkpoint {} does not match recovery capsule '{}'",
5034                    outcome.epoch, outcome.checkpoint_id, reference.sha256
5035                ))
5036                .into());
5037            }
5038            Some(capsule)
5039        } else {
5040            None
5041        };
5042        Ok(Some((outcome, capsule)))
5043    }
5044
5045    /// Audit and return every live cluster outcome in ascending epoch order.
5046    ///
5047    /// # Errors
5048    ///
5049    /// Fails when the durable authority history is unavailable or invalid.
5050    pub async fn cluster_outcome_inventory(
5051        &self,
5052    ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5053        let (head, outcomes) = self.audited_cluster_outcomes().await?;
5054        Ok(Self::cluster_outcome_inventory_from_audit(
5055            head.as_ref(),
5056            &outcomes,
5057        ))
5058    }
5059
5060    fn cluster_outcome_inventory_from_audit(
5061        head: Option<&LeaderAuthorityRecord>,
5062        outcomes: &[CheckpointOutcome],
5063    ) -> ClusterOutcomeInventory {
5064        let floor = head.and_then(|head| head.outcome_floor.as_ref());
5065        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5066        ClusterOutcomeInventory {
5067            outcomes: outcomes
5068                .iter()
5069                .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5070                .cloned()
5071                .collect(),
5072            retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5073        }
5074    }
5075
5076    /// Audit and return every live cluster outcome in ascending epoch order.
5077    ///
5078    /// # Errors
5079    ///
5080    /// Fails when the durable authority history is unavailable or invalid.
5081    pub async fn cluster_outcomes(
5082        &self,
5083    ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5084        Ok(self.cluster_outcome_inventory().await?.outcomes)
5085    }
5086
5087    /// Greatest live cluster commit recovery cut.
5088    ///
5089    /// # Errors
5090    ///
5091    /// Fails when the durable authority history is unavailable or invalid.
5092    pub async fn highest_cluster_committed_outcome(
5093        &self,
5094    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5095        Ok(self
5096            .cluster_outcomes()
5097            .await?
5098            .into_iter()
5099            .rev()
5100            .find(CheckpointOutcome::is_commit))
5101    }
5102
5103    /// Greatest terminal cluster outcome, including the compacted continuity anchor.
5104    ///
5105    /// # Errors
5106    ///
5107    /// Fails when the durable authority history is unavailable or invalid.
5108    pub async fn highest_cluster_terminal_outcome(
5109        &self,
5110    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5111        Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5112    }
5113
5114    /// Return the exact immutable outcome for `attempt`, or the first audited terminal outcome
5115    /// known to close that older checkpoint. Compacted continuity anchors are included in the
5116    /// audit.
5117    ///
5118    /// # Errors
5119    /// Returns an error for a noncanonical attempt identity or an unavailable or invalid durable
5120    /// authority chain.
5121    pub async fn cluster_attempt_settlement(
5122        &self,
5123        attempt: crate::state::CheckpointAttempt,
5124    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5125        if !attempt.is_canonical() {
5126            return Err(DecisionError::Conflict(
5127                "cluster checkpoint settlement requires one nonzero canonical checkpoint ID".into(),
5128            )
5129            .into());
5130        }
5131        let outcomes = self.audited_cluster_outcomes().await?.1;
5132        if let Ok(index) = outcomes.binary_search_by_key(&attempt.epoch, |outcome| outcome.epoch) {
5133            return Ok(Some(outcomes[index].clone()));
5134        }
5135        let Some(highest) = outcomes.last() else {
5136            return Ok(None);
5137        };
5138        if highest.checkpoint_id > attempt.checkpoint_id {
5139            Ok(Some(highest.clone()))
5140        } else {
5141            Ok(None)
5142        }
5143    }
5144
5145    async fn validate_cluster_recovery_cut(
5146        &self,
5147        floor: &AuthorityOutcomeFloor,
5148        audited_outcomes: &[CheckpointOutcome],
5149    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5150        if floor.artifact_before_epoch == 0 {
5151            return Ok(None);
5152        }
5153        let recovery_cut = audited_outcomes
5154            .iter()
5155            .rev()
5156            .find(|outcome| outcome.epoch >= floor.artifact_before_epoch && outcome.is_commit())
5157            .ok_or_else(|| {
5158                DecisionError::Conflict(format!(
5159                    "cluster outcome floor {} has no live commit recovery cut",
5160                    floor.artifact_before_epoch
5161                ))
5162            })?;
5163        CheckpointDecisionStore::new(Arc::clone(&self.store))
5164            .validate_recovery_capsule_for_outcome(recovery_cut)
5165            .await?;
5166        Ok(Some(recovery_cut.clone()))
5167    }
5168
5169    async fn preflight_cluster_recovery_cut<V, Fut>(
5170        &self,
5171        floor: &AuthorityOutcomeFloor,
5172        audited_outcomes: &[CheckpointOutcome],
5173        validate_artifacts: &V,
5174    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError>
5175    where
5176        V: Fn(CheckpointOutcome) -> Fut,
5177        Fut: std::future::Future<Output = Result<(), String>>,
5178    {
5179        let recovery_cut = self
5180            .validate_cluster_recovery_cut(floor, audited_outcomes)
5181            .await?;
5182        if let Some(recovery_cut) = recovery_cut.as_ref() {
5183            validate_artifacts(recovery_cut.clone())
5184                .await
5185                .map_err(|error| {
5186                    DecisionError::Conflict(format!(
5187                        "cluster recovery cut epoch {} checkpoint {} failed durable recovery metadata preflight: {error}",
5188                        recovery_cut.epoch, recovery_cut.checkpoint_id
5189                    ))
5190                })?;
5191        }
5192        Ok(recovery_cut)
5193    }
5194
5195    /// Exact continuity boundary for cluster outcomes compacted from the authority history.
5196    ///
5197    /// # Errors
5198    ///
5199    /// Fails when the durable authority history is unavailable or invalid.
5200    pub async fn cluster_outcome_retention_boundary(
5201        &self,
5202    ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
5203        let head = self.load_record().await?;
5204        Ok(ClusterOutcomeRetentionBoundary::from_floor(
5205            head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
5206        ))
5207    }
5208
5209    /// Read an existing cluster retention boundary after auditing its outcome chain and selected
5210    /// recovery capsule, without invoking a caller-supplied state-artifact preflight.
5211    ///
5212    /// # Errors
5213    ///
5214    /// Returns an error when the authority history or selected recovery capsule is invalid.
5215    pub async fn audited_cluster_outcome_retention_boundary(
5216        &self,
5217    ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
5218        Ok(self
5219            .validated_cluster_outcome_inventory(|_| async { Ok(()) })
5220            .await?
5221            .retention_boundary)
5222    }
5223
5224    /// Read live outcomes and their retention boundary only after the selected live Commit passes
5225    /// the caller's durable recovery metadata preflight and both outcome heads and the floor remain
5226    /// unchanged.
5227    ///
5228    /// # Errors
5229    ///
5230    /// Fails when authority history is invalid or artifact validation fails.
5231    pub async fn validated_cluster_outcome_inventory<V, Fut>(
5232        &self,
5233        validate_artifacts: V,
5234    ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError>
5235    where
5236        V: Fn(CheckpointOutcome) -> Fut,
5237        Fut: std::future::Future<Output = Result<(), String>>,
5238    {
5239        loop {
5240            let current = self
5241                .load_record()
5242                .await?
5243                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5244            let snapshot = self.cached_audited_cluster_outcomes_from(&current).await?;
5245            if let Some(floor) = current.outcome_floor.as_ref() {
5246                self.preflight_cluster_recovery_cut(floor, &snapshot.outcomes, &validate_artifacts)
5247                    .await?;
5248            }
5249            let rechecked = self
5250                .load_record()
5251                .await?
5252                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5253            if rechecked.outcome_head == current.outcome_head
5254                && rechecked.commit_head == current.commit_head
5255                && rechecked.outcome_floor == current.outcome_floor
5256            {
5257                return Ok(Self::cluster_outcome_inventory_from_audit(
5258                    Some(&current),
5259                    &snapshot.outcomes,
5260                ));
5261            }
5262            tokio::task::yield_now().await;
5263        }
5264    }
5265
5266    /// Run one bounded recovery-capsule cleanup step below the durable artifact horizon.
5267    ///
5268    /// This is deliberately independent of floor publication: cleanup failure cannot revoke an
5269    /// already-authorized manifest/state retention horizon.
5270    ///
5271    /// # Errors
5272    ///
5273    /// Fails when the authority history is invalid or cleanup storage operations fail.
5274    pub async fn maintain_cluster_recovery_capsules(
5275        &self,
5276    ) -> Result<crate::checkpoint_decision::RecoveryCapsuleGcStep, ClusterCheckpointAuthorityError>
5277    {
5278        let current = self
5279            .load_record()
5280            .await?
5281            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5282        let audited = self.cached_audited_cluster_outcomes_from(&current).await?;
5283        let Some(floor) = current.outcome_floor.as_ref() else {
5284            return Ok(crate::checkpoint_decision::RecoveryCapsuleGcStep {
5285                examined: 0,
5286                deleted: 0,
5287                quarantined: 0,
5288                pending: false,
5289            });
5290        };
5291        if floor.artifact_before_epoch == 0 {
5292            return Ok(crate::checkpoint_decision::RecoveryCapsuleGcStep {
5293                examined: 0,
5294                deleted: 0,
5295                quarantined: 0,
5296                pending: false,
5297            });
5298        }
5299        let mut known_live_digests = BTreeSet::new();
5300        known_live_digests.extend(
5301            audited
5302                .outcomes
5303                .iter()
5304                .filter(|outcome| outcome.epoch >= floor.artifact_before_epoch)
5305                .filter_map(|outcome| outcome.recovery_capsule.as_ref())
5306                .map(|reference| reference.sha256.clone()),
5307        );
5308        CheckpointDecisionStore::new(Arc::clone(&self.store))
5309            .sweep_recovery_capsules_step(floor.artifact_before_epoch, &known_live_digests)
5310            .await
5311            .map_err(Into::into)
5312    }
5313
5314    /// Advance the cluster outcome floor through the exact next authority sequence.
5315    ///
5316    /// At least one live commit remains at or above the requested horizon. Outcome-bearing
5317    /// records below the floor and unreferenced old recovery capsules become eligible for
5318    /// best-effort deletion only after the floor is durable.
5319    ///
5320    /// # Errors
5321    ///
5322    /// Fails for a stale proof, invalid horizon, failed artifact validation, or storage failure.
5323    pub async fn prune_cluster_outcomes_before<V, Fut>(
5324        &self,
5325        proof: &LeaderProof,
5326        before_epoch: u64,
5327        validate_artifacts: V,
5328    ) -> Result<u64, ClusterCheckpointAuthorityError>
5329    where
5330        V: Fn(CheckpointOutcome) -> Fut,
5331        Fut: std::future::Future<Output = Result<(), String>>,
5332    {
5333        if before_epoch == 0 {
5334            return Ok(self
5335                .cluster_outcome_retention_boundary()
5336                .await?
5337                .artifact_before_epoch);
5338        }
5339        if !proof.is_canonical() {
5340            return Err(ClusterCheckpointAuthorityError::Fenced);
5341        }
5342        loop {
5343            let current = self
5344                .load_record()
5345                .await?
5346                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5347            if !current.lease.matches_proof(proof) {
5348                return Err(ClusterCheckpointAuthorityError::Fenced);
5349            }
5350            if let Some(floor) = current
5351                .outcome_floor
5352                .as_ref()
5353                .filter(|floor| floor.artifact_before_epoch >= before_epoch)
5354            {
5355                self.schedule_history_prune();
5356                return Ok(floor.artifact_before_epoch);
5357            }
5358            let snapshot = self.cached_audited_cluster_outcomes_from(&current).await?;
5359            if !snapshot
5360                .outcomes
5361                .iter()
5362                .any(|outcome| outcome.epoch >= before_epoch && outcome.is_commit())
5363            {
5364                return Err(DecisionError::Conflict(format!(
5365                    "cannot advance cluster outcome floor to {before_epoch}: no live commit recovery cut would remain"
5366                ))
5367                .into());
5368            }
5369            let authority_before_epoch = current
5370                .outcome_floor
5371                .as_ref()
5372                .map_or(before_epoch, |floor| {
5373                    floor.authority_before_epoch.max(before_epoch)
5374                });
5375            let floor = self
5376                .build_cluster_outcome_floor(
5377                    &current,
5378                    &snapshot,
5379                    before_epoch,
5380                    authority_before_epoch,
5381                )
5382                .await?;
5383            self.preflight_cluster_recovery_cut(&floor, &snapshot.outcomes, &validate_artifacts)
5384                .await?;
5385
5386            // Lease renewals and catalog seals may advance the shared sequence while the complete
5387            // recovery metadata preflight performs remote reads. They are harmless only when both the
5388            // outcome heads and retention floor remain exactly the same.
5389            let published = self
5390                .load_published_authority_head()
5391                .await?
5392                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5393            let rechecked = &published.record;
5394            if !rechecked.lease.matches_proof(proof) {
5395                return Err(ClusterCheckpointAuthorityError::Fenced);
5396            }
5397            if rechecked.outcome_head != current.outcome_head
5398                || rechecked.commit_head != current.commit_head
5399                || rechecked.outcome_floor != current.outcome_floor
5400            {
5401                tokio::task::yield_now().await;
5402                continue;
5403            }
5404            let base_sequence = rechecked.lease.seq;
5405            let sequence = base_sequence
5406                .checked_add(1)
5407                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5408            let mut next = rechecked.preserve_with_lease(LeaderLease {
5409                seq: sequence,
5410                renewal_sequence: rechecked.lease.renewal_sequence,
5411                token: rechecked.lease.token,
5412                owner: rechecked.lease.owner.clone(),
5413                expires_at_ms: rechecked.lease.expires_at_ms,
5414                catalog_manifest: rechecked.lease.catalog_manifest.clone(),
5415            });
5416            next.outcome_floor = Some(floor.clone());
5417            next.validate()?;
5418            match self
5419                .create_authority_record(Some(&published), &next)
5420                .await?
5421            {
5422                AuthorityCreateOutcome::Created => {
5423                    self.install_cluster_outcome_audit(
5424                        Self::cluster_outcome_audit_key(&next),
5425                        next.lease.seq,
5426                        Self::outcomes_retained_by_floor(&floor, &snapshot),
5427                    );
5428                    return Ok(before_epoch);
5429                }
5430                AuthorityCreateOutcome::ExistingIdentical => {
5431                    let winner_floor = next.outcome_floor.as_ref().ok_or_else(|| {
5432                        LeaseError::Invalid(
5433                            "durable floor winner lost its retention boundary".into(),
5434                        )
5435                    })?;
5436                    let winner_snapshot = self.cached_audited_cluster_outcomes_from(&next).await?;
5437                    self.preflight_cluster_recovery_cut(
5438                        winner_floor,
5439                        &winner_snapshot.outcomes,
5440                        &validate_artifacts,
5441                    )
5442                    .await?;
5443                    let confirmed = self
5444                        .load_record()
5445                        .await?
5446                        .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5447                    if !confirmed.lease.matches_proof(proof) {
5448                        return Err(ClusterCheckpointAuthorityError::Fenced);
5449                    }
5450                    if confirmed.outcome_head == next.outcome_head
5451                        && confirmed.commit_head == next.commit_head
5452                        && confirmed.outcome_floor == next.outcome_floor
5453                    {
5454                        return Ok(before_epoch);
5455                    }
5456                    tokio::task::yield_now().await;
5457                }
5458                AuthorityCreateOutcome::Contended(winner) => {
5459                    if !winner.lease.matches_proof(proof) {
5460                        return Err(ClusterCheckpointAuthorityError::Fenced);
5461                    }
5462                    if winner.lease.seq <= base_sequence {
5463                        return Err(LeaseError::Invalid(
5464                            "outcome floor contention did not advance the authority sequence"
5465                                .into(),
5466                        )
5467                        .into());
5468                    }
5469                    tokio::task::yield_now().await;
5470                }
5471            }
5472        }
5473    }
5474
5475    /// Acquire an empty authority as a new term, or rotate the fencing token when this exact
5476    /// process incarnation already owns the durable head. Rival wall clocks never authorize
5477    /// takeover.
5478    ///
5479    /// # Errors
5480    /// Fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
5481    pub async fn begin_new_term(
5482        &self,
5483        owner: &LeaderLeaseOwner,
5484        now_ms: i64,
5485    ) -> Result<LeaseOutcome, LeaseError> {
5486        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
5487            owner,
5488            now_ms,
5489            SameOwnerToken::Rotate,
5490        ))
5491        .await
5492    }
5493
5494    /// Renew only the exact durable authority term identified by `token`.
5495    ///
5496    /// # Errors
5497    /// Returns [`LeaseError::Fenced`] if the durable head is absent or belongs to another owner or
5498    /// term. Also fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
5499    pub async fn renew_exact(
5500        &self,
5501        owner: &LeaderLeaseOwner,
5502        token: u64,
5503        now_ms: i64,
5504    ) -> Result<LeaseOutcome, LeaseError> {
5505        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
5506            owner,
5507            now_ms,
5508            SameOwnerToken::Exact(token),
5509        ))
5510        .await
5511    }
5512
5513    #[cfg(test)]
5514    pub(crate) async fn acquire_or_renew_current_term_for_test(
5515        &self,
5516        owner: &LeaderLeaseOwner,
5517        now_ms: i64,
5518    ) -> Result<LeaseOutcome, LeaseError> {
5519        self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
5520            .await
5521    }
5522
5523    async fn acquire_or_renew_current_term_for_test_inner(
5524        &self,
5525        owner: &LeaderLeaseOwner,
5526        now_ms: i64,
5527        same_owner_token: SameOwnerToken,
5528    ) -> Result<LeaseOutcome, LeaseError> {
5529        owner.validate()?;
5530        self.ttl()?;
5531        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
5532        loop {
5533            let published = self.load_published_authority_head().await?;
5534            let candidate = match published.as_ref() {
5535                None => {
5536                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5537                        return Err(LeaseError::Fenced(
5538                            "exact leader renewal lost the durable authority head".into(),
5539                        ));
5540                    }
5541                    LeaderAuthorityRecord::initial(LeaderLease {
5542                        seq: 1,
5543                        renewal_sequence: 1,
5544                        token: 1,
5545                        owner: owner.clone(),
5546                        expires_at_ms,
5547                        catalog_manifest: None,
5548                    })
5549                }
5550                Some(head) if head.record.lease.owner == *owner => {
5551                    let record = &head.record;
5552                    let token = match same_owner_token {
5553                        #[cfg(test)]
5554                        SameOwnerToken::Preserve => record.lease.token,
5555                        SameOwnerToken::Rotate => {
5556                            record.lease.token.checked_add(1).ok_or_else(|| {
5557                                LeaseError::Invalid("leader fencing token exhausted".into())
5558                            })?
5559                        }
5560                        SameOwnerToken::Exact(expected) if expected == record.lease.token => {
5561                            expected
5562                        }
5563                        SameOwnerToken::Exact(_) => {
5564                            return Err(LeaseError::Fenced(
5565                                "exact leader renewal was superseded by a newer local term".into(),
5566                            ));
5567                        }
5568                    };
5569                    let lease = LeaderLease {
5570                        seq: record.lease.seq.checked_add(1).ok_or_else(|| {
5571                            LeaseError::Invalid("lease sequence exhausted".into())
5572                        })?,
5573                        renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
5574                            || LeaseError::Invalid("lease renewal sequence exhausted".into()),
5575                        )?,
5576                        token,
5577                        owner: owner.clone(),
5578                        expires_at_ms,
5579                        catalog_manifest: record.lease.catalog_manifest.clone(),
5580                    };
5581                    record.preserve_with_lease(lease)
5582                }
5583                Some(head) => {
5584                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5585                        return Err(LeaseError::Fenced(
5586                            "exact leader renewal was superseded by a rival owner".into(),
5587                        ));
5588                    }
5589                    return Ok(LeaseOutcome::Held(head.record.lease.clone()));
5590                }
5591            };
5592            match self
5593                .create_authority_record(published.as_ref(), &candidate)
5594                .await?
5595            {
5596                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5597                    return Ok(LeaseOutcome::Acquired(candidate.lease));
5598                }
5599                AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
5600                    if matches!(
5601                        same_owner_token,
5602                        SameOwnerToken::Exact(expected) if winner.lease.token != expected
5603                    ) {
5604                        return Err(LeaseError::Fenced(
5605                            "exact leader renewal lost a same-owner CAS race".into(),
5606                        ));
5607                    }
5608                    tokio::task::yield_now().await;
5609                }
5610                AuthorityCreateOutcome::Contended(winner) => {
5611                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
5612                        return Err(LeaseError::Fenced(
5613                            "exact leader renewal lost a rival-owner CAS race".into(),
5614                        ));
5615                    }
5616                    return Ok(LeaseOutcome::Held(winner.lease));
5617                }
5618            }
5619        }
5620    }
5621
5622    /// Start a candidate-local observation of a rival durable liveness identity.
5623    ///
5624    /// # Errors
5625    /// Rejects malformed state or an observation of the candidate itself.
5626    pub fn observe_rival(
5627        &self,
5628        owner: &LeaderLeaseOwner,
5629        lease: &LeaderLease,
5630    ) -> Result<LeaderLeaseObservation, LeaseError> {
5631        owner.validate()?;
5632        lease.validate()?;
5633        self.ttl()?;
5634        if lease.owner == *owner {
5635            return Err(LeaseError::Invalid(
5636                "leader takeover observation must belong to a rival".into(),
5637            ));
5638        }
5639        Ok(LeaderLeaseObservation {
5640            lease: lease.clone(),
5641            started: Instant::now(),
5642        })
5643    }
5644
5645    /// Take over only after the rival's owner, fencing token, and renewal sequence remained
5646    /// current for a full TTL on the candidate's monotonic clock.
5647    ///
5648    /// # Errors
5649    /// Fails closed on early observation, invalid state, I/O, or arithmetic exhaustion.
5650    pub async fn try_takeover(
5651        &self,
5652        owner: &LeaderLeaseOwner,
5653        observation: &LeaderLeaseObservation,
5654        now_ms: i64,
5655    ) -> Result<LeaseOutcome, LeaseError> {
5656        owner.validate()?;
5657        observation.lease.validate()?;
5658        let ttl = self.ttl()?;
5659        if observation.lease.owner == *owner {
5660            return Err(LeaseError::Invalid(
5661                "leader takeover observation must belong to a rival".into(),
5662            ));
5663        }
5664        if observation.started.elapsed() < ttl {
5665            return Ok(LeaseOutcome::Held(observation.lease.clone()));
5666        }
5667        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
5668        loop {
5669            let published = self
5670                .load_published_authority_head()
5671                .await?
5672                .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
5673            let current = &published.record;
5674            if !current.lease.has_same_liveness_identity(&observation.lease) {
5675                return Ok(LeaseOutcome::Held(current.lease.clone()));
5676            }
5677            let candidate_lease = LeaderLease {
5678                seq: current
5679                    .lease
5680                    .seq
5681                    .checked_add(1)
5682                    .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
5683                renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
5684                    || LeaseError::Invalid("lease renewal sequence exhausted".into()),
5685                )?,
5686                token: current
5687                    .lease
5688                    .token
5689                    .checked_add(1)
5690                    .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
5691                owner: owner.clone(),
5692                expires_at_ms,
5693                catalog_manifest: current.lease.catalog_manifest.clone(),
5694            };
5695            let candidate = current.preserve_with_lease(candidate_lease);
5696            match self
5697                .create_authority_record(Some(&published), &candidate)
5698                .await?
5699            {
5700                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5701                    return Ok(LeaseOutcome::Acquired(candidate.lease));
5702                }
5703                AuthorityCreateOutcome::Contended(winner)
5704                    if winner.lease.has_same_liveness_identity(&observation.lease) =>
5705                {
5706                    tokio::task::yield_now().await;
5707                }
5708                AuthorityCreateOutcome::Contended(winner) => {
5709                    return Ok(LeaseOutcome::Held(winner.lease));
5710                }
5711            }
5712        }
5713    }
5714}
5715
5716/// Renewal timings for the leader lease.
5717#[derive(Debug, Clone, Copy)]
5718pub struct LeaderLeaseConfig {
5719    /// Lease lifetime.
5720    pub ttl: Duration,
5721    /// Renewal cadence, strictly below the lifetime.
5722    pub renew_interval: Duration,
5723}
5724
5725impl Default for LeaderLeaseConfig {
5726    fn default() -> Self {
5727        Self {
5728            ttl: Duration::from_secs(5),
5729            renew_interval: Duration::from_secs(2),
5730        }
5731    }
5732}
5733
5734/// Whether the exact owner has a durable record and a live process-local deadline.
5735#[must_use]
5736pub fn lease_grants_leadership(
5737    lease: &Option<LeaderLease>,
5738    owner: &LeaderLeaseOwner,
5739    deadline: &LeaseDeadline,
5740) -> bool {
5741    deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
5742}
5743
5744/// Whether a captured proof still matches the current exact grant and local deadline.
5745#[must_use]
5746pub fn lease_grants_proof(
5747    lease: &Option<LeaderLease>,
5748    owner: &LeaderLeaseOwner,
5749    deadline: &LeaseDeadline,
5750    proof: &LeaderProof,
5751) -> bool {
5752    deadline.is_live()
5753        && proof.owner == owner.proof_owner()
5754        && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
5755}
5756
5757/// Acquires and renews leadership while candidacy remains true.
5758pub struct LeaderLeaseManager {
5759    store: Arc<LeaderLeaseStore>,
5760    owner: LeaderLeaseOwner,
5761    config: LeaderLeaseConfig,
5762    lease_tx: watch::Sender<Option<LeaderLease>>,
5763    deadline: Arc<LeaseDeadline>,
5764}
5765
5766#[cfg(feature = "cluster")]
5767enum LeaseOperationEvent {
5768    Shutdown,
5769    Candidacy(Result<(), watch::error::RecvError>),
5770    Deadline,
5771    Completed {
5772        result: Result<LeaseOutcome, LeaseError>,
5773        valid_until: tokio::time::Instant,
5774    },
5775}
5776
5777#[cfg(feature = "cluster")]
5778async fn wait_for_deadline(deadline: Option<tokio::time::Instant>) {
5779    if let Some(deadline) = deadline {
5780        tokio::time::sleep_until(deadline).await;
5781    } else {
5782        std::future::pending::<()>().await;
5783    }
5784}
5785
5786impl std::fmt::Debug for LeaderLeaseManager {
5787    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
5788        formatter
5789            .debug_struct("LeaderLeaseManager")
5790            .field("owner", &self.owner)
5791            .field("config", &self.config)
5792            .finish_non_exhaustive()
5793    }
5794}
5795
5796impl LeaderLeaseManager {
5797    /// Construct a manager bound to this boot's acquired stable-node process lease.
5798    ///
5799    /// # Errors
5800    /// Rejects invalid ownership or inconsistent renewal timings.
5801    pub fn new(
5802        store: Arc<LeaderLeaseStore>,
5803        process_lease: &ProcessLease,
5804        config: LeaderLeaseConfig,
5805    ) -> Result<Self, LeaseError> {
5806        let owner = LeaderLeaseOwner::from_process_lease(process_lease)?;
5807        let ttl_ms = i64::try_from(config.ttl.as_millis())
5808            .map_err(|_| LeaseError::Invalid("lease TTL exceeds diagnostic range".into()))?;
5809        let exact_ttl = u64::try_from(ttl_ms)
5810            .ok()
5811            .map(Duration::from_millis)
5812            .is_some_and(|ttl| ttl == config.ttl);
5813        if ttl_ms <= 0
5814            || !exact_ttl
5815            || config.renew_interval.is_zero()
5816            || config.renew_interval >= config.ttl
5817            || ttl_ms != store.ttl_ms
5818        {
5819            return Err(LeaseError::Invalid(
5820                "manager requires a renewal interval below the store's matching TTL".into(),
5821            ));
5822        }
5823        let (lease_tx, _lease_rx) = watch::channel(None);
5824        Ok(Self {
5825            store,
5826            owner,
5827            config,
5828            lease_tx,
5829            deadline: Arc::new(LeaseDeadline::uninitialized()),
5830        })
5831    }
5832
5833    /// Exact process incarnation this manager may publish.
5834    #[must_use]
5835    pub fn owner(&self) -> &LeaderLeaseOwner {
5836        &self.owner
5837    }
5838
5839    /// Subscribe to the locally authorized leader record.
5840    #[must_use]
5841    pub fn lease_watch(&self) -> watch::Receiver<Option<LeaderLease>> {
5842        self.lease_tx.subscribe()
5843    }
5844
5845    /// Shared local-monotonic liveness gate for leader hot paths.
5846    #[must_use]
5847    pub fn deadline(&self) -> Arc<LeaseDeadline> {
5848        Arc::clone(&self.deadline)
5849    }
5850
5851    #[cfg(feature = "cluster")]
5852    fn withdraw(&self) {
5853        self.deadline.withdraw();
5854        self.lease_tx.send_replace(None);
5855    }
5856
5857    #[cfg(feature = "cluster")]
5858    fn fence(&self) {
5859        self.deadline.fence();
5860        self.lease_tx.send_replace(None);
5861    }
5862
5863    #[cfg(feature = "cluster")]
5864    async fn attempt_lease(
5865        &self,
5866        shutdown: &tokio_util::sync::CancellationToken,
5867        candidate: &mut watch::Receiver<LeaderCandidacy>,
5868        valid_until: Option<tokio::time::Instant>,
5869        observation: Option<&LeaderLeaseObservation>,
5870        held_token: Option<u64>,
5871    ) -> LeaseOperationEvent {
5872        let Some(attempt_valid_until) = tokio::time::Instant::now().checked_add(self.config.ttl)
5873        else {
5874            return LeaseOperationEvent::Deadline;
5875        };
5876        let operation_deadline = valid_until.map_or(attempt_valid_until, |current| {
5877            current.min(attempt_valid_until)
5878        });
5879        let operation = async {
5880            if let Some(observation) = observation {
5881                Box::pin(
5882                    self.store
5883                        .try_takeover(&self.owner, observation, now_millis()),
5884                )
5885                .await
5886            } else if let Some(token) = held_token {
5887                Box::pin(self.store.renew_exact(&self.owner, token, now_millis())).await
5888            } else {
5889                Box::pin(self.store.begin_new_term(&self.owner, now_millis())).await
5890            }
5891        };
5892        tokio::select! {
5893            biased;
5894            () = shutdown.cancelled() => LeaseOperationEvent::Shutdown,
5895            changed = candidate.changed() => LeaseOperationEvent::Candidacy(changed),
5896            () = wait_for_deadline(Some(operation_deadline)) => LeaseOperationEvent::Deadline,
5897            result = operation => LeaseOperationEvent::Completed {
5898                result,
5899                valid_until: attempt_valid_until,
5900            },
5901        }
5902    }
5903
5904    #[cfg(feature = "cluster")]
5905    async fn wait_for_candidacy_change(
5906        &self,
5907        shutdown: &tokio_util::sync::CancellationToken,
5908        candidate: &mut watch::Receiver<LeaderCandidacy>,
5909    ) -> bool {
5910        tokio::select! {
5911            biased;
5912            () = shutdown.cancelled() => {
5913                self.fence();
5914                false
5915            }
5916            changed = candidate.changed() => {
5917                if changed.is_err() {
5918                    self.fence();
5919                    return false;
5920                }
5921                true
5922            }
5923        }
5924    }
5925
5926    #[cfg(feature = "cluster")]
5927    async fn run(
5928        self,
5929        shutdown: tokio_util::sync::CancellationToken,
5930        mut candidate: watch::Receiver<LeaderCandidacy>,
5931    ) {
5932        let mut ticker = tokio::time::interval(self.config.renew_interval);
5933        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
5934        let mut valid_until: Option<tokio::time::Instant> = None;
5935        let mut observation: Option<LeaderLeaseObservation> = None;
5936        // A newly constructed manager and every locally withdrawn grant need a new durable
5937        // fencing token. Only uninterrupted renewals may preserve the current token.
5938        let mut held_token = None;
5939        let mut candidacy_generation = candidate.borrow().generation;
5940
5941        loop {
5942            let candidacy = *candidate.borrow_and_update();
5943            if candidacy.generation != candidacy_generation {
5944                self.withdraw();
5945                observation = None;
5946                valid_until = None;
5947                held_token = None;
5948                candidacy_generation = candidacy.generation;
5949            }
5950            if !candidacy.eligible {
5951                self.withdraw();
5952                observation = None;
5953                valid_until = None;
5954                held_token = None;
5955                if !self
5956                    .wait_for_candidacy_change(&shutdown, &mut candidate)
5957                    .await
5958                {
5959                    return;
5960                }
5961                continue;
5962            }
5963
5964            tokio::select! {
5965                biased;
5966                () = shutdown.cancelled() => {
5967                    self.fence();
5968                    return;
5969                }
5970                changed = candidate.changed() => {
5971                    if changed.is_err() {
5972                        self.fence();
5973                        return;
5974                    }
5975                    continue;
5976                }
5977                () = wait_for_deadline(valid_until) => {
5978                    self.fence();
5979                    return;
5980                }
5981                _ = ticker.tick() => {}
5982            }
5983
5984            let (result, attempt_valid_until) = match Box::pin(self.attempt_lease(
5985                &shutdown,
5986                &mut candidate,
5987                valid_until,
5988                observation.as_ref(),
5989                held_token,
5990            ))
5991            .await
5992            {
5993                LeaseOperationEvent::Shutdown | LeaseOperationEvent::Deadline => {
5994                    self.fence();
5995                    return;
5996                }
5997                LeaseOperationEvent::Candidacy(changed) => {
5998                    if changed.is_err() {
5999                        self.fence();
6000                        return;
6001                    }
6002                    continue;
6003                }
6004                LeaseOperationEvent::Completed {
6005                    result,
6006                    valid_until: attempt_valid_until,
6007                } => {
6008                    let response_at = tokio::time::Instant::now();
6009                    if response_at >= attempt_valid_until
6010                        || valid_until.is_some_and(|current| response_at >= current)
6011                    {
6012                        self.fence();
6013                        return;
6014                    }
6015                    (result, attempt_valid_until)
6016                }
6017            };
6018
6019            match result {
6020                Ok(LeaseOutcome::Acquired(lease)) if lease.owner == self.owner => {
6021                    let publication_at = tokio::time::Instant::now();
6022                    if publication_at >= attempt_valid_until
6023                        || valid_until.is_some_and(|current| publication_at >= current)
6024                    {
6025                        self.fence();
6026                        return;
6027                    }
6028                    observation = None;
6029                    valid_until = Some(attempt_valid_until);
6030                    held_token = Some(lease.token);
6031                    self.deadline.extend_until(attempt_valid_until.into_std());
6032                    self.lease_tx.send_replace(Some(lease));
6033                }
6034                Ok(LeaseOutcome::Acquired(_)) => {
6035                    self.fence();
6036                    return;
6037                }
6038                Ok(LeaseOutcome::Held(rival)) => {
6039                    self.withdraw();
6040                    valid_until = None;
6041                    held_token = None;
6042                    let unchanged = observation
6043                        .as_ref()
6044                        .is_some_and(|observed| observed.lease.has_same_liveness_identity(&rival));
6045                    if !unchanged {
6046                        match self.store.observe_rival(&self.owner, &rival) {
6047                            Ok(new_observation) => observation = Some(new_observation),
6048                            Err(error) => {
6049                                tracing::warn!(%error, "leader lease observation rejected");
6050                                observation = None;
6051                            }
6052                        }
6053                    }
6054                }
6055                Err(LeaseError::Fenced(error)) => {
6056                    tracing::warn!(%error, "leader lease renewal was fenced");
6057                    self.fence();
6058                    return;
6059                }
6060                Err(error) => {
6061                    tracing::warn!(%error, "leader lease operation failed");
6062                }
6063            }
6064        }
6065    }
6066
6067    /// Spawn the renewal loop. Loss of candidacy withdraws the current local grant so this
6068    /// manager can contend again later. Shutdown, a missed renewal, or an invalid lease outcome
6069    /// terminally fences the manager.
6070    #[cfg(feature = "cluster")]
6071    #[must_use]
6072    pub fn spawn(
6073        self,
6074        shutdown: tokio_util::sync::CancellationToken,
6075        candidate: watch::Receiver<LeaderCandidacy>,
6076    ) -> tokio::task::JoinHandle<()> {
6077        tokio::spawn(self.run(shutdown, candidate))
6078    }
6079}
6080
6081#[cfg(test)]
6082mod tests;