Skip to main content

laminar_core/cluster/control/leader_lease/
mod.rs

1//! Durable, append-only leader fencing.
2
3mod artifact_admission;
4mod attempt_status;
5mod subscription_replay;
6
7pub use attempt_status::ClusterAttemptStatus;
8pub use subscription_replay::{
9    SubscriptionReplayPin, SubscriptionReplayPinAcquire, SUBSCRIPTION_REPLAY_PIN_RENEW_INTERVAL,
10};
11
12use std::collections::{BTreeMap, BTreeSet};
13use std::sync::atomic::{AtomicBool, Ordering};
14use std::sync::{Arc, Weak};
15use std::time::{Duration, Instant};
16
17use bytes::Bytes;
18use object_store::path::Path as OsPath;
19use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
20use parking_lot::Mutex;
21use serde::{Deserialize, Serialize};
22use sha2::{Digest as _, Sha256};
23use tokio::sync::watch;
24use tokio_stream::StreamExt;
25use uuid::Uuid;
26
27use crate::checkpoint::{
28    probe_object_store_conditional_update, AssignmentDrainId, AssignmentDrainTransition,
29    CheckpointAssignmentFence, CheckpointStoreError, CommittedCheckpointIndex,
30    CommittedCheckpointRef, LeaderProof, LeaderProofOwner, MAX_CHECKPOINT_PARTICIPANTS,
31};
32use crate::checkpoint_decision::{
33    CheckpointArtifactInventory, CheckpointDecisionStore, CheckpointOutcome, CheckpointScope,
34    CheckpointVerdict, DecisionError, RecordOutcomeResult,
35};
36use crate::cluster::discovery::NodeId;
37
38use super::catalog_manifest::{
39    CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
40};
41use super::controller::{
42    RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
43    RecoveryFaultDisposition, RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
44    MAX_RECOVERY_ANNOUNCEMENT_BYTES,
45};
46use super::lease_deadline::LeaseDeadline;
47use super::process_lease::{ProcessLease, ProcessLeaseFence};
48use super::snapshot::{
49    AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
50};
51
52const LEASE_PREFIX: &str = "control/leader-lease/";
53const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
54const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
55const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
56const AUTHORITY_RECORD_VERSION: u32 = 12;
57const AUTHORITY_HEAD_VERSION: u32 = 1;
58const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
59const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
60const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
61    + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
62    + 2;
63const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
64const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
65const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
66const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
67const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
68const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
69const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
70const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
71const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
72const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
73const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
74const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
75#[cfg(test)]
76const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
77
78fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
79    match error {
80        SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
81        error => LeaseError::Invalid(format!("{context}: {error}")),
82    }
83}
84
85fn lease_path(sequence: u64) -> OsPath {
86    OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
87}
88
89fn authority_head_path() -> OsPath {
90    OsPath::from(AUTHORITY_HEAD_PATH)
91}
92
93fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
94    OsPath::from(format!(
95        "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
96        reference.release.generation(),
97        reference.sha256
98    ))
99}
100
101fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
102    let raw = path
103        .as_ref()
104        .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
105        .and_then(|value| value.strip_prefix("generation="))
106        .ok_or_else(|| {
107            LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
108        })?;
109    let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
110        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
111    })?;
112    let digest = digest.strip_suffix(".json").ok_or_else(|| {
113        LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
114    })?;
115    if generation.len() != 20
116        || !generation.bytes().all(|byte| byte.is_ascii_digit())
117        || digest.len() != 64
118        || !digest
119            .bytes()
120            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
121    {
122        return Err(LeaseError::Invalid(format!(
123            "invalid recovery release terminal path {path}"
124        )));
125    }
126    let generation = generation.parse::<u64>().map_err(|error| {
127        LeaseError::Invalid(format!(
128            "invalid recovery release terminal generation in {path}: {error}"
129        ))
130    })?;
131    if generation == 0 {
132        return Err(LeaseError::Invalid(format!(
133            "invalid recovery release terminal path {path}"
134        )));
135    }
136    Ok((generation, digest.to_owned()))
137}
138
139fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
140    let raw = path
141        .as_ref()
142        .strip_prefix(LEASE_PREFIX)
143        .and_then(|file| file.strip_prefix('v'))
144        .and_then(|file| file.strip_suffix(".json"))
145        .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
146    if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
147        return Err(LeaseError::Invalid(format!(
148            "invalid leader authority sequence in {path}"
149        )));
150    }
151    let sequence = raw.parse::<u64>().map_err(|error| {
152        LeaseError::Invalid(format!(
153            "invalid leader authority sequence in {path}: {error}"
154        ))
155    })?;
156    if sequence == 0 || lease_path(sequence) != *path {
157        return Err(LeaseError::Invalid(format!(
158            "noncanonical leader authority path {path}"
159        )));
160    }
161    Ok(sequence)
162}
163
164fn consume_live_authority_link(traversed: &mut usize) -> bool {
165    if *traversed == MAX_LIVE_AUTHORITY_LINKS {
166        return false;
167    }
168    *traversed += 1;
169    true
170}
171
172fn now_millis() -> i64 {
173    std::time::SystemTime::now()
174        .duration_since(std::time::UNIX_EPOCH)
175        .ok()
176        .and_then(|duration| i64::try_from(duration.as_millis()).ok())
177        .unwrap_or(i64::MAX)
178}
179
180async fn read_authority_record(
181    store: &dyn ObjectStore,
182    sequence: u64,
183) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
184    let result = match store.get(&lease_path(sequence)).await {
185        Ok(result) => result,
186        Err(object_store::Error::NotFound { .. }) => return Ok(None),
187        Err(error) => return Err(LeaseError::Io(error.to_string())),
188    };
189    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
190        return Err(LeaseError::Invalid(format!(
191            "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
192            result.meta.size
193        )));
194    }
195    let bytes = result
196        .bytes()
197        .await
198        .map_err(|error| LeaseError::Io(error.to_string()))?;
199    let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
200    record.validate()?;
201    if record.lease.seq != sequence {
202        return Err(LeaseError::Invalid(
203            "authority record sequence does not match its object name".into(),
204        ));
205    }
206    let canonical = serde_json::to_vec(&record)?;
207    if canonical.as_slice() != bytes.as_ref() {
208        return Err(LeaseError::Invalid(format!(
209            "leader authority record {sequence} does not use its canonical body"
210        )));
211    }
212    Ok(Some(record))
213}
214
215#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
216#[serde(deny_unknown_fields)]
217struct AuthorityHeadPointer {
218    version: u32,
219    sequence: u64,
220    nonce: Uuid,
221}
222
223impl AuthorityHeadPointer {
224    fn new(sequence: u64) -> Result<Self, LeaseError> {
225        let pointer = Self {
226            version: AUTHORITY_HEAD_VERSION,
227            sequence,
228            nonce: Uuid::new_v4(),
229        };
230        pointer.validate()?;
231        Ok(pointer)
232    }
233
234    fn validate(self) -> Result<(), LeaseError> {
235        if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
236            return Err(LeaseError::Invalid(
237                "leader authority head has an unsupported version, zero sequence, or nil nonce"
238                    .into(),
239            ));
240        }
241        Ok(())
242    }
243}
244
245#[derive(Clone)]
246struct VersionedAuthorityHeadPointer {
247    pointer: AuthorityHeadPointer,
248    update_version: UpdateVersion,
249}
250
251struct PublishedAuthorityHead {
252    record: LeaderAuthorityRecord,
253    pointer: VersionedAuthorityHeadPointer,
254}
255
256async fn read_authority_head_pointer(
257    store: &dyn ObjectStore,
258) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
259    let result = match store.get(&authority_head_path()).await {
260        Ok(result) => result,
261        Err(object_store::Error::NotFound { .. }) => return Ok(None),
262        Err(error) => return Err(LeaseError::Io(error.to_string())),
263    };
264    if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
265        return Err(LeaseError::Invalid(format!(
266            "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
267            result.meta.size
268        )));
269    }
270    let update_version = UpdateVersion {
271        e_tag: result.meta.e_tag.clone(),
272        version: result.meta.version.clone(),
273    };
274    let bytes = result
275        .bytes()
276        .await
277        .map_err(|error| LeaseError::Io(error.to_string()))?;
278    let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
279    pointer.validate()?;
280    let canonical = serde_json::to_vec(&pointer)?;
281    if canonical.as_slice() != bytes.as_ref() {
282        return Err(LeaseError::Invalid(
283            "leader authority head does not use its canonical body".into(),
284        ));
285    }
286    Ok(Some(VersionedAuthorityHeadPointer {
287        pointer,
288        update_version,
289    }))
290}
291
292fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
293    let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
294    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
295    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
296        return Err(LeaseError::Invalid(format!(
297            "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
298            encoded.len()
299        )));
300    }
301    Ok(Bytes::from(encoded))
302}
303
304fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
305    record.validate()?;
306    let encoded = serde_json::to_vec(record)?;
307    let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
308    if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
309        return Err(LeaseError::Invalid(format!(
310            "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
311            encoded.len()
312        )));
313    }
314    if record.recovery_fault_slots.iter().any(|slot| slot.active) {
315        ensure_recovery_fault_authority_headroom(encoded_len)?;
316    }
317    Ok(Bytes::from(encoded))
318}
319
320fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
321    let limit = MAX_AUTHORITY_RECORD_BYTES
322        .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
323        .expect("recovery fault headroom must fit the authority record");
324    if encoded_len > limit {
325        return Err(LeaseError::Invalid(format!(
326            "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
327        )));
328    }
329    Ok(())
330}
331
332/// Exact process incarnation eligible to hold the leader lease.
333#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
334pub struct LeaderLeaseOwner {
335    /// Stable cluster node identity.
336    pub node: NodeId,
337    /// Boot-unique process identity.
338    pub boot: Uuid,
339    /// Durable stable-node process term.
340    pub process_term: u64,
341}
342
343impl LeaderLeaseOwner {
344    fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
345        process
346            .validate(process.node)
347            .map_err(|error| LeaseError::Invalid(error.to_string()))?;
348        Ok(Self {
349            node: process.node,
350            boot: process.owner,
351            process_term: process.term,
352        })
353    }
354
355    fn validate(&self) -> Result<(), LeaseError> {
356        if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
357            return Err(LeaseError::Invalid(
358                "leader owner node, boot identity, and process term must be nonzero".into(),
359            ));
360        }
361        Ok(())
362    }
363
364    fn proof_owner(&self) -> LeaderProofOwner {
365        LeaderProofOwner {
366            node_id: self.node.0,
367            boot_id: self.boot,
368            process_term: self.process_term,
369        }
370    }
371}
372
373/// Durable leader lease record.
374#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
375pub struct LeaderLease {
376    /// Append-only compare-and-set sequence.
377    pub seq: u64,
378    /// Monotonic liveness sequence, advanced only by an acquisition or renewal.
379    pub renewal_sequence: u64,
380    /// Fencing token, stable across uninterrupted renewals and advanced for each authority term.
381    pub token: u64,
382    /// Exact process incarnation holding the lease.
383    pub owner: LeaderLeaseOwner,
384    /// Owner-written wall-clock expiry for diagnostics only.
385    pub expires_at_ms: i64,
386    /// Immutable catalog content reference, once sealed for this control namespace.
387    pub catalog_manifest: Option<CatalogManifestRef>,
388}
389
390#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
391#[serde(deny_unknown_fields)]
392struct OutcomeLink {
393    sequence: u64,
394    epoch: u64,
395    checkpoint_id: u64,
396}
397
398impl OutcomeLink {
399    fn validate(self) -> Result<(), LeaseError> {
400        let attempt = crate::checkpoint::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
401        if self.sequence == 0 || !attempt.is_canonical() {
402            return Err(LeaseError::Invalid(
403                "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
404                    .into(),
405            ));
406        }
407        Ok(())
408    }
409}
410
411#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
412#[serde(deny_unknown_fields)]
413struct AssignmentDecisionLink {
414    sequence: u64,
415    target_version: u64,
416}
417
418#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
419#[serde(deny_unknown_fields)]
420struct AuthorityRecoveryFaultSlot {
421    publisher: RecoveryFaultPublisher,
422    request_sequence: u64,
423    fault_sequence: u64,
424    #[serde(
425        default,
426        skip_serializing_if = "RecoveryFaultDisposition::is_recoverable"
427    )]
428    disposition: RecoveryFaultDisposition,
429    active: bool,
430}
431
432impl AuthorityRecoveryFaultSlot {
433    fn validate(&self) -> Result<(), LeaseError> {
434        self.publisher.validate().map_err(LeaseError::Invalid)?;
435        if self.request_sequence == 0 || self.fault_sequence == 0 {
436            return Err(LeaseError::Invalid(
437                "recovery fault request and authority sequence must be nonzero".into(),
438            ));
439        }
440        if self.disposition == RecoveryFaultDisposition::Terminal && !self.active {
441            return Err(LeaseError::Invalid(
442                "terminal recovery fault authority cannot be tombstoned".into(),
443            ));
444        }
445        Ok(())
446    }
447
448    fn fault(&self) -> RecoveryFault {
449        RecoveryFault {
450            reporter: NodeId(self.publisher.participant.node_id),
451            sequence: self.fault_sequence,
452            disposition: self.disposition,
453        }
454    }
455
456    fn matches_request(
457        &self,
458        publisher: RecoveryFaultPublisher,
459        request_sequence: u64,
460        disposition: RecoveryFaultDisposition,
461    ) -> bool {
462        self.publisher == publisher
463            && self.request_sequence == request_sequence
464            && self.disposition == disposition
465    }
466}
467
468#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
469#[serde(deny_unknown_fields)]
470pub(crate) struct RecoveryReleaseTerminalRef {
471    release: RecoveryReleaseId,
472    sha256: String,
473    encoded_len: u64,
474}
475
476impl RecoveryReleaseTerminalRef {
477    fn validate(&self) -> Result<(), LeaseError> {
478        if !self.release.is_canonical()
479            || self.sha256.len() != 64
480            || !self
481                .sha256
482                .bytes()
483                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
484            || self.encoded_len == 0
485            || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
486        {
487            return Err(LeaseError::Invalid(
488                "recovery release terminal reference is not canonical".into(),
489            ));
490        }
491        Ok(())
492    }
493
494    pub(crate) fn generation(&self) -> u64 {
495        self.release.generation()
496    }
497}
498
499#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
500#[serde(deny_unknown_fields)]
501struct AuthorityRecoveryReleaseCommit {
502    terminal: RecoveryReleaseTerminalRef,
503    leader_proof: LeaderProof,
504}
505
506impl AuthorityRecoveryReleaseCommit {
507    fn validate(&self) -> Result<(), LeaseError> {
508        self.terminal.validate()?;
509        if !self.leader_proof.is_canonical() {
510            return Err(LeaseError::Invalid(
511                "recovery release commit has no canonical leader proof".into(),
512            ));
513        }
514        Ok(())
515    }
516}
517
518#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
519#[serde(deny_unknown_fields)]
520pub(crate) struct RecoveryReleaseLink {
521    sequence: u64,
522    terminal: RecoveryReleaseTerminalRef,
523}
524
525impl RecoveryReleaseLink {
526    fn validate(&self) -> Result<(), LeaseError> {
527        if self.sequence == 0 {
528            return Err(LeaseError::Invalid(
529                "recovery release authority link has a zero sequence".into(),
530            ));
531        }
532        self.terminal.validate()
533    }
534}
535
536#[derive(Debug, Clone, PartialEq, Eq)]
537pub(crate) enum RecordRecoveryReleaseCommitResult {
538    Created(RecoveryReleaseTerminalRef),
539    Unchanged(RecoveryReleaseTerminalRef),
540    Conflict { winner: RecoveryReleaseTerminalRef },
541    FaultsChanged,
542}
543
544#[derive(Debug, Clone, Copy, PartialEq, Eq)]
545pub(crate) enum RecordRecoveryFaultResult {
546    Active,
547    AlreadyCleared,
548    CoveredByNewerRequest,
549    TerminalFenceActive,
550    Superseded,
551}
552
553fn recovery_release_id_for_terminal(
554    terminal: &RecoveryAnnouncement,
555) -> Result<RecoveryReleaseId, LeaseError> {
556    terminal.validate().map_err(LeaseError::Invalid)?;
557    let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
558        return Err(LeaseError::Invalid(
559            "recovery release terminal must carry ReleaseCommitted".into(),
560        ));
561    };
562    RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
563        round: terminal.round.clone(),
564        phase: RecoverPhase::Release { epoch },
565    })
566    .map_err(LeaseError::Invalid)
567}
568
569fn encode_recovery_release_terminal(
570    terminal: &RecoveryAnnouncement,
571) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
572    let release = recovery_release_id_for_terminal(terminal)?;
573    let encoded = serde_json::to_vec(terminal)?;
574    let encoded_len = u64::try_from(encoded.len())
575        .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
576    if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
577        return Err(LeaseError::Invalid(format!(
578            "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
579        )));
580    }
581    let reference = RecoveryReleaseTerminalRef {
582        release,
583        sha256: format!("{:x}", Sha256::digest(&encoded)),
584        encoded_len,
585    };
586    reference.validate()?;
587    Ok((Bytes::from(encoded), reference))
588}
589
590/// Immutable settlement of one exact assignment-drain transition.
591#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
592#[serde(deny_unknown_fields)]
593pub struct AssignmentDrainDecision {
594    /// Exact predecessor-to-target transition being settled.
595    pub transition: AssignmentDrainTransition,
596    /// Durable leader term that decided the transition.
597    pub leader_proof: LeaderProof,
598    /// Whether to install the certified target or restore the predecessor map.
599    pub verdict: AssignmentDrainVerdict,
600    /// Exact predecessor-fenced checkpoint used to hand moved vnodes to the target assignment.
601    /// Present if and only if the drain commits.
602    pub handoff_checkpoint: Option<CommittedCheckpointRef>,
603}
604
605impl AssignmentDrainDecision {
606    /// Commit an exact canonical transition against its state handoff cut.
607    ///
608    /// # Errors
609    /// Rejects malformed authority or checkpoint references.
610    pub fn commit(
611        transition: &AssignmentDrainTransition,
612        leader_proof: LeaderProof,
613        handoff_checkpoint: CommittedCheckpointRef,
614    ) -> Result<Self, String> {
615        handoff_checkpoint.validate()?;
616        if !transition.is_canonical() || leader_proof != transition.leader {
617            return Err("assignment drain decision requires a canonical transition".into());
618        }
619        Ok(Self {
620            transition: transition.clone(),
621            leader_proof,
622            verdict: AssignmentDrainVerdict::Commit,
623            handoff_checkpoint: Some(handoff_checkpoint),
624        })
625    }
626
627    /// Abort an exact canonical transition under the deciding leader term.
628    ///
629    /// # Errors
630    /// Rejects malformed transition or leader authority.
631    pub fn abort(
632        transition: &AssignmentDrainTransition,
633        leader_proof: LeaderProof,
634    ) -> Result<Self, String> {
635        if !transition.is_canonical() || !leader_proof.is_canonical() {
636            return Err("assignment drain decision requires a canonical transition".into());
637        }
638        Ok(Self {
639            transition: transition.clone(),
640            leader_proof,
641            verdict: AssignmentDrainVerdict::Abort,
642            handoff_checkpoint: None,
643        })
644    }
645
646    /// Compact identity used by source-drain receipts and authority lookups.
647    #[must_use]
648    pub fn round(&self) -> AssignmentDrainId {
649        self.transition.id()
650    }
651
652    /// Target assignment version settled by this decision.
653    #[must_use]
654    pub fn target_version(&self) -> u64 {
655        self.transition.target.assignment_version
656    }
657
658    fn validate(&self) -> Result<(), LeaseError> {
659        if !self.transition.is_canonical()
660            || !self.leader_proof.is_canonical()
661            || (self.verdict == AssignmentDrainVerdict::Commit
662                && self.leader_proof != self.transition.leader)
663        {
664            return Err(LeaseError::Invalid(
665                "assignment drain decision is not canonical".into(),
666            ));
667        }
668        match (self.verdict, self.handoff_checkpoint.as_ref()) {
669            (AssignmentDrainVerdict::Commit, Some(reference)) => {
670                reference.validate().map_err(LeaseError::Invalid)?;
671            }
672            (AssignmentDrainVerdict::Abort, None) => {}
673            _ => {
674                return Err(LeaseError::Invalid(
675                    "assignment drain decision has an invalid handoff checkpoint".into(),
676                ));
677            }
678        }
679        Ok(())
680    }
681}
682
683/// Terminal result for an assignment-drain transition.
684#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
685#[serde(rename_all = "snake_case")]
686pub enum AssignmentDrainVerdict {
687    /// Install the transition's certified target assignment.
688    Commit,
689    /// Restore the predecessor owner map at the target version.
690    Abort,
691}
692
693/// Result of admitting a drain decision through the shared authority sequence.
694#[derive(Debug, Clone, PartialEq, Eq)]
695pub enum RecordAssignmentDrainDecisionResult {
696    /// This call created the immutable decision.
697    Created(AssignmentDrainDecision),
698    /// The same decision was already durable.
699    Unchanged(AssignmentDrainDecision),
700    /// Another terminal decision already won for this transition version.
701    Conflict {
702        /// Immutable winner.
703        winner: AssignmentDrainDecision,
704    },
705}
706
707/// Immutable authorization to install one staged failure-recovery assignment.
708#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
709#[serde(deny_unknown_fields)]
710pub struct AssignmentRecoveryDecision {
711    /// Exact assignment being replaced.
712    pub predecessor: CheckpointAssignmentFence,
713    /// Exact successor assignment authorized for installation.
714    pub target: CheckpointAssignmentFence,
715    /// Content-addressed staged snapshot whose assignment must equal `target`.
716    pub proposal: AssignmentSnapshotRef,
717    /// Exact process-lease takeover proofs for every predecessor process absent from `target`.
718    pub removed_process_fences: Vec<ProcessLeaseFence>,
719    /// Exact committed state cut installed by this recovery.
720    pub recovery_checkpoint: CommittedCheckpointRef,
721    /// Durable leader term that authorized this recovery.
722    pub leader_proof: LeaderProof,
723}
724
725impl AssignmentRecoveryDecision {
726    /// Construct one exact failure-recovery authorization.
727    ///
728    /// # Errors
729    /// Rejects invalid fences, a non-successor proposal, or an incomplete process-removal proof.
730    pub fn new(
731        predecessor: CheckpointAssignmentFence,
732        target: CheckpointAssignmentFence,
733        proposal: AssignmentSnapshotRef,
734        removed_process_fences: Vec<ProcessLeaseFence>,
735        recovery_checkpoint: CommittedCheckpointRef,
736        leader_proof: LeaderProof,
737    ) -> Result<Self, String> {
738        let decision = Self {
739            predecessor,
740            target,
741            proposal,
742            removed_process_fences,
743            recovery_checkpoint,
744            leader_proof,
745        };
746        decision.validate().map_err(|error| error.to_string())?;
747        Ok(decision)
748    }
749
750    /// Target assignment version settled by this decision.
751    #[must_use]
752    pub fn target_version(&self) -> u64 {
753        self.target.assignment_version
754    }
755
756    fn validate(&self) -> Result<(), LeaseError> {
757        if !self.predecessor.is_canonical()
758            || !self.target.is_canonical()
759            || !self.leader_proof.is_canonical()
760            || self.predecessor.vnode_count != self.target.vnode_count
761            || self.predecessor.assignment_version.checked_add(1)
762                != Some(self.target.assignment_version)
763            || self.proposal.validate().is_err()
764            || self.proposal.version != self.target.assignment_version
765            || self.recovery_checkpoint.validate().is_err()
766        {
767            return Err(LeaseError::Invalid(
768                "assignment recovery decision is not a canonical successor".into(),
769            ));
770        }
771
772        let removed: Vec<_> = self
773            .predecessor
774            .participants
775            .iter()
776            .filter(|participant| {
777                self.target.participant_incarnation(participant.node_id)
778                    != Some(participant.boot_incarnation)
779            })
780            .collect();
781        if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
782            return Err(LeaseError::Invalid(
783                "assignment recovery decision requires the exact removed predecessor process set"
784                    .into(),
785            ));
786        }
787        for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
788            if !fence.is_canonical()
789                || fence.predecessor.node.0 != participant.node_id
790                || fence.predecessor.owner != participant.boot_incarnation
791            {
792                return Err(LeaseError::Invalid(
793                    "assignment recovery process fences are incomplete, unsorted, or unrelated"
794                        .into(),
795                ));
796            }
797        }
798        Ok(())
799    }
800}
801
802/// Result of admitting a recovery decision through the shared authority sequence.
803#[derive(Debug, Clone, PartialEq, Eq)]
804pub enum RecordAssignmentRecoveryDecisionResult {
805    /// This call created the immutable decision.
806    Created(AssignmentRecoveryDecision),
807    /// The same decision was already durable.
808    Unchanged(AssignmentRecoveryDecision),
809    /// Another recovery decision already won for this target version.
810    Conflict {
811        /// Immutable winner.
812        winner: AssignmentRecoveryDecision,
813    },
814}
815
816#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
817#[serde(
818    tag = "kind",
819    content = "decision",
820    rename_all = "snake_case",
821    deny_unknown_fields
822)]
823enum AuthorityAssignmentDecision {
824    Drain(AssignmentDrainDecision),
825    Recovery(AssignmentRecoveryDecision),
826}
827
828impl AuthorityAssignmentDecision {
829    fn target_version(&self) -> u64 {
830        match self {
831            Self::Drain(decision) => decision.target_version(),
832            Self::Recovery(decision) => decision.target_version(),
833        }
834    }
835
836    fn predecessor(&self) -> &CheckpointAssignmentFence {
837        match self {
838            Self::Drain(decision) => &decision.transition.predecessor,
839            Self::Recovery(decision) => &decision.predecessor,
840        }
841    }
842
843    fn materialized_target(&self) -> CheckpointAssignmentFence {
844        match self {
845            Self::Drain(decision) => match decision.verdict {
846                AssignmentDrainVerdict::Commit => decision.transition.target.clone(),
847                AssignmentDrainVerdict::Abort => {
848                    let mut target = decision.transition.predecessor.clone();
849                    target.assignment_version = decision.transition.target.assignment_version;
850                    target
851                }
852            },
853            Self::Recovery(decision) => decision.target.clone(),
854        }
855    }
856
857    fn leader_proof(&self) -> &LeaderProof {
858        match self {
859            Self::Drain(decision) => &decision.leader_proof,
860            Self::Recovery(decision) => &decision.leader_proof,
861        }
862    }
863
864    fn validate(&self) -> Result<(), LeaseError> {
865        match self {
866            Self::Drain(decision) => decision.validate(),
867            Self::Recovery(decision) => decision.validate(),
868        }
869    }
870}
871
872enum RecordAuthorityAssignmentDecisionResult {
873    Created(AuthorityAssignmentDecision),
874    Unchanged(AuthorityAssignmentDecision),
875    Conflict { winner: AuthorityAssignmentDecision },
876}
877
878/// Destructive phase authorized by the cluster leader authority.
879#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
880#[serde(rename_all = "snake_case")]
881pub enum ClusterArtifactCleanupPhase {
882    /// Delete state objects that are not referenced by the protected recovery cuts.
883    DeleteData,
884    /// Delete the exact participant manifests and committed-checkpoint index.
885    DeleteMetadata,
886}
887
888/// Exact, crash-resumable cluster checkpoint artifact cleanup position.
889#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
890#[serde(deny_unknown_fields)]
891pub struct ClusterArtifactCleanupCursor {
892    /// Oldest live committed checkpoint protected by this retention segment.
893    pub protected: CommittedCheckpointRef,
894    /// Exact expired committed checkpoint being reclaimed.
895    pub current: CommittedCheckpointRef,
896    /// Exact older checkpoint to process after `current`, if any.
897    pub next: Option<CommittedCheckpointRef>,
898    /// Previously reclaimed checkpoint at which this segment must stop.
899    pub stop_before: Option<CommittedCheckpointRef>,
900    /// Exact participants whose manifests belong to `current`.
901    pub participant_ids: Vec<u64>,
902    /// Destructive operation durably authorized for `current`.
903    pub phase: ClusterArtifactCleanupPhase,
904}
905
906impl ClusterArtifactCleanupCursor {
907    fn validate(&self) -> Result<(), LeaseError> {
908        for (name, reference) in [
909            ("protected", Some(&self.protected)),
910            ("current", Some(&self.current)),
911            ("next", self.next.as_ref()),
912            ("stop-before", self.stop_before.as_ref()),
913        ] {
914            if let Some(reference) = reference {
915                reference.validate().map_err(|error| {
916                    LeaseError::Invalid(format!(
917                        "cluster artifact cleanup {name} reference: {error}"
918                    ))
919                })?;
920            }
921        }
922        if self.current.epoch >= self.protected.epoch
923            || self.current.checkpoint_id >= self.protected.checkpoint_id
924        {
925            return Err(LeaseError::Invalid(
926                "cluster artifact cleanup current checkpoint is not older than its protected cut"
927                    .into(),
928            ));
929        }
930        if let Some(next) = self.next.as_ref() {
931            if next.epoch >= self.current.epoch || next.checkpoint_id >= self.current.checkpoint_id
932            {
933                return Err(LeaseError::Invalid(
934                    "cluster artifact cleanup next checkpoint is not older than its current checkpoint"
935                        .into(),
936                ));
937            }
938        }
939        if let Some(stop) = self.stop_before.as_ref() {
940            if stop.epoch >= self.current.epoch || stop.checkpoint_id >= self.current.checkpoint_id
941            {
942                return Err(LeaseError::Invalid(
943                    "cluster artifact cleanup stop boundary is not older than its current checkpoint"
944                        .into(),
945                ));
946            }
947            let next = self.next.as_ref().ok_or_else(|| {
948                LeaseError::Invalid(
949                    "cluster artifact cleanup lost the path to its stop boundary".into(),
950                )
951            })?;
952            if next.epoch < stop.epoch || next.checkpoint_id < stop.checkpoint_id {
953                return Err(LeaseError::Invalid(
954                    "cluster artifact cleanup next checkpoint crosses its stop boundary".into(),
955                ));
956            }
957        }
958        if self.participant_ids.is_empty()
959            || self.participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
960            || self.participant_ids[0] == 0
961            || !self
962                .participant_ids
963                .windows(2)
964                .all(|pair| pair[0] < pair[1])
965        {
966            return Err(LeaseError::Invalid(
967                "cluster artifact cleanup participants are not canonical".into(),
968            ));
969        }
970        Ok(())
971    }
972}
973
974#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
975#[serde(deny_unknown_fields)]
976struct AuthorityOutcomeFloor {
977    deployment_id: String,
978    artifact_before_epoch: u64,
979    authority_before_epoch: u64,
980    terminal_anchor: Option<CheckpointOutcome>,
981    terminal_anchor_link: Option<OutcomeLink>,
982    committed_anchor: Option<CheckpointOutcome>,
983    committed_anchor_link: Option<OutcomeLink>,
984}
985
986#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
987#[serde(deny_unknown_fields)]
988struct AuthorityAssignmentDecisionFloor {
989    before_target_version: u64,
990    terminal_anchor: Option<AuthorityAssignmentDecision>,
991    terminal_anchor_link: Option<AssignmentDecisionLink>,
992}
993
994#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
995#[serde(deny_unknown_fields)]
996struct AssignmentHandoffPin {
997    target: CheckpointAssignmentFence,
998    checkpoint: CommittedCheckpointRef,
999}
1000
1001impl AssignmentHandoffPin {
1002    fn validate(&self) -> Result<(), LeaseError> {
1003        if !self.target.is_canonical() {
1004            return Err(LeaseError::Invalid(
1005                "assignment handoff pin has an invalid target fence".into(),
1006            ));
1007        }
1008        self.checkpoint.validate().map_err(LeaseError::Invalid)
1009    }
1010}
1011
1012/// One immutable entry in the cluster's shared leadership and checkpoint-decision sequence.
1013#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1014#[serde(deny_unknown_fields)]
1015struct LeaderAuthorityRecord {
1016    version: u32,
1017    lease: LeaderLease,
1018    /// Present only on the sequence that admitted this terminal outcome.
1019    checkpoint_outcome: Option<CheckpointOutcome>,
1020    /// Link to the preceding admitted outcome, present only on an outcome-bearing record.
1021    previous_outcome: Option<OutcomeLink>,
1022    /// Latest admitted outcome. Renewals, takeovers, catalog seals, and floor advances preserve it.
1023    outcome_head: Option<OutcomeLink>,
1024    /// Link to the preceding admitted Commit, present only on a Commit-bearing record.
1025    previous_commit: Option<OutcomeLink>,
1026    /// Latest admitted Commit. Every authority mutation preserves it; Abort does not advance it.
1027    commit_head: Option<OutcomeLink>,
1028    /// Monotonic cluster outcome retention boundary and its continuity anchors.
1029    outcome_floor: Option<AuthorityOutcomeFloor>,
1030    /// Exact leader-fenced artifact cleanup position, preserved across leadership terms.
1031    artifact_cleanup: Option<ClusterArtifactCleanupCursor>,
1032    /// Latest replay-retention floor linearized through this authority sequence.
1033    #[serde(default, skip_serializing_if = "Option::is_none")]
1034    subscription_cleanup_commit: Option<subscription_replay::SubscriptionCleanupCommit>,
1035    /// Exact unresolved checkpoint attempt whose artifacts require a terminal decision or cleanup.
1036    active_checkpoint_artifacts: Option<CheckpointArtifactInventory>,
1037    /// Leader term that admitted `active_checkpoint_artifacts` before any artifact write.
1038    active_checkpoint_artifact_leader_proof: Option<LeaderProof>,
1039    /// Present only on the sequence that admitted an assignment decision.
1040    assignment_decision: Option<AuthorityAssignmentDecision>,
1041    /// Link to the preceding assignment decision, present only on a decision-bearing record.
1042    previous_assignment_decision: Option<AssignmentDecisionLink>,
1043    /// Latest admitted assignment decision. Every other authority mutation preserves it.
1044    assignment_decision_head: Option<AssignmentDecisionLink>,
1045    /// Monotonic assignment-decision retention boundary and its continuity anchor.
1046    assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
1047    /// State handoff cut retained until a complete checkpoint commits under its target fence.
1048    assignment_handoff_pin: Option<AssignmentHandoffPin>,
1049    /// Authority sequence that admitted the current recovery-fault slot state.
1050    recovery_fault_revision: u64,
1051    /// One active or tombstoned request identity per stable node.
1052    recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
1053    /// Present only on the sequence that admitted the latest recovery release.
1054    recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
1055    /// Latest admitted recovery release, preserved by every later authority mutation.
1056    recovery_release_head: Option<RecoveryReleaseLink>,
1057}
1058
1059impl LeaderLease {
1060    fn validate(&self) -> Result<(), LeaseError> {
1061        self.owner.validate()?;
1062        if self.seq == 0
1063            || self.renewal_sequence == 0
1064            || self.renewal_sequence > self.seq
1065            || self.token == 0
1066        {
1067            return Err(LeaseError::Invalid(
1068                "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
1069                    .into(),
1070            ));
1071        }
1072        if let Some(reference) = &self.catalog_manifest {
1073            reference
1074                .validate()
1075                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1076        }
1077        Ok(())
1078    }
1079
1080    /// Exact feature-neutral proof for this ownership term.
1081    #[must_use]
1082    pub fn proof(&self) -> LeaderProof {
1083        LeaderProof {
1084            owner: self.owner.proof_owner(),
1085            fencing_token: self.token,
1086        }
1087    }
1088
1089    /// Whether `proof` names this exact owner and fencing token.
1090    #[must_use]
1091    pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
1092        proof.is_canonical()
1093            && proof.owner == self.owner.proof_owner()
1094            && proof.fencing_token == self.token
1095    }
1096
1097    fn has_same_liveness_identity(&self, other: &Self) -> bool {
1098        self.owner == other.owner
1099            && self.token == other.token
1100            && self.renewal_sequence == other.renewal_sequence
1101    }
1102}
1103
1104impl AuthorityOutcomeFloor {
1105    fn validate(&self) -> Result<(), LeaseError> {
1106        let deployment = Uuid::parse_str(&self.deployment_id)
1107            .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
1108        if deployment.is_nil()
1109            || deployment.to_string() != self.deployment_id
1110            || self.authority_before_epoch == 0
1111            || self.artifact_before_epoch > self.authority_before_epoch
1112        {
1113            return Err(LeaseError::Invalid(
1114                "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
1115                    .into(),
1116            ));
1117        }
1118        for link in [self.terminal_anchor_link, self.committed_anchor_link]
1119            .into_iter()
1120            .flatten()
1121        {
1122            link.validate()?;
1123        }
1124        for (name, anchor) in [
1125            ("terminal", self.terminal_anchor.as_ref()),
1126            ("committed", self.committed_anchor.as_ref()),
1127        ] {
1128            let Some(anchor) = anchor else { continue };
1129            anchor
1130                .validate_shape(anchor.epoch)
1131                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1132            if anchor.scope != CheckpointScope::Cluster
1133                || anchor.deployment_id != self.deployment_id
1134            {
1135                return Err(LeaseError::Invalid(format!(
1136                    "outcome floor has an invalid {name} anchor"
1137                )));
1138            }
1139        }
1140        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1141            (Some(anchor), Some(link))
1142                if anchor.epoch < self.authority_before_epoch
1143                    && link.sequence != 0
1144                    && link.epoch == anchor.epoch
1145                    && link.checkpoint_id == anchor.checkpoint_id => {}
1146            (None, None) => {}
1147            _ => {
1148                return Err(LeaseError::Invalid(
1149                    "outcome floor terminal anchor does not match its exact authority link".into(),
1150                ));
1151            }
1152        }
1153        match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
1154            (Some(anchor), Some(link))
1155                if self.artifact_before_epoch != 0
1156                    && anchor.is_commit()
1157                    && anchor.epoch < self.artifact_before_epoch
1158                    && link.sequence != 0
1159                    && link.epoch == anchor.epoch
1160                    && link.checkpoint_id == anchor.checkpoint_id => {}
1161            (None, None) => {}
1162            _ => {
1163                return Err(LeaseError::Invalid(
1164                    "outcome floor committed anchor does not match its exact Commit link".into(),
1165                ));
1166            }
1167        }
1168        if let (Some(committed), Some(terminal)) = (
1169            self.committed_anchor.as_ref(),
1170            self.terminal_anchor.as_ref(),
1171        ) {
1172            let ordered = if committed.epoch == terminal.epoch {
1173                committed == terminal
1174            } else {
1175                committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
1176            };
1177            if !ordered {
1178                return Err(LeaseError::Invalid(
1179                    "outcome floor anchors are not monotonically ordered".into(),
1180                ));
1181            }
1182            let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
1183                (Some(committed_link), Some(terminal_link)) if committed == terminal => {
1184                    committed_link == terminal_link
1185                }
1186                (Some(committed_link), Some(terminal_link)) => {
1187                    committed_link.sequence < terminal_link.sequence
1188                }
1189                _ => false,
1190            };
1191            if !links_ordered {
1192                return Err(LeaseError::Invalid(
1193                    "outcome floor anchor links are not ordered with their exact outcomes".into(),
1194                ));
1195            }
1196        } else if self.committed_anchor.is_some() {
1197            return Err(LeaseError::Invalid(
1198                "outcome floor has a committed anchor without terminal continuity".into(),
1199            ));
1200        }
1201        Ok(())
1202    }
1203}
1204
1205impl AuthorityAssignmentDecisionFloor {
1206    fn validate(&self) -> Result<(), LeaseError> {
1207        if self.before_target_version == 0 {
1208            return Err(LeaseError::Invalid(
1209                "assignment decision floor requires a nonzero target version".into(),
1210            ));
1211        }
1212        match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1213            (Some(anchor), Some(link)) => {
1214                anchor.validate()?;
1215                if link.sequence == 0
1216                    || link.target_version != anchor.target_version()
1217                    || anchor.target_version() >= self.before_target_version
1218                {
1219                    return Err(LeaseError::Invalid(
1220                        "assignment decision floor anchor does not match its exact authority link"
1221                            .into(),
1222                    ));
1223                }
1224            }
1225            (None, None) => {}
1226            _ => {
1227                return Err(LeaseError::Invalid(
1228                    "assignment decision floor has an incomplete terminal anchor".into(),
1229                ));
1230            }
1231        }
1232        Ok(())
1233    }
1234}
1235
1236impl LeaderAuthorityRecord {
1237    fn initial(lease: LeaderLease) -> Self {
1238        Self {
1239            version: AUTHORITY_RECORD_VERSION,
1240            lease,
1241            checkpoint_outcome: None,
1242            previous_outcome: None,
1243            outcome_head: None,
1244            previous_commit: None,
1245            commit_head: None,
1246            outcome_floor: None,
1247            artifact_cleanup: None,
1248            subscription_cleanup_commit: None,
1249            active_checkpoint_artifacts: None,
1250            active_checkpoint_artifact_leader_proof: None,
1251            assignment_decision: None,
1252            previous_assignment_decision: None,
1253            assignment_decision_head: None,
1254            assignment_decision_floor: None,
1255            assignment_handoff_pin: None,
1256            recovery_fault_revision: 0,
1257            recovery_fault_slots: Vec::new(),
1258            recovery_release_commit: None,
1259            recovery_release_head: None,
1260        }
1261    }
1262
1263    fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1264        Self {
1265            version: AUTHORITY_RECORD_VERSION,
1266            lease,
1267            checkpoint_outcome: None,
1268            previous_outcome: None,
1269            outcome_head: self.outcome_head,
1270            previous_commit: None,
1271            commit_head: self.commit_head,
1272            outcome_floor: self.outcome_floor.clone(),
1273            artifact_cleanup: self.artifact_cleanup.clone(),
1274            subscription_cleanup_commit: self.subscription_cleanup_commit.clone(),
1275            active_checkpoint_artifacts: self.active_checkpoint_artifacts.clone(),
1276            active_checkpoint_artifact_leader_proof: self
1277                .active_checkpoint_artifact_leader_proof
1278                .clone(),
1279            assignment_decision: None,
1280            previous_assignment_decision: None,
1281            assignment_decision_head: self.assignment_decision_head,
1282            assignment_decision_floor: self.assignment_decision_floor.clone(),
1283            assignment_handoff_pin: self.assignment_handoff_pin.clone(),
1284            recovery_fault_revision: self.recovery_fault_revision,
1285            recovery_fault_slots: self.recovery_fault_slots.clone(),
1286            recovery_release_commit: None,
1287            recovery_release_head: self.recovery_release_head.clone(),
1288        }
1289    }
1290
1291    fn validate(&self) -> Result<(), LeaseError> {
1292        if self.version != AUTHORITY_RECORD_VERSION {
1293            return Err(LeaseError::Invalid(format!(
1294                "authority record version {} is unsupported",
1295                self.version
1296            )));
1297        }
1298        self.lease.validate()?;
1299        for link in [
1300            self.previous_outcome,
1301            self.outcome_head,
1302            self.previous_commit,
1303            self.commit_head,
1304        ]
1305        .into_iter()
1306        .flatten()
1307        {
1308            link.validate()?;
1309        }
1310        self.subscription_cleanup_commit
1311            .iter()
1312            .try_for_each(subscription_replay::SubscriptionCleanupCommit::validate)?;
1313        self.validate_recovery_fault_inventory()?;
1314        self.validate_checkpoint_outcome_chain()?;
1315        self.validate_outcome_floor()?;
1316        self.validate_artifact_cleanup()?;
1317        self.validate_checkpoint_artifact_inventory()?;
1318        self.validate_assignment_decision_chain()?;
1319        self.validate_assignment_decision_floor()?;
1320        self.validate_assignment_handoff_pin()?;
1321        self.validate_recovery_release()?;
1322        Ok(())
1323    }
1324
1325    fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1326        if self.recovery_fault_slots.windows(2).any(|pair| {
1327            pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1328        }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1329            || (self.recovery_fault_revision != 0
1330                && self.recovery_fault_slots.is_empty()
1331                && self
1332                    .recovery_release_head
1333                    .as_ref()
1334                    .map(|head| head.sequence)
1335                    != Some(self.recovery_fault_revision))
1336            || self.recovery_fault_revision > self.lease.seq
1337            || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1338        {
1339            return Err(LeaseError::Invalid(
1340                "leader authority recovery-fault inventory is not canonical".into(),
1341            ));
1342        }
1343        let mut fault_sequences = BTreeSet::new();
1344        for slot in &self.recovery_fault_slots {
1345            slot.validate()?;
1346            if slot.fault_sequence > self.recovery_fault_revision
1347                || slot.fault_sequence > self.lease.seq
1348                || !fault_sequences.insert(slot.fault_sequence)
1349            {
1350                return Err(LeaseError::Invalid(
1351                    "leader authority recovery-fault sequences are not canonical".into(),
1352                ));
1353            }
1354        }
1355        if [
1356            self.checkpoint_outcome.is_some(),
1357            self.assignment_decision.is_some(),
1358            self.recovery_release_commit.is_some(),
1359        ]
1360        .into_iter()
1361        .filter(|present| *present)
1362        .count()
1363            > 1
1364        {
1365            return Err(LeaseError::Invalid(
1366                "one authority sequence cannot admit two terminal domains".into(),
1367            ));
1368        }
1369        Ok(())
1370    }
1371
1372    fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1373        if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1374            outcome
1375                .validate_shape(outcome.epoch)
1376                .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1377            if outcome.scope != CheckpointScope::Cluster {
1378                return Err(LeaseError::Invalid(
1379                    "leader authority can only admit cluster checkpoint outcomes".into(),
1380                ));
1381            }
1382            let proof = outcome
1383                .leader_proof
1384                .as_ref()
1385                .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1386            let current_link = OutcomeLink {
1387                sequence: self.lease.seq,
1388                epoch: outcome.epoch,
1389                checkpoint_id: outcome.checkpoint_id,
1390            };
1391            if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1392                return Err(LeaseError::Invalid(
1393                    "cluster outcome is not bound to its exact authority sequence and term".into(),
1394                ));
1395            }
1396            if let Some(previous) = self.previous_outcome {
1397                if previous.sequence >= self.lease.seq
1398                    || previous.epoch >= outcome.epoch
1399                    || previous.checkpoint_id >= outcome.checkpoint_id
1400                {
1401                    return Err(LeaseError::Invalid(
1402                        "cluster outcome link does not move backward in sequence and epoch".into(),
1403                    ));
1404                }
1405            }
1406            if outcome.is_commit() {
1407                if self.commit_head != Some(current_link) {
1408                    return Err(LeaseError::Invalid(
1409                        "cluster Commit is not bound to its exact Commit-chain head".into(),
1410                    ));
1411                }
1412                if let Some(previous) = self.previous_commit {
1413                    if previous.sequence >= self.lease.seq
1414                        || previous.epoch >= outcome.epoch
1415                        || previous.checkpoint_id >= outcome.checkpoint_id
1416                    {
1417                        return Err(LeaseError::Invalid(
1418                            "cluster Commit link does not move backward in sequence and epoch"
1419                                .into(),
1420                        ));
1421                    }
1422                    if self.outcome_floor.as_ref().is_some_and(|floor| {
1423                        previous.epoch < floor.artifact_before_epoch
1424                            && Some(previous) != floor.committed_anchor_link
1425                    }) {
1426                        return Err(LeaseError::Invalid(
1427                            "cluster Commit link crosses the artifact floor without its exact anchor"
1428                                .into(),
1429                        ));
1430                    }
1431                }
1432            } else if self.previous_commit.is_some() {
1433                return Err(LeaseError::Invalid(
1434                    "cluster Abort carries a previous-Commit link".into(),
1435                ));
1436            }
1437            if let Some(floor) = &self.outcome_floor {
1438                if outcome.deployment_id != floor.deployment_id
1439                    || outcome.epoch < floor.authority_before_epoch
1440                {
1441                    return Err(LeaseError::Invalid(
1442                        "cluster outcome is below or outside its durable authority floor".into(),
1443                    ));
1444                }
1445            }
1446        } else {
1447            if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1448                return Err(LeaseError::Invalid(
1449                    "non-outcome authority record carries an outcome-chain link".into(),
1450                ));
1451            }
1452            if self.outcome_head.is_some_and(|head| {
1453                head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1454            }) {
1455                return Err(LeaseError::Invalid(
1456                    "authority outcome head is outside the durable sequence".into(),
1457                ));
1458            }
1459        }
1460        if self.commit_head.is_some_and(|head| {
1461            head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1462        }) {
1463            return Err(LeaseError::Invalid(
1464                "authority Commit head is outside the durable sequence".into(),
1465            ));
1466        }
1467        match (self.commit_head, self.outcome_head) {
1468            (Some(commit), Some(terminal)) => {
1469                let equal = commit == terminal;
1470                let strictly_older = commit.sequence < terminal.sequence
1471                    && commit.epoch < terminal.epoch
1472                    && commit.checkpoint_id < terminal.checkpoint_id;
1473                if !equal && !strictly_older {
1474                    return Err(LeaseError::Invalid(
1475                        "authority Commit head is not ordered behind the terminal head".into(),
1476                    ));
1477                }
1478                if self
1479                    .checkpoint_outcome
1480                    .as_ref()
1481                    .is_some_and(|outcome| !outcome.is_commit())
1482                    && !strictly_older
1483                {
1484                    return Err(LeaseError::Invalid(
1485                        "cluster Abort did not preserve a strictly older Commit head".into(),
1486                    ));
1487                }
1488            }
1489            (Some(_), None) => {
1490                return Err(LeaseError::Invalid(
1491                    "authority Commit head exists without terminal continuity".into(),
1492                ));
1493            }
1494            (None, _) => {}
1495        }
1496        Ok(())
1497    }
1498
1499    fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1500        if let Some(floor) = self.outcome_floor.as_ref() {
1501            floor.validate()?;
1502            if floor
1503                .terminal_anchor_link
1504                .is_some_and(|link| link.sequence >= self.lease.seq)
1505            {
1506                return Err(LeaseError::Invalid(
1507                    "outcome floor anchor is outside the authority sequence".into(),
1508                ));
1509            }
1510            if floor
1511                .committed_anchor_link
1512                .is_some_and(|link| link.sequence >= self.lease.seq)
1513            {
1514                return Err(LeaseError::Invalid(
1515                    "outcome floor Commit anchor is outside the authority sequence".into(),
1516                ));
1517            }
1518            if let Some(head) = self.outcome_head {
1519                if head.epoch < floor.authority_before_epoch
1520                    && Some(head) != floor.terminal_anchor_link
1521                {
1522                    return Err(LeaseError::Invalid(
1523                        "authority outcome head does not meet its durable compaction floor".into(),
1524                    ));
1525                }
1526            } else if floor.terminal_anchor_link.is_some() {
1527                return Err(LeaseError::Invalid(
1528                    "outcome floor anchor is disconnected from the authority head".into(),
1529                ));
1530            }
1531            if let Some(head) = self.commit_head {
1532                if head.epoch < floor.artifact_before_epoch
1533                    && Some(head) != floor.committed_anchor_link
1534                {
1535                    return Err(LeaseError::Invalid(
1536                        "authority Commit head does not meet its durable artifact floor".into(),
1537                    ));
1538                }
1539            } else if floor.committed_anchor_link.is_some() {
1540                return Err(LeaseError::Invalid(
1541                    "outcome floor Commit anchor is disconnected from the Commit head".into(),
1542                ));
1543            }
1544        }
1545        Ok(())
1546    }
1547
1548    fn validate_artifact_cleanup(&self) -> Result<(), LeaseError> {
1549        let Some(cursor) = self.artifact_cleanup.as_ref() else {
1550            return Ok(());
1551        };
1552        cursor.validate()?;
1553        let floor = self.outcome_floor.as_ref().ok_or_else(|| {
1554            LeaseError::Invalid(
1555                "cluster artifact cleanup exists without an outcome retention floor".into(),
1556            )
1557        })?;
1558        if floor.artifact_before_epoch != cursor.protected.epoch
1559            || self
1560                .commit_head
1561                .is_none_or(|head| head.epoch < cursor.protected.epoch)
1562        {
1563            return Err(LeaseError::Invalid(
1564                "cluster artifact cleanup is not bound to its protected retention cut".into(),
1565            ));
1566        }
1567        Ok(())
1568    }
1569
1570    fn validate_checkpoint_artifact_inventory(&self) -> Result<(), LeaseError> {
1571        let (Some(inventory), Some(admitting_proof)) = (
1572            self.active_checkpoint_artifacts.as_ref(),
1573            self.active_checkpoint_artifact_leader_proof.as_ref(),
1574        ) else {
1575            if self.active_checkpoint_artifacts.is_some()
1576                || self.active_checkpoint_artifact_leader_proof.is_some()
1577            {
1578                return Err(LeaseError::Invalid(
1579                    "cluster checkpoint artifact inventory has incomplete leader authority".into(),
1580                ));
1581            }
1582            return Ok(());
1583        };
1584
1585        inventory.validate().map_err(|error| {
1586            LeaseError::Invalid(format!("cluster checkpoint artifact inventory: {error}"))
1587        })?;
1588        let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
1589            LeaseError::Invalid(
1590                "cluster checkpoint artifact inventory has no assignment fence".into(),
1591            )
1592        })?;
1593        if !admitting_proof.is_canonical()
1594            || assignment_fence.participant_incarnation(admitting_proof.owner.node_id)
1595                != Some(admitting_proof.owner.boot_id)
1596        {
1597            return Err(LeaseError::Invalid(
1598                "cluster checkpoint artifact inventory has no canonical admitting leader".into(),
1599            ));
1600        }
1601        if self
1602            .outcome_floor
1603            .as_ref()
1604            .is_some_and(|floor| floor.deployment_id != inventory.deployment_id)
1605        {
1606            return Err(LeaseError::Invalid(
1607                "cluster checkpoint artifact inventory belongs to a foreign deployment".into(),
1608            ));
1609        }
1610        if let Some(head) = self.outcome_head {
1611            let attempt = crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
1612            if matches!(
1613                inventory.attempt.relation_to(attempt),
1614                crate::checkpoint::CheckpointAttemptRelation::Older
1615                    | crate::checkpoint::CheckpointAttemptRelation::Conflict
1616            ) {
1617                return Err(LeaseError::Invalid(
1618                    "cluster checkpoint artifact inventory is behind the terminal outcome head"
1619                        .into(),
1620                ));
1621            }
1622        }
1623        if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1624            let exact = outcome.deployment_id == inventory.deployment_id
1625                && outcome.epoch == inventory.attempt.epoch
1626                && outcome.checkpoint_id == inventory.attempt.checkpoint_id
1627                && outcome.assignment_fence.as_ref() == inventory.assignment_fence.as_ref();
1628            if outcome.is_commit() || !exact {
1629                return Err(LeaseError::Invalid(
1630                    "terminal cluster outcome has inconsistent checkpoint artifact inventory"
1631                        .into(),
1632                ));
1633            }
1634        }
1635        Ok(())
1636    }
1637
1638    fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1639        if let Some(decision) = self.assignment_decision.as_ref() {
1640            decision.validate()?;
1641            if self
1642                .assignment_decision_floor
1643                .as_ref()
1644                .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1645            {
1646                return Err(LeaseError::Invalid(
1647                    "assignment decision is below its durable authority floor".into(),
1648                ));
1649            }
1650            if !self.lease.matches_proof(decision.leader_proof())
1651                || self.assignment_decision_head
1652                    != Some(AssignmentDecisionLink {
1653                        sequence: self.lease.seq,
1654                        target_version: decision.target_version(),
1655                    })
1656            {
1657                return Err(LeaseError::Invalid(
1658                    "assignment decision is not bound to its exact authority sequence and term"
1659                        .into(),
1660                ));
1661            }
1662            if let Some(previous) = self.previous_assignment_decision {
1663                if previous.sequence >= self.lease.seq
1664                    || previous.target_version >= decision.target_version()
1665                {
1666                    return Err(LeaseError::Invalid(
1667                        "assignment decision link does not move backward".into(),
1668                    ));
1669                }
1670            }
1671        } else {
1672            if self.previous_assignment_decision.is_some() {
1673                return Err(LeaseError::Invalid(
1674                    "non-decision authority record carries a previous assignment-decision link"
1675                        .into(),
1676                ));
1677            }
1678            if self
1679                .assignment_decision_head
1680                .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1681            {
1682                return Err(LeaseError::Invalid(
1683                    "authority assignment-decision head is outside the durable sequence".into(),
1684                ));
1685            }
1686        }
1687        Ok(())
1688    }
1689
1690    fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1691        if let Some(floor) = self.assignment_decision_floor.as_ref() {
1692            floor.validate()?;
1693            if floor
1694                .terminal_anchor_link
1695                .is_some_and(|link| link.sequence >= self.lease.seq)
1696            {
1697                return Err(LeaseError::Invalid(
1698                    "assignment decision floor anchor is outside the authority sequence".into(),
1699                ));
1700            }
1701            if let Some(head) = self.assignment_decision_head {
1702                if head.target_version < floor.before_target_version
1703                    && Some(head) != floor.terminal_anchor_link
1704                {
1705                    return Err(LeaseError::Invalid(
1706                        "authority assignment-decision head does not meet its durable floor".into(),
1707                    ));
1708                }
1709            } else if floor.terminal_anchor_link.is_some() {
1710                return Err(LeaseError::Invalid(
1711                    "assignment decision floor anchor is disconnected from the authority head"
1712                        .into(),
1713                ));
1714            }
1715        }
1716        Ok(())
1717    }
1718
1719    fn validate_assignment_handoff_pin(&self) -> Result<(), LeaseError> {
1720        let Some(pin) = self.assignment_handoff_pin.as_ref() else {
1721            return Ok(());
1722        };
1723        pin.validate()?;
1724        if self.commit_head.is_none_or(|head| {
1725            head.epoch != pin.checkpoint.epoch || head.checkpoint_id != pin.checkpoint.checkpoint_id
1726        }) || self
1727            .outcome_floor
1728            .as_ref()
1729            .is_some_and(|floor| floor.artifact_before_epoch > pin.checkpoint.epoch)
1730            || self
1731                .artifact_cleanup
1732                .as_ref()
1733                .is_some_and(|cursor| cursor.protected.epoch > pin.checkpoint.epoch)
1734        {
1735            return Err(LeaseError::Invalid(
1736                "assignment handoff pin is outside its live checkpoint retention range".into(),
1737            ));
1738        }
1739        Ok(())
1740    }
1741
1742    fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1743        match self.recovery_release_commit.as_ref() {
1744            Some(commit) => {
1745                commit.validate()?;
1746                let expected = RecoveryReleaseLink {
1747                    sequence: self.lease.seq,
1748                    terminal: commit.terminal.clone(),
1749                };
1750                if !self.lease.matches_proof(&commit.leader_proof)
1751                    || self.recovery_release_head.as_ref() != Some(&expected)
1752                    || self.recovery_fault_revision != self.lease.seq
1753                    || self.recovery_fault_slots.iter().any(|slot| {
1754                        slot.active || slot.disposition == RecoveryFaultDisposition::Terminal
1755                    })
1756                {
1757                    return Err(LeaseError::Invalid(
1758                        "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1759                            .into(),
1760                    ));
1761                }
1762            }
1763            None => {
1764                if let Some(head) = self.recovery_release_head.as_ref() {
1765                    head.validate()?;
1766                    if head.sequence >= self.lease.seq {
1767                        return Err(LeaseError::Invalid(
1768                            "recovery release head is outside the durable authority sequence"
1769                                .into(),
1770                        ));
1771                    }
1772                    if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1773                        && self.recovery_fault_revision != head.sequence
1774                    {
1775                        return Err(LeaseError::Invalid(
1776                            "settled recovery fault inventory is not bound to its release head"
1777                                .into(),
1778                        ));
1779                    }
1780                }
1781            }
1782        }
1783        Ok(())
1784    }
1785}
1786
1787/// Result of an acquisition or renewal attempt.
1788#[derive(Debug, Clone, PartialEq, Eq)]
1789pub enum LeaseOutcome {
1790    /// The caller's exact process incarnation owns the returned record.
1791    Acquired(LeaderLease),
1792    /// A rival exact process incarnation owns the returned record.
1793    Held(LeaderLease),
1794}
1795
1796enum AuthorityCreateOutcome {
1797    Created,
1798    ExistingIdentical,
1799    Contended(Box<LeaderAuthorityRecord>),
1800}
1801
1802#[derive(Clone, Copy)]
1803enum SameOwnerToken {
1804    #[cfg(test)]
1805    Preserve,
1806    Rotate,
1807    Exact(u64),
1808}
1809
1810/// Candidate-local proof that one rival liveness identity remained current for a full TTL.
1811#[derive(Debug)]
1812pub struct LeaderLeaseObservation {
1813    lease: LeaderLease,
1814    started: Instant,
1815}
1816
1817/// Coalescing-safe local candidacy state. A generation change means a published grant was lost
1818/// and must never reuse its fencing token, even when the current state is eligible again.
1819#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1820pub struct LeaderCandidacy {
1821    eligible: bool,
1822    generation: u64,
1823}
1824
1825impl LeaderCandidacy {
1826    pub(crate) const fn initial(eligible: bool) -> Self {
1827        Self {
1828            eligible,
1829            generation: 1,
1830        }
1831    }
1832
1833    pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1834        let generation = if self.eligible && !eligible {
1835            self.generation.checked_add(1)?
1836        } else {
1837            self.generation
1838        };
1839        Some(Self {
1840            eligible,
1841            generation,
1842        })
1843    }
1844
1845    pub(crate) const fn terminal() -> Self {
1846        Self {
1847            eligible: false,
1848            generation: u64::MAX,
1849        }
1850    }
1851
1852    /// Whether this process is currently eligible to contend for the leader lease.
1853    #[must_use]
1854    pub const fn is_eligible(self) -> bool {
1855        self.eligible
1856    }
1857}
1858
1859/// Leader lease storage or validation failure.
1860#[derive(Debug, thiserror::Error)]
1861pub enum LeaseError {
1862    /// Underlying object-store failure.
1863    #[error("object store I/O: {0}")]
1864    Io(String),
1865    /// Malformed configuration, owner, or durable record.
1866    #[error("invalid leader lease: {0}")]
1867    Invalid(String),
1868    /// A newer durable authority term superseded an exact renewal.
1869    #[error("leader lease fenced: {0}")]
1870    Fenced(String),
1871    /// JSON encoding or decoding failure.
1872    #[error("JSON: {0}")]
1873    Json(#[from] serde_json::Error),
1874}
1875
1876/// Failure while using the leader sequence as the cluster checkpoint-decision authority.
1877#[derive(Debug, thiserror::Error)]
1878pub enum ClusterCheckpointAuthorityError {
1879    /// Cluster runtime did not wire the durable leader authority.
1880    #[error("cluster checkpoint authority is not installed")]
1881    NotConfigured,
1882    /// Shared append-only authority failed.
1883    #[error("leader authority: {0}")]
1884    Authority(#[from] LeaseError),
1885    /// Checkpoint metadata or content-addressed recovery state was invalid.
1886    #[error("checkpoint decision: {0}")]
1887    Decision(#[from] DecisionError),
1888    /// The supplied proof no longer names the exact durable leader term.
1889    #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1890    Fenced,
1891}
1892
1893/// Exact continuity retained when old cluster checkpoint outcomes are compacted.
1894#[derive(Debug, Clone, PartialEq, Eq)]
1895pub struct ClusterOutcomeRetentionBoundary {
1896    /// Checkpoint artifacts below this epoch are eligible for garbage collection.
1897    pub artifact_before_epoch: u64,
1898    /// Terminal outcome links below this epoch have been compacted.
1899    pub terminal_before_epoch: u64,
1900    /// Greatest committed outcome below the artifact-retention horizon.
1901    pub committed_anchor: Option<CheckpointOutcome>,
1902    /// Greatest terminal outcome compacted from the authority chain, including aborts.
1903    pub terminal_anchor: Option<CheckpointOutcome>,
1904}
1905
1906impl ClusterOutcomeRetentionBoundary {
1907    fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1908        floor.map_or(
1909            Self {
1910                artifact_before_epoch: 0,
1911                terminal_before_epoch: 0,
1912                committed_anchor: None,
1913                terminal_anchor: None,
1914            },
1915            |floor| Self {
1916                artifact_before_epoch: floor.artifact_before_epoch,
1917                terminal_before_epoch: floor.authority_before_epoch,
1918                committed_anchor: floor.committed_anchor.clone(),
1919                terminal_anchor: floor.terminal_anchor.clone(),
1920            },
1921        )
1922    }
1923}
1924
1925/// Live cluster outcomes and their retention horizons from one audited authority head.
1926#[derive(Debug, Clone, PartialEq, Eq)]
1927pub struct ClusterOutcomeInventory {
1928    /// Outcomes at or above the artifact-retention horizon, in ascending epoch order.
1929    pub outcomes: Vec<CheckpointOutcome>,
1930    /// Artifact and terminal-history horizons paired with `outcomes`.
1931    pub retention_boundary: ClusterOutcomeRetentionBoundary,
1932}
1933
1934/// Append-only object-store authority for the cluster leader.
1935pub struct LeaderLeaseStore {
1936    store: Arc<dyn ObjectStore>,
1937    ttl_ms: i64,
1938    prune_running: Arc<AtomicBool>,
1939    outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
1940    outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
1941}
1942
1943#[derive(Clone, PartialEq, Eq)]
1944struct ClusterOutcomeAuditKey {
1945    terminal_head: Option<OutcomeLink>,
1946    commit_head: Option<OutcomeLink>,
1947    floor: Option<AuthorityOutcomeFloor>,
1948}
1949
1950struct ClusterOutcomeAuditCache {
1951    key: ClusterOutcomeAuditKey,
1952    authority_sequence: u64,
1953    snapshot: ClusterOutcomeAuditSnapshot,
1954}
1955
1956#[derive(Clone)]
1957struct ClusterOutcomeAuditSnapshot {
1958    outcomes: Arc<[CheckpointOutcome]>,
1959    terminal_links: Arc<[OutcomeLink]>,
1960    commit_links: Arc<[OutcomeLink]>,
1961}
1962
1963struct ClusterOutcomeAuditFlight {
1964    key: ClusterOutcomeAuditKey,
1965    gate: Weak<tokio::sync::Mutex<()>>,
1966}
1967
1968struct PruneLatchGuard(Arc<AtomicBool>);
1969
1970impl Drop for PruneLatchGuard {
1971    fn drop(&mut self) {
1972        self.0.store(false, Ordering::Release);
1973    }
1974}
1975
1976impl std::fmt::Debug for LeaderLeaseStore {
1977    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1978        formatter
1979            .debug_struct("LeaderLeaseStore")
1980            .field("ttl_ms", &self.ttl_ms)
1981            .finish_non_exhaustive()
1982    }
1983}
1984
1985impl LeaderLeaseStore {
1986    /// Create a leader lease authority.
1987    ///
1988    /// The store must provide linearizable `PutMode::Create`/`Update` and GET `ETag` or version
1989    /// metadata; unsupported conditional updates fail closed.
1990    #[must_use]
1991    pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
1992        Self {
1993            store,
1994            ttl_ms,
1995            prune_running: Arc::new(AtomicBool::new(false)),
1996            outcome_audit_cache: Mutex::new(None),
1997            outcome_audit_flights: Mutex::new(Vec::new()),
1998        }
1999    }
2000
2001    /// Verify the conditional-write contract required by the mutable authority head.
2002    ///
2003    /// Validation and cleanup each receive the supplied timeout. The probe uses a unique path and
2004    /// cleanup is attempted after every validation outcome.
2005    ///
2006    /// # Errors
2007    /// Returns an error when the store does not enforce native conditional writes, omits update
2008    /// metadata, times out, or cannot remove the probe.
2009    pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
2010        probe_object_store_conditional_update(
2011            self.store.as_ref(),
2012            STORE_CONTRACT_PROBE_PREFIX,
2013            timeout,
2014        )
2015        .await
2016        .map_err(|error| match error {
2017            CheckpointStoreError::Invalid(message) => LeaseError::Invalid(message),
2018            error => LeaseError::Io(error.to_string()),
2019        })
2020    }
2021
2022    fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
2023        RecoveryFaultInventory {
2024            revision: record.recovery_fault_revision,
2025            faults: record
2026                .recovery_fault_slots
2027                .iter()
2028                .filter(|slot| slot.active)
2029                .map(AuthorityRecoveryFaultSlot::fault)
2030                .collect(),
2031        }
2032    }
2033
2034    pub(crate) async fn recovery_fault_inventory(
2035        &self,
2036    ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
2037        let record = self
2038            .load_record()
2039            .await?
2040            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2041        Ok(Self::recovery_fault_inventory_from(&record))
2042    }
2043
2044    #[cfg(test)]
2045    pub(crate) async fn record_recovery_fault(
2046        &self,
2047        publisher: RecoveryFaultPublisher,
2048        request_sequence: u64,
2049    ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2050        self.record_recovery_fault_with_disposition(
2051            publisher,
2052            request_sequence,
2053            RecoveryFaultDisposition::Recoverable,
2054        )
2055        .await
2056    }
2057
2058    pub(crate) async fn record_recovery_fault_with_disposition(
2059        &self,
2060        publisher: RecoveryFaultPublisher,
2061        request_sequence: u64,
2062        disposition: RecoveryFaultDisposition,
2063    ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2064        publisher.validate().map_err(LeaseError::Invalid)?;
2065        if request_sequence == 0 {
2066            return Err(LeaseError::Invalid(
2067                "recovery fault request sequence must be nonzero".into(),
2068            )
2069            .into());
2070        }
2071
2072        loop {
2073            let head = self
2074                .load_published_authority_head()
2075                .await?
2076                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2077            let current = &head.record;
2078            let node_id = publisher.participant.node_id;
2079            let slot_index = current
2080                .recovery_fault_slots
2081                .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
2082            let (insert_at, replace) = match slot_index {
2083                Ok(index) => {
2084                    let slot = &current.recovery_fault_slots[index];
2085                    if slot.matches_request(publisher, request_sequence, disposition) {
2086                        return Ok(if slot.active {
2087                            RecordRecoveryFaultResult::Active
2088                        } else {
2089                            RecordRecoveryFaultResult::AlreadyCleared
2090                        });
2091                    }
2092                    // A terminal slot is an operator-owned cluster fence. Neither a later request
2093                    // nor a replacement process may silently downgrade or tombstone it.
2094                    if slot.disposition == RecoveryFaultDisposition::Terminal {
2095                        return Ok(RecordRecoveryFaultResult::TerminalFenceActive);
2096                    }
2097                    if publisher.process_term < slot.publisher.process_term {
2098                        return Ok(RecordRecoveryFaultResult::Superseded);
2099                    }
2100                    if publisher.process_term == slot.publisher.process_term
2101                        && publisher.participant.boot_incarnation
2102                            != slot.publisher.participant.boot_incarnation
2103                    {
2104                        return Err(LeaseError::Invalid(format!(
2105                            "stable node {node_id} has two recovery fault publishers for process term {}",
2106                            publisher.process_term
2107                        ))
2108                        .into());
2109                    }
2110                    if publisher.process_term == slot.publisher.process_term
2111                        && request_sequence < slot.request_sequence
2112                    {
2113                        return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
2114                    }
2115                    (index, true)
2116                }
2117                Err(index) => (index, false),
2118            };
2119
2120            let sequence =
2121                current.lease.seq.checked_add(1).ok_or_else(|| {
2122                    LeaseError::Invalid("leader authority sequence exhausted".into())
2123                })?;
2124            let mut lease = current.lease.clone();
2125            lease.seq = sequence;
2126            let mut candidate = current.preserve_with_lease(lease);
2127            let slot = AuthorityRecoveryFaultSlot {
2128                publisher,
2129                request_sequence,
2130                // Authority sequence is globally monotonic even after an obsolete tombstone is
2131                // compacted. A delayed retry can therefore become a conservative fresh fault,
2132                // but can never alias a sequence already remembered by a recovery monitor.
2133                fault_sequence: sequence,
2134                disposition,
2135                active: true,
2136            };
2137            if replace {
2138                candidate.recovery_fault_slots[insert_at] = slot;
2139            } else {
2140                candidate.recovery_fault_slots.insert(insert_at, slot);
2141            }
2142            candidate.recovery_fault_revision = sequence;
2143            candidate.validate()?;
2144
2145            match self
2146                .create_authority_record(Some(&head), &candidate)
2147                .await?
2148            {
2149                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2150                    return Ok(RecordRecoveryFaultResult::Active);
2151                }
2152                AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
2153            }
2154        }
2155    }
2156
2157    pub(crate) async fn authorize_recovery_release(
2158        &self,
2159        clearer: RecoveryFaultPublisher,
2160        terminal: &RecoveryAnnouncement,
2161    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2162        clearer.validate().map_err(LeaseError::Invalid)?;
2163        let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2164        let reporter = NodeId(clearer.participant.node_id);
2165        let expected = terminal.round.fault_sequence(reporter);
2166        let current = self
2167            .load_record()
2168            .await?
2169            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2170        let Some(release_head) = current.recovery_release_head.as_ref() else {
2171            return Ok(false);
2172        };
2173        if release_head.terminal != terminal_reference
2174            || current.recovery_fault_revision != release_head.sequence
2175        {
2176            return Ok(false);
2177        }
2178        if current
2179            .recovery_fault_slots
2180            .iter()
2181            .any(|slot| slot.active || slot.disposition == RecoveryFaultDisposition::Terminal)
2182        {
2183            return Ok(false);
2184        }
2185        let slot_index = current
2186            .recovery_fault_slots
2187            .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2188        let Some(expected_sequence) = expected else {
2189            return Ok(match slot_index {
2190                Ok(index) => !current.recovery_fault_slots[index].active,
2191                Err(_) => true,
2192            });
2193        };
2194        let Ok(index) = slot_index else {
2195            return Ok(false);
2196        };
2197        let slot = &current.recovery_fault_slots[index];
2198        Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2199    }
2200
2201    pub(crate) async fn stage_recovery_release_terminal(
2202        &self,
2203        terminal: &RecoveryAnnouncement,
2204    ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2205        let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2206        let path = recovery_release_terminal_path(&reference);
2207        let put_error = match self
2208            .store
2209            .put_opts(
2210                &path,
2211                PutPayload::from(encoded),
2212                PutOptions {
2213                    mode: PutMode::Create,
2214                    ..PutOptions::default()
2215                },
2216            )
2217            .await
2218        {
2219            Ok(_)
2220            | Err(
2221                object_store::Error::AlreadyExists { .. }
2222                | object_store::Error::Precondition { .. },
2223            ) => None,
2224            Err(error) => Some(error),
2225        };
2226        match self.load_recovery_release_terminal(&reference).await {
2227            Ok(stored) if stored == *terminal => Ok(reference),
2228            Ok(_) => Err(LeaseError::Invalid(format!(
2229                "recovery release terminal '{}' differs from the proposed content",
2230                reference.sha256
2231            ))
2232            .into()),
2233            Err(reconcile_error) => {
2234                if let Some(put_error) = put_error {
2235                    Err(LeaseError::Io(format!(
2236                        "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2237                    ))
2238                    .into())
2239                } else {
2240                    Err(reconcile_error)
2241                }
2242            }
2243        }
2244    }
2245
2246    pub(crate) async fn load_recovery_release_terminal(
2247        &self,
2248        reference: &RecoveryReleaseTerminalRef,
2249    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2250        reference.validate()?;
2251        let result = match self
2252            .store
2253            .get(&recovery_release_terminal_path(reference))
2254            .await
2255        {
2256            Ok(result) => result,
2257            Err(object_store::Error::NotFound { .. }) => {
2258                return Err(LeaseError::Invalid(format!(
2259                    "recovery release terminal '{}' is missing",
2260                    reference.sha256
2261                ))
2262                .into());
2263            }
2264            Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2265        };
2266        if result.meta.size != reference.encoded_len {
2267            return Err(LeaseError::Invalid(format!(
2268                "recovery release terminal '{}' is {} bytes, expected {}",
2269                reference.sha256, result.meta.size, reference.encoded_len
2270            ))
2271            .into());
2272        }
2273        let bytes = result
2274            .bytes()
2275            .await
2276            .map_err(|error| LeaseError::Io(error.to_string()))?;
2277        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2278            return Err(LeaseError::Invalid(format!(
2279                "recovery release terminal '{}' payload length changed while reading",
2280                reference.sha256
2281            ))
2282            .into());
2283        }
2284        let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2285            LeaseError::Invalid(format!(
2286                "recovery release terminal '{}': {error}",
2287                reference.sha256
2288            ))
2289        })?;
2290        let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2291        if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2292            return Err(LeaseError::Invalid(format!(
2293                "recovery release terminal '{}' does not match its content-addressed reference",
2294                reference.sha256
2295            ))
2296            .into());
2297        }
2298        Ok(terminal)
2299    }
2300
2301    pub(crate) async fn record_recovery_release_commit(
2302        &self,
2303        proof: &LeaderProof,
2304        reference: RecoveryReleaseTerminalRef,
2305    ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2306        reference.validate()?;
2307        let terminal = self.load_recovery_release_terminal(&reference).await?;
2308        if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2309            return Err(ClusterCheckpointAuthorityError::Fenced);
2310        }
2311
2312        loop {
2313            let published = self
2314                .load_published_authority_head()
2315                .await?
2316                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2317            let current = &published.record;
2318            if let Some(winner) = current.recovery_release_head.as_ref() {
2319                if winner.terminal == reference
2320                    || winner.terminal.generation() >= reference.generation()
2321                {
2322                    self.load_recovery_release_terminal(&winner.terminal)
2323                        .await?;
2324                    return if winner.terminal == reference {
2325                        Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2326                    } else {
2327                        Ok(RecordRecoveryReleaseCommitResult::Conflict {
2328                            winner: winner.terminal.clone(),
2329                        })
2330                    };
2331                }
2332            }
2333            if !current.lease.matches_proof(proof) {
2334                return Err(ClusterCheckpointAuthorityError::Fenced);
2335            }
2336            let fault_inventory = Self::recovery_fault_inventory_from(current);
2337            if fault_inventory.revision != terminal.round.fault_revision()
2338                || fault_inventory.faults != terminal.round.faults
2339                || fault_inventory
2340                    .faults
2341                    .iter()
2342                    .copied()
2343                    .any(RecoveryFault::is_terminal)
2344            {
2345                return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2346            }
2347
2348            let base_sequence = current.lease.seq;
2349            let sequence = base_sequence
2350                .checked_add(1)
2351                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2352            let mut candidate = current.preserve_with_lease(LeaderLease {
2353                seq: sequence,
2354                renewal_sequence: current.lease.renewal_sequence,
2355                token: current.lease.token,
2356                owner: current.lease.owner.clone(),
2357                expires_at_ms: current.lease.expires_at_ms,
2358                catalog_manifest: current.lease.catalog_manifest.clone(),
2359            });
2360            // Only a process frozen into the stopped roster may consume this terminal. Covered
2361            // unavailable publishers remain fenced and conservatively republish if they return.
2362            candidate.recovery_fault_slots.retain(|slot| {
2363                slot.active
2364                    && terminal
2365                        .round
2366                        .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2367                        == Some(slot.publisher.participant.boot_incarnation)
2368            });
2369            for slot in &mut candidate.recovery_fault_slots {
2370                slot.active = false;
2371            }
2372            candidate.recovery_fault_revision = sequence;
2373            candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2374                terminal: reference.clone(),
2375                leader_proof: proof.clone(),
2376            });
2377            candidate.recovery_release_head = Some(RecoveryReleaseLink {
2378                sequence,
2379                terminal: reference.clone(),
2380            });
2381            candidate.validate()?;
2382
2383            match self
2384                .create_authority_record(Some(&published), &candidate)
2385                .await?
2386            {
2387                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2388                    return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2389                }
2390                AuthorityCreateOutcome::Contended(winner_head) => {
2391                    if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2392                        if winner.terminal == reference
2393                            || winner.terminal.generation() >= reference.generation()
2394                        {
2395                            self.load_recovery_release_terminal(&winner.terminal)
2396                                .await?;
2397                            return if winner.terminal == reference {
2398                                Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2399                            } else {
2400                                Ok(RecordRecoveryReleaseCommitResult::Conflict {
2401                                    winner: winner.terminal.clone(),
2402                                })
2403                            };
2404                        }
2405                    }
2406                    if !winner_head.lease.matches_proof(proof) {
2407                        return Err(ClusterCheckpointAuthorityError::Fenced);
2408                    }
2409                    if winner_head.lease.seq <= base_sequence {
2410                        return Err(LeaseError::Invalid(
2411                            "recovery release authority contention did not advance the sequence"
2412                                .into(),
2413                        )
2414                        .into());
2415                    }
2416                    tokio::task::yield_now().await;
2417                }
2418            }
2419        }
2420    }
2421
2422    async fn recovery_release_terminal_from(
2423        &self,
2424        head: &LeaderAuthorityRecord,
2425        link: &RecoveryReleaseLink,
2426    ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2427        let admission = if link.sequence == head.lease.seq {
2428            head.clone()
2429        } else {
2430            read_authority_record(self.store.as_ref(), link.sequence)
2431                .await?
2432                .ok_or_else(|| {
2433                    LeaseError::Invalid(
2434                        "recovery release admission is missing from retained authority history"
2435                            .into(),
2436                    )
2437                })?
2438        };
2439        let commit = admission
2440            .recovery_release_commit
2441            .as_ref()
2442            .filter(|commit| {
2443                commit.terminal == link.terminal
2444                    && admission.recovery_release_head.as_ref() == Some(link)
2445            })
2446            .ok_or_else(|| {
2447                LeaseError::Invalid(
2448                    "recovery release head does not match its admitting authority record".into(),
2449                )
2450            })?;
2451        let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2452        if terminal.round.leader_proof != commit.leader_proof {
2453            return Err(LeaseError::Invalid(
2454                "recovery release terminal does not match its admitting leader proof".into(),
2455            )
2456            .into());
2457        }
2458        Ok(terminal)
2459    }
2460
2461    pub(crate) async fn recovery_admission_snapshot(
2462        &self,
2463    ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2464        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2465            let head = self
2466                .load_record()
2467                .await?
2468                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2469            let Some(link) = head.recovery_release_head.clone() else {
2470                return Ok(RecoveryAdmissionSnapshot {
2471                    committed_release: None,
2472                    fault_inventory: Self::recovery_fault_inventory_from(&head),
2473                    authority_sequence: head.lease.seq,
2474                    release_head: None,
2475                });
2476            };
2477            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2478            let rechecked = self
2479                .load_record()
2480                .await?
2481                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2482            if rechecked.lease.seq >= head.lease.seq
2483                && rechecked.recovery_release_head == head.recovery_release_head
2484                && rechecked.recovery_fault_revision == head.recovery_fault_revision
2485                && rechecked.recovery_fault_slots == head.recovery_fault_slots
2486            {
2487                return Ok(RecoveryAdmissionSnapshot {
2488                    committed_release: Some(terminal?),
2489                    fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2490                    authority_sequence: rechecked.lease.seq,
2491                    release_head: rechecked.recovery_release_head,
2492                });
2493            }
2494            tokio::task::yield_now().await;
2495        }
2496        Err(LeaseError::Io(format!(
2497            "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2498        ))
2499        .into())
2500    }
2501
2502    pub(crate) async fn recovery_admission_is_current(
2503        &self,
2504        snapshot: &RecoveryAdmissionSnapshot,
2505        leader_proof: &LeaderProof,
2506    ) -> Result<bool, ClusterCheckpointAuthorityError> {
2507        let current = self
2508            .load_record()
2509            .await?
2510            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2511        let inventory = Self::recovery_fault_inventory_from(&current);
2512        Ok(current.lease.matches_proof(leader_proof)
2513            && current.lease.seq >= snapshot.authority_sequence
2514            && current.recovery_release_head == snapshot.release_head
2515            && inventory == snapshot.fault_inventory
2516            && inventory.faults().is_empty())
2517    }
2518
2519    pub(crate) async fn latest_recovery_release_terminal(
2520        &self,
2521    ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2522        for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2523            let Some(head) = self.load_record().await? else {
2524                return Ok(None);
2525            };
2526            let Some(link) = head.recovery_release_head.clone() else {
2527                return Ok(None);
2528            };
2529            let terminal = self.recovery_release_terminal_from(&head, &link).await;
2530            let rechecked = self
2531                .load_record()
2532                .await?
2533                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2534            if rechecked.recovery_release_head.as_ref() == Some(&link) {
2535                return Ok(Some(terminal?));
2536            }
2537            tokio::task::yield_now().await;
2538        }
2539        Err(LeaseError::Io(format!(
2540            "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2541        ))
2542        .into())
2543    }
2544
2545    fn ttl(&self) -> Result<Duration, LeaseError> {
2546        let ttl = u64::try_from(self.ttl_ms)
2547            .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2548        if ttl == 0 {
2549            return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2550        }
2551        Ok(Duration::from_millis(ttl))
2552    }
2553
2554    fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2555        now_ms
2556            .checked_add(self.ttl_ms)
2557            .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2558    }
2559
2560    #[cfg(test)]
2561    async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2562        let prefix = OsPath::from(LEASE_PREFIX);
2563        let mut entries = self.store.list(Some(&prefix));
2564        let mut sequences = Vec::new();
2565        while let Some(entry) = entries.next().await {
2566            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2567            if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2568                return Err(LeaseError::Invalid(format!(
2569                    "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2570                )));
2571            }
2572            sequences.push(lease_sequence_from_path(&entry.location)?);
2573        }
2574        sequences.sort_unstable();
2575        sequences.dedup();
2576        Ok(sequences)
2577    }
2578
2579    fn schedule_history_prune(&self) {
2580        if self
2581            .prune_running
2582            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2583            .is_err()
2584        {
2585            return;
2586        }
2587        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2588            self.prune_running.store(false, Ordering::Release);
2589            return;
2590        };
2591        let store = Arc::clone(&self.store);
2592        let prune_running = Arc::clone(&self.prune_running);
2593        let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2594        runtime.spawn(async move {
2595            let _latch = PruneLatchGuard(prune_running);
2596            match tokio::time::timeout(
2597                LEADER_LEASE_PRUNE_TIMEOUT,
2598                Self::prune_history(&store, grace_ms),
2599            )
2600            .await
2601            {
2602                Ok(Ok(())) => {}
2603                Ok(Err(error)) => {
2604                    tracing::warn!(%error, "leader lease history prune failed");
2605                }
2606                Err(_) => {
2607                    tracing::warn!(
2608                        timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2609                        "leader lease history prune timed out"
2610                    );
2611                }
2612            }
2613        });
2614    }
2615
2616    async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2617        let authority = Self::new(Arc::clone(store), 1);
2618        let Some(head) = authority.load_record().await? else {
2619            return Ok(());
2620        };
2621        let head_sequence = head.lease.seq;
2622        let mut retained = BTreeSet::from([head_sequence]);
2623        if let Some(previous) = head_sequence
2624            .checked_sub(1)
2625            .filter(|sequence| *sequence != 0)
2626        {
2627            retained.insert(previous);
2628        }
2629        let floor = head
2630            .outcome_floor
2631            .as_ref()
2632            .map_or(0, |floor| floor.authority_before_epoch);
2633        let artifact_floor = head
2634            .outcome_floor
2635            .as_ref()
2636            .map_or(0, |floor| floor.artifact_before_epoch);
2637        let mut link = head.outcome_head;
2638        let mut outcome_links = 0;
2639        let mut terminal_commit_links = BTreeSet::new();
2640        let mut expected_commit_head = head.commit_head;
2641        while let Some(current) = link {
2642            if current.epoch < floor {
2643                if head
2644                    .outcome_floor
2645                    .as_ref()
2646                    .and_then(|floor| floor.terminal_anchor_link)
2647                    != Some(current)
2648                {
2649                    return Err(LeaseError::Invalid(
2650                        "outcome floor does not anchor the retained authority chain".into(),
2651                    ));
2652                }
2653                break;
2654            }
2655            if !consume_live_authority_link(&mut outcome_links) {
2656                return Err(LeaseError::Invalid(format!(
2657                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2658                )));
2659            }
2660            let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2661                .await?
2662                .filter(|record| {
2663                    record.outcome_head == Some(current)
2664                        && record
2665                            .checkpoint_outcome
2666                            .as_ref()
2667                            .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2668                            == Some((current.epoch, current.checkpoint_id))
2669                })
2670                .ok_or_else(|| {
2671                    LeaseError::Invalid("retained outcome authority chain is broken".into())
2672                })?;
2673            if outcome_record.commit_head != expected_commit_head {
2674                return Err(LeaseError::Invalid(
2675                    "retained outcome chain did not preserve the exact Commit head".into(),
2676                ));
2677            }
2678            retained.insert(current.sequence);
2679            if outcome_record
2680                .checkpoint_outcome
2681                .as_ref()
2682                .is_some_and(CheckpointOutcome::is_commit)
2683            {
2684                terminal_commit_links.insert(current);
2685                expected_commit_head = outcome_record.previous_commit;
2686            } else {
2687                expected_commit_head = outcome_record.commit_head;
2688            }
2689            link = outcome_record.previous_outcome;
2690        }
2691        let mut commit_link = head.commit_head;
2692        let mut commit_links = 0;
2693        let mut retained_commit_links = BTreeSet::new();
2694        while let Some(current) = commit_link {
2695            if current.epoch < artifact_floor {
2696                break;
2697            }
2698            if !consume_live_authority_link(&mut commit_links) {
2699                return Err(LeaseError::Invalid(format!(
2700                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2701                )));
2702            }
2703            let commit_record = read_authority_record(store.as_ref(), current.sequence)
2704                .await?
2705                .filter(|record| {
2706                    record.commit_head == Some(current)
2707                        && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2708                            outcome.is_commit()
2709                                && outcome.epoch == current.epoch
2710                                && outcome.checkpoint_id == current.checkpoint_id
2711                        })
2712                })
2713                .ok_or_else(|| {
2714                    LeaseError::Invalid("retained Commit authority chain is broken".into())
2715                })?;
2716            retained.insert(current.sequence);
2717            retained_commit_links.insert(current);
2718            commit_link = commit_record.previous_commit;
2719        }
2720        match (
2721            commit_link,
2722            head.outcome_floor
2723                .as_ref()
2724                .and_then(|floor| floor.committed_anchor_link),
2725            head.outcome_floor
2726                .as_ref()
2727                .and_then(|floor| floor.committed_anchor.as_ref()),
2728        ) {
2729            (Some(link), Some(anchor_link), Some(anchor))
2730                if link == anchor_link
2731                    && link.epoch == anchor.epoch
2732                    && link.checkpoint_id == anchor.checkpoint_id => {}
2733            (None, None, None) => {}
2734            _ => {
2735                return Err(LeaseError::Invalid(
2736                    "Commit chain does not meet the retained artifact-floor anchor".into(),
2737                ));
2738            }
2739        }
2740        let boundary_commit_head = head
2741            .outcome_floor
2742            .as_ref()
2743            .and_then(|floor| floor.terminal_anchor.as_ref())
2744            .and_then(|anchor| {
2745                retained_commit_links
2746                    .iter()
2747                    .rev()
2748                    .find(|link| link.epoch <= anchor.epoch)
2749                    .copied()
2750                    .or_else(|| {
2751                        head.outcome_floor
2752                            .as_ref()
2753                            .and_then(|floor| floor.committed_anchor_link)
2754                    })
2755            });
2756        if expected_commit_head != boundary_commit_head {
2757            return Err(LeaseError::Invalid(
2758                "retained terminal chain lost Commit continuity at its durable floor".into(),
2759            ));
2760        }
2761        if !terminal_commit_links.is_subset(&retained_commit_links) {
2762            return Err(LeaseError::Invalid(
2763                "terminal Commit records are not linked from the retained Commit chain".into(),
2764            ));
2765        }
2766        if let Some((anchor, anchor_link)) = head.outcome_floor.as_ref().and_then(|floor| {
2767            floor
2768                .terminal_anchor
2769                .as_ref()
2770                .zip(floor.terminal_anchor_link)
2771        }) {
2772            if anchor.is_commit() {
2773                let linked = if anchor.epoch >= artifact_floor {
2774                    retained_commit_links.contains(&anchor_link)
2775                } else {
2776                    head.outcome_floor.as_ref().is_some_and(|floor| {
2777                        floor.committed_anchor.as_ref() == Some(anchor)
2778                            && floor.committed_anchor_link == Some(anchor_link)
2779                    })
2780                };
2781                if !linked {
2782                    return Err(LeaseError::Invalid(
2783                        "terminal Commit anchor is not linked from the retained Commit chain"
2784                            .into(),
2785                    ));
2786                }
2787            }
2788        }
2789        let mut assignment_link = head.assignment_decision_head;
2790        let mut assignment_links = 0;
2791        while let Some(current) = assignment_link {
2792            if !consume_live_authority_link(&mut assignment_links) {
2793                return Err(LeaseError::Invalid(format!(
2794                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2795                )));
2796            }
2797            if head
2798                .assignment_decision_floor
2799                .as_ref()
2800                .is_some_and(|floor| current.target_version < floor.before_target_version)
2801            {
2802                if head
2803                    .assignment_decision_floor
2804                    .as_ref()
2805                    .and_then(|floor| floor.terminal_anchor_link)
2806                    != Some(current)
2807                {
2808                    return Err(LeaseError::Invalid(
2809                        "assignment-decision floor does not anchor the retained authority chain"
2810                            .into(),
2811                    ));
2812                }
2813                break;
2814            }
2815            let decision_record = read_authority_record(store.as_ref(), current.sequence)
2816                .await?
2817                .filter(|record| {
2818                    record.assignment_decision_head == Some(current)
2819                        && record
2820                            .assignment_decision
2821                            .as_ref()
2822                            .map(AuthorityAssignmentDecision::target_version)
2823                            == Some(current.target_version)
2824                })
2825                .ok_or_else(|| {
2826                    LeaseError::Invalid(
2827                        "retained assignment-decision authority chain is broken".into(),
2828                    )
2829                })?;
2830            retained.insert(current.sequence);
2831            assignment_link = decision_record.previous_assignment_decision;
2832        }
2833        let retained_release = match head.recovery_release_head.as_ref() {
2834            Some(link) => {
2835                let admission = read_authority_record(store.as_ref(), link.sequence)
2836                    .await?
2837                    .filter(|record| {
2838                        record.recovery_release_head.as_ref() == Some(link)
2839                            && record
2840                                .recovery_release_commit
2841                                .as_ref()
2842                                .is_some_and(|commit| commit.terminal == link.terminal)
2843                    })
2844                    .ok_or_else(|| {
2845                        LeaseError::Invalid(
2846                            "retained recovery release authority link is broken".into(),
2847                        )
2848                    })?;
2849                if admission.lease.seq != link.sequence {
2850                    return Err(LeaseError::Invalid(
2851                        "recovery release admission sequence does not match its retained link"
2852                            .into(),
2853                    ));
2854                }
2855                let terminal = authority
2856                    .load_recovery_release_terminal(&link.terminal)
2857                    .await
2858                    .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2859                if admission
2860                    .recovery_release_commit
2861                    .as_ref()
2862                    .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2863                {
2864                    return Err(LeaseError::Invalid(
2865                        "retained recovery release terminal does not match its admission".into(),
2866                    ));
2867                }
2868                retained.insert(link.sequence);
2869                Some(link.terminal.clone())
2870            }
2871            None => None,
2872        };
2873
2874        let mut history_exhausted = false;
2875        for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2876            let (candidates, exhausted) =
2877                Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2878            if candidates.is_empty() {
2879                history_exhausted = true;
2880                break;
2881            }
2882            let deletions =
2883                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2884            let mut results = store.delete_stream(Box::pin(deletions));
2885            while let Some(result) = results.next().await {
2886                if let Err(error) = result {
2887                    if !matches!(error, object_store::Error::NotFound { .. }) {
2888                        return Err(LeaseError::Io(error.to_string()));
2889                    }
2890                }
2891            }
2892            if exhausted {
2893                history_exhausted = true;
2894                break;
2895            }
2896            tokio::task::yield_now().await;
2897        }
2898        if !history_exhausted {
2899            return Err(LeaseError::Io(
2900                "leader lease history still exceeds the bounded prune budget".into(),
2901            ));
2902        }
2903        Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await
2904    }
2905
2906    async fn prune_recovery_release_terminals(
2907        store: &Arc<dyn ObjectStore>,
2908        retained: Option<&RecoveryReleaseTerminalRef>,
2909        grace_ms: i64,
2910    ) -> Result<(), LeaseError> {
2911        let Some(retained) = retained else {
2912            return Ok(());
2913        };
2914        retained.validate()?;
2915        for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
2916            let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
2917            let mut listed = store.list(Some(&prefix));
2918            let now = now_millis();
2919            let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
2920            let mut exhausted = true;
2921            while let Some(entry) = listed.next().await {
2922                let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2923                let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
2924                let canonical = OsPath::from(format!(
2925                    "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
2926                ));
2927                if canonical != entry.location {
2928                    return Err(LeaseError::Invalid(format!(
2929                        "noncanonical recovery release terminal path {}",
2930                        entry.location
2931                    )));
2932                }
2933                if entry.location == recovery_release_terminal_path(retained)
2934                    || generation > retained.generation()
2935                    || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2936                {
2937                    continue;
2938                }
2939                candidates.push(entry.location);
2940                if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
2941                    exhausted = false;
2942                    break;
2943                }
2944            }
2945            if candidates.is_empty() {
2946                return Ok(());
2947            }
2948            let deletions =
2949                futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2950            let mut results = store.delete_stream(Box::pin(deletions));
2951            while let Some(result) = results.next().await {
2952                if let Err(error) = result {
2953                    if !matches!(error, object_store::Error::NotFound { .. }) {
2954                        return Err(LeaseError::Io(error.to_string()));
2955                    }
2956                }
2957            }
2958            if exhausted {
2959                return Ok(());
2960            }
2961            tokio::task::yield_now().await;
2962        }
2963        Ok(())
2964    }
2965
2966    async fn prune_candidates(
2967        store: &Arc<dyn ObjectStore>,
2968        retained: &BTreeSet<u64>,
2969        snapshot_head_sequence: u64,
2970        grace_ms: i64,
2971    ) -> Result<(Vec<OsPath>, bool), LeaseError> {
2972        let prefix = OsPath::from(LEASE_PREFIX);
2973        let mut listed = store.list(Some(&prefix));
2974        let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
2975        let now = now_millis();
2976        let mut exhausted = true;
2977        while let Some(entry) = listed.next().await {
2978            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2979            let sequence = lease_sequence_from_path(&entry.location)?;
2980            if sequence >= snapshot_head_sequence
2981                || retained.contains(&sequence)
2982                || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
2983            {
2984                continue;
2985            }
2986            candidates.push(entry.location);
2987            if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
2988                exhausted = false;
2989                break;
2990            }
2991        }
2992        Ok((candidates, exhausted))
2993    }
2994
2995    /// Load the highest durable sequence.
2996    ///
2997    /// # Errors
2998    /// Fails closed on object-store I/O or malformed durable state.
2999    pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
3000        Ok(self.load_record().await?.map(|record| record.lease))
3001    }
3002
3003    async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3004        Ok(self
3005            .load_published_authority_head()
3006            .await?
3007            .map(|head| head.record))
3008    }
3009
3010    async fn load_published_authority_head(
3011        &self,
3012    ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
3013        let pointer = if let Some(pointer) =
3014            read_authority_head_pointer(self.store.as_ref()).await?
3015        {
3016            pointer
3017        } else {
3018            let Some(discovered) = self.discover_authority_head().await? else {
3019                let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
3020                    return Ok(None);
3021                };
3022                return self.load_authority_head_target(pointer).await.map(Some);
3023            };
3024            let discovered_sequence = discovered.lease.seq;
3025            let published_sequence = self
3026                .publish_authority_head(discovered_sequence, None)
3027                .await?;
3028            let pointer = self
3029                .reload_authority_head_pointer(published_sequence)
3030                .await?;
3031            if published_sequence > discovered_sequence
3032                || pointer.pointer.sequence > published_sequence
3033            {
3034                return self.load_authority_head_target(pointer).await.map(Some);
3035            }
3036            pointer
3037        };
3038
3039        let sequence = pointer.pointer.sequence;
3040        let successor_sequence = sequence.checked_add(1);
3041        let (record, successor) = if let Some(successor_sequence) = successor_sequence {
3042            tokio::try_join!(
3043                read_authority_record(self.store.as_ref(), sequence),
3044                read_authority_record(self.store.as_ref(), successor_sequence)
3045            )?
3046        } else {
3047            (
3048                read_authority_record(self.store.as_ref(), sequence).await?,
3049                None,
3050            )
3051        };
3052        let Some(record) = record else {
3053            let rechecked = read_authority_head_pointer(self.store.as_ref())
3054                .await?
3055                .ok_or_else(|| {
3056                    LeaseError::Invalid(
3057                        "leader authority head disappeared while reading its target".into(),
3058                    )
3059                })?;
3060            if rechecked.pointer.sequence > sequence {
3061                return self
3062                    .reload_published_authority_snapshot(rechecked.pointer.sequence)
3063                    .await
3064                    .map(Some);
3065            }
3066            if rechecked.pointer.sequence < sequence {
3067                return Err(LeaseError::Invalid(format!(
3068                    "leader authority head regressed from sequence {sequence} to {}",
3069                    rechecked.pointer.sequence
3070                )));
3071            }
3072            return Err(LeaseError::Invalid(format!(
3073                "leader authority head points ahead to missing sequence {sequence}"
3074            )));
3075        };
3076        let Some(successor_sequence) = successor_sequence else {
3077            return Ok(Some(PublishedAuthorityHead { record, pointer }));
3078        };
3079        if successor.is_none() {
3080            return Ok(Some(PublishedAuthorityHead { record, pointer }));
3081        }
3082
3083        if let Some(after_successor) = successor_sequence.checked_add(1) {
3084            if read_authority_record(self.store.as_ref(), after_successor)
3085                .await?
3086                .is_some()
3087            {
3088                let rechecked = read_authority_head_pointer(self.store.as_ref())
3089                    .await?
3090                    .ok_or_else(|| {
3091                        LeaseError::Invalid(
3092                            "leader authority head disappeared while checking pointer lag".into(),
3093                        )
3094                    })?;
3095                if rechecked.pointer.sequence == sequence {
3096                    return Err(LeaseError::Invalid(format!(
3097                        "leader authority head at sequence {sequence} lags by more than one record"
3098                    )));
3099                }
3100                if rechecked.pointer.sequence < sequence {
3101                    return Err(LeaseError::Invalid(format!(
3102                        "leader authority head regressed from sequence {sequence} to {}",
3103                        rechecked.pointer.sequence
3104                    )));
3105                }
3106                return self
3107                    .reload_published_authority_snapshot(rechecked.pointer.sequence)
3108                    .await
3109                    .map(Some);
3110            }
3111        }
3112
3113        let published_sequence = self
3114            .publish_authority_head(successor_sequence, Some(&pointer))
3115            .await?;
3116        self.reload_published_authority_snapshot(published_sequence)
3117            .await
3118            .map(Some)
3119    }
3120
3121    async fn reload_published_authority_snapshot(
3122        &self,
3123        minimum_sequence: u64,
3124    ) -> Result<PublishedAuthorityHead, LeaseError> {
3125        let pointer = self.reload_authority_head_pointer(minimum_sequence).await?;
3126        self.load_authority_head_target(pointer).await
3127    }
3128
3129    async fn reload_authority_head_pointer(
3130        &self,
3131        minimum_sequence: u64,
3132    ) -> Result<VersionedAuthorityHeadPointer, LeaseError> {
3133        let pointer = read_authority_head_pointer(self.store.as_ref())
3134            .await?
3135            .ok_or_else(|| {
3136                LeaseError::Invalid(
3137                    "leader authority head disappeared while reloading its published snapshot"
3138                        .into(),
3139                )
3140            })?;
3141        if pointer.pointer.sequence < minimum_sequence {
3142            return Err(LeaseError::Invalid(format!(
3143                "leader authority head regressed from sequence {minimum_sequence} to {}",
3144                pointer.pointer.sequence
3145            )));
3146        }
3147        Ok(pointer)
3148    }
3149
3150    async fn load_authority_head_target(
3151        &self,
3152        mut pointer: VersionedAuthorityHeadPointer,
3153    ) -> Result<PublishedAuthorityHead, LeaseError> {
3154        for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
3155            let sequence = pointer.pointer.sequence;
3156            if let Some(record) = read_authority_record(self.store.as_ref(), sequence).await? {
3157                return Ok(PublishedAuthorityHead { record, pointer });
3158            }
3159
3160            let rechecked = read_authority_head_pointer(self.store.as_ref())
3161                .await?
3162                .ok_or_else(|| {
3163                    LeaseError::Invalid(
3164                        "leader authority head disappeared while reading its target".into(),
3165                    )
3166                })?;
3167            if rechecked.pointer.sequence < sequence {
3168                return Err(LeaseError::Invalid(format!(
3169                    "leader authority head regressed from sequence {sequence} to {}",
3170                    rechecked.pointer.sequence
3171                )));
3172            }
3173            if rechecked.pointer.sequence == sequence {
3174                return Err(LeaseError::Invalid(format!(
3175                    "leader authority head points ahead to missing sequence {sequence}"
3176                )));
3177            }
3178            pointer = rechecked;
3179            if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
3180                tokio::task::yield_now().await;
3181            }
3182        }
3183        Err(LeaseError::Io(format!(
3184            "leader authority head target changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
3185        )))
3186    }
3187
3188    async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3189        let prefix = OsPath::from(LEASE_PREFIX);
3190        let mut listed = self.store.list(Some(&prefix));
3191        let mut discovered = 0usize;
3192        let mut maximum = None;
3193        while let Some(entry) = listed.next().await {
3194            let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3195            discovered = discovered.checked_add(1).ok_or_else(|| {
3196                LeaseError::Invalid("leader authority discovery count overflowed".into())
3197            })?;
3198            if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
3199                return Err(LeaseError::Invalid(format!(
3200                    "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
3201                )));
3202            }
3203            let sequence = lease_sequence_from_path(&entry.location)?;
3204            maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
3205        }
3206        let Some(sequence) = maximum else {
3207            return Ok(None);
3208        };
3209        read_authority_record(self.store.as_ref(), sequence)
3210            .await?
3211            .map(Some)
3212            .ok_or_else(|| {
3213                LeaseError::Io(format!(
3214                    "discovered leader authority sequence {sequence} vanished before publication"
3215                ))
3216            })
3217    }
3218
3219    async fn publish_authority_head(
3220        &self,
3221        sequence: u64,
3222        expected: Option<&VersionedAuthorityHeadPointer>,
3223    ) -> Result<u64, LeaseError> {
3224        if let Some(expected) = expected {
3225            if expected.pointer.sequence.checked_add(1) != Some(sequence) {
3226                return Err(LeaseError::Invalid(
3227                    "leader authority head update is not the exact successor".into(),
3228                ));
3229            }
3230            if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3231            {
3232                return Err(LeaseError::Invalid(
3233                    "leader authority store did not provide a native conditional update version"
3234                        .into(),
3235                ));
3236            }
3237        }
3238
3239        let options = PutOptions {
3240            mode: expected.map_or(PutMode::Create, |expected| {
3241                PutMode::Update(expected.update_version.clone())
3242            }),
3243            ..PutOptions::default()
3244        };
3245        let result = self
3246            .store
3247            .put_opts(
3248                &authority_head_path(),
3249                PutPayload::from(encode_authority_head_pointer(sequence)?),
3250                options,
3251            )
3252            .await;
3253        if result.is_ok() {
3254            return Ok(sequence);
3255        }
3256        let write_error = result
3257            .expect_err("successful authority head writes return above")
3258            .to_string();
3259        let current = read_authority_head_pointer(self.store.as_ref())
3260            .await?
3261            .ok_or_else(|| {
3262                LeaseError::Io(format!(
3263                    "leader authority head write failed ({write_error}) and no pointer was durable"
3264                ))
3265            })?;
3266        if let Some(expected) = expected {
3267            if current.pointer.sequence < expected.pointer.sequence {
3268                return Err(LeaseError::Invalid(format!(
3269                    "leader authority head regressed from sequence {} to {}",
3270                    expected.pointer.sequence, current.pointer.sequence
3271                )));
3272            }
3273        }
3274        if current.pointer.sequence >= sequence {
3275            return Ok(current.pointer.sequence);
3276        }
3277        Err(LeaseError::Io(format!(
3278            "leader authority head write failed ({write_error}) and remained at sequence {}",
3279            current.pointer.sequence
3280        )))
3281    }
3282
3283    async fn read_catalog_manifest_blob(
3284        &self,
3285        reference: &CatalogManifestRef,
3286    ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3287        reference.validate()?;
3288        let path = reference.object_path();
3289        let result = match self.store.get(&path).await {
3290            Ok(result) => result,
3291            Err(object_store::Error::NotFound { .. }) => {
3292                return Err(CatalogManifestError::Invalid(format!(
3293                    "catalog manifest blob '{}' is missing",
3294                    reference.sha256
3295                )));
3296            }
3297            Err(error) => {
3298                return Err(CatalogManifestError::Authority(LeaseError::Io(
3299                    error.to_string(),
3300                )));
3301            }
3302        };
3303        if result.meta.size != reference.encoded_len {
3304            return Err(CatalogManifestError::Invalid(format!(
3305                "catalog manifest blob '{}' is {} bytes, expected {}",
3306                reference.sha256, result.meta.size, reference.encoded_len
3307            )));
3308        }
3309        let bytes = result
3310            .bytes()
3311            .await
3312            .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3313        if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3314            return Err(CatalogManifestError::Invalid(format!(
3315                "catalog manifest blob '{}' payload length changed while reading",
3316                reference.sha256
3317            )));
3318        }
3319        let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3320        let (canonical, actual_reference) = manifest.encode_and_reference()?;
3321        if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3322            return Err(CatalogManifestError::Invalid(format!(
3323                "catalog manifest blob '{}' does not match its sealed reference",
3324                reference.sha256
3325            )));
3326        }
3327        Ok((manifest, bytes))
3328    }
3329
3330    pub(super) async fn load_catalog_manifest(
3331        &self,
3332        reference: &CatalogManifestRef,
3333    ) -> Result<CatalogManifest, CatalogManifestError> {
3334        self.read_catalog_manifest_blob(reference)
3335            .await
3336            .map(|(manifest, _)| manifest)
3337    }
3338
3339    async fn ensure_catalog_manifest_blob(
3340        &self,
3341        encoded: &[u8],
3342        reference: &CatalogManifestRef,
3343    ) -> Result<(), CatalogManifestError> {
3344        reference.validate()?;
3345        let options = PutOptions {
3346            mode: PutMode::Create,
3347            ..PutOptions::default()
3348        };
3349        let path = reference.object_path();
3350        let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3351        let put_error = match self.store.put_opts(&path, payload, options).await {
3352            Ok(_)
3353            | Err(
3354                object_store::Error::AlreadyExists { .. }
3355                | object_store::Error::Precondition { .. },
3356            ) => None,
3357            Err(error) => Some(error),
3358        };
3359
3360        match self.read_catalog_manifest_blob(reference).await {
3361            Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3362            Ok(_) => Err(CatalogManifestError::Invalid(format!(
3363                "catalog manifest blob '{}' differs from the proposed content",
3364                reference.sha256
3365            ))),
3366            Err(error) => {
3367                if let Some(put_error) = put_error {
3368                    Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3369                        "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3370                    ))))
3371                } else {
3372                    Err(error)
3373                }
3374            }
3375        }
3376    }
3377
3378    async fn create_authority_record(
3379        &self,
3380        expected: Option<&PublishedAuthorityHead>,
3381        candidate: &LeaderAuthorityRecord,
3382    ) -> Result<AuthorityCreateOutcome, LeaseError> {
3383        let encoded = encode_authority_record(candidate)?;
3384        let expected_pointer = match expected {
3385            None => {
3386                if candidate.lease.seq != 1 {
3387                    return Err(LeaseError::Invalid(format!(
3388                        "cannot append authority sequence {} to an empty published namespace",
3389                        candidate.lease.seq
3390                    )));
3391                }
3392                None
3393            }
3394            Some(head) if head.record.lease.seq == candidate.lease.seq => {
3395                return if head.record == *candidate {
3396                    Ok(AuthorityCreateOutcome::ExistingIdentical)
3397                } else {
3398                    Ok(AuthorityCreateOutcome::Contended(Box::new(
3399                        head.record.clone(),
3400                    )))
3401                };
3402            }
3403            Some(head) if head.record.lease.seq > candidate.lease.seq => {
3404                return Ok(AuthorityCreateOutcome::Contended(Box::new(
3405                    head.record.clone(),
3406                )));
3407            }
3408            Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3409                Some(&head.pointer)
3410            }
3411            Some(head) => {
3412                return Err(LeaseError::Invalid(format!(
3413                    "cannot append authority sequence {} after published sequence {}",
3414                    candidate.lease.seq, head.record.lease.seq
3415                )));
3416            }
3417        };
3418        let options = PutOptions {
3419            mode: PutMode::Create,
3420            ..PutOptions::default()
3421        };
3422        let result = self
3423            .store
3424            .put_opts(
3425                &lease_path(candidate.lease.seq),
3426                PutPayload::from(encoded),
3427                options,
3428            )
3429            .await;
3430        let created = result.is_ok();
3431        if let Err(error) = result {
3432            let Some(at_sequence) =
3433                read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3434            else {
3435                return Err(LeaseError::Io(format!(
3436                    "authority record {} write failed ({error}) and no record was durable",
3437                    candidate.lease.seq
3438                )));
3439            };
3440            if at_sequence != *candidate {
3441                let winner = self.load_record().await?.ok_or_else(|| {
3442                    LeaseError::Io("authority contender was not published or readable".into())
3443                })?;
3444                return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3445            }
3446        }
3447
3448        let published_sequence = self
3449            .publish_authority_head(candidate.lease.seq, expected_pointer)
3450            .await?;
3451        if published_sequence > candidate.lease.seq {
3452            if created {
3453                self.schedule_history_prune();
3454            }
3455            let winner = self
3456                .load_record()
3457                .await?
3458                .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3459            return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3460        }
3461        self.schedule_history_prune();
3462        if created {
3463            Ok(AuthorityCreateOutcome::Created)
3464        } else {
3465            Ok(AuthorityCreateOutcome::ExistingIdentical)
3466        }
3467    }
3468
3469    /// Seal the complete catalog by appending it to the exact durable leader term.
3470    ///
3471    /// Renewals that race this operation are retried under the same proof. A takeover either
3472    /// observes and preserves the seal or wins the next sequence and fences this writer.
3473    ///
3474    /// # Errors
3475    /// Rejects malformed inventories, divergent existing inventories, stale proofs, and durable
3476    /// storage failures.
3477    pub(super) async fn seal_catalog(
3478        &self,
3479        proof: &LeaderProof,
3480        manifest: &CatalogManifest,
3481    ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3482        if !proof.is_canonical() {
3483            return Err(CatalogManifestError::Fenced);
3484        }
3485        let (encoded, reference) = manifest.encode_and_reference()?;
3486
3487        let current = self
3488            .load_record()
3489            .await?
3490            .ok_or(CatalogManifestError::Fenced)?;
3491        if let Some(sealed) = &current.lease.catalog_manifest {
3492            let durable = self.load_catalog_manifest(sealed).await?;
3493            return if durable == *manifest {
3494                Ok(CatalogSealOutcome::ExistingIdentical)
3495            } else {
3496                Err(CatalogManifestError::Conflict)
3497            };
3498        }
3499        if !current.lease.matches_proof(proof) {
3500            return Err(CatalogManifestError::Fenced);
3501        }
3502        self.ensure_catalog_manifest_blob(&encoded, &reference)
3503            .await?;
3504
3505        loop {
3506            let published = self
3507                .load_published_authority_head()
3508                .await?
3509                .ok_or(CatalogManifestError::Fenced)?;
3510            let current = &published.record;
3511            if let Some(sealed) = &current.lease.catalog_manifest {
3512                let durable = self.load_catalog_manifest(sealed).await?;
3513                return if durable == *manifest {
3514                    Ok(CatalogSealOutcome::ExistingIdentical)
3515                } else {
3516                    Err(CatalogManifestError::Conflict)
3517                };
3518            }
3519            if !current.lease.matches_proof(proof) {
3520                return Err(CatalogManifestError::Fenced);
3521            }
3522
3523            let base_sequence = current.lease.seq;
3524            let candidate_lease = LeaderLease {
3525                seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3526                    CatalogManifestError::Authority(LeaseError::Invalid(
3527                        "lease sequence exhausted".into(),
3528                    ))
3529                })?,
3530                renewal_sequence: current.lease.renewal_sequence,
3531                token: current.lease.token,
3532                owner: current.lease.owner.clone(),
3533                expires_at_ms: current.lease.expires_at_ms,
3534                catalog_manifest: Some(reference.clone()),
3535            };
3536            let candidate = current.preserve_with_lease(candidate_lease);
3537            match self
3538                .create_authority_record(Some(&published), &candidate)
3539                .await?
3540            {
3541                AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3542                AuthorityCreateOutcome::ExistingIdentical => {
3543                    return Ok(CatalogSealOutcome::ExistingIdentical);
3544                }
3545                AuthorityCreateOutcome::Contended(winner) => {
3546                    if let Some(sealed) = &winner.lease.catalog_manifest {
3547                        let durable = self.load_catalog_manifest(sealed).await?;
3548                        return if durable == *manifest {
3549                            Ok(CatalogSealOutcome::ExistingIdentical)
3550                        } else {
3551                            Err(CatalogManifestError::Conflict)
3552                        };
3553                    }
3554                    if !winner.lease.matches_proof(proof) {
3555                        return Err(CatalogManifestError::Fenced);
3556                    }
3557                    if winner.lease.seq <= base_sequence {
3558                        return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3559                            "catalog seal contention did not advance the authority sequence".into(),
3560                        )));
3561                    }
3562                    tokio::task::yield_now().await;
3563                }
3564            }
3565        }
3566    }
3567
3568    async fn audited_cluster_outcomes_from(
3569        &self,
3570        head: &LeaderAuthorityRecord,
3571    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3572        let floor = head.outcome_floor.as_ref();
3573        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3574        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3575        let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3576        let mut terminal_newest_first = Vec::new();
3577        let mut terminal_links = Vec::new();
3578        let mut link = head.outcome_head;
3579        let mut expected_commit_head = head.commit_head;
3580        let mut traversed = 0;
3581        while let Some(current) = link {
3582            if current.epoch < authority_before_epoch {
3583                break;
3584            }
3585            if !consume_live_authority_link(&mut traversed) {
3586                return Err(DecisionError::Conflict(format!(
3587                    "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3588                ))
3589                .into());
3590            }
3591            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3592                record.clone()
3593            } else {
3594                let record = read_authority_record(self.store.as_ref(), current.sequence)
3595                    .await?
3596                    .ok_or_else(|| {
3597                        DecisionError::InventoryChanged(format!(
3598                            "cluster outcome authority record {} disappeared during audit",
3599                            current.sequence
3600                        ))
3601                    })?;
3602                terminal_records.insert(current.sequence, record.clone());
3603                record
3604            };
3605            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3606                DecisionError::Conflict(format!(
3607                    "cluster outcome head epoch {} points to non-outcome authority record {}",
3608                    current.epoch, current.sequence
3609                ))
3610            })?;
3611            if record.outcome_head != Some(current)
3612                || outcome.epoch != current.epoch
3613                || outcome.checkpoint_id != current.checkpoint_id
3614            {
3615                return Err(DecisionError::Conflict(format!(
3616                    "cluster outcome link epoch {} sequence {} does not match its authority record",
3617                    current.epoch, current.sequence
3618                ))
3619                .into());
3620            }
3621            if record.commit_head != expected_commit_head {
3622                return Err(DecisionError::Conflict(format!(
3623                    "cluster outcome epoch {} did not preserve the exact Commit head",
3624                    outcome.epoch
3625                ))
3626                .into());
3627            }
3628            expected_commit_head = if outcome.is_commit() {
3629                record.previous_commit
3630            } else {
3631                record.commit_head
3632            };
3633            terminal_newest_first.push(outcome);
3634            terminal_links.push(current);
3635            link = record.previous_outcome;
3636        }
3637
3638        if let Some(floor) = floor {
3639            match (
3640                link,
3641                floor.terminal_anchor_link,
3642                floor.terminal_anchor.as_ref(),
3643            ) {
3644                (Some(link), Some(anchor_link), Some(anchor))
3645                    if link == anchor_link
3646                        && link.epoch == anchor.epoch
3647                        && link.checkpoint_id == anchor.checkpoint_id => {}
3648                (None, None, None) => {}
3649                _ => {
3650                    return Err(DecisionError::Conflict(format!(
3651                        "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3652                        floor.authority_before_epoch
3653                    ))
3654                    .into());
3655                }
3656            }
3657        } else if link.is_some() {
3658            return Err(DecisionError::Conflict(
3659                "cluster outcome chain stopped without a durable retention floor".into(),
3660            )
3661            .into());
3662        }
3663
3664        let mut commit_newest_first = Vec::new();
3665        let mut commit_links = Vec::new();
3666        link = head.commit_head;
3667        traversed = 0;
3668        while let Some(current) = link {
3669            if current.epoch < artifact_before_epoch {
3670                break;
3671            }
3672            if !consume_live_authority_link(&mut traversed) {
3673                return Err(DecisionError::Conflict(format!(
3674                    "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3675                ))
3676                .into());
3677            }
3678            let record = if let Some(record) = terminal_records.get(&current.sequence) {
3679                record.clone()
3680            } else {
3681                read_authority_record(self.store.as_ref(), current.sequence)
3682                    .await?
3683                    .ok_or_else(|| {
3684                        DecisionError::InventoryChanged(format!(
3685                            "cluster Commit authority record {} disappeared during audit",
3686                            current.sequence
3687                        ))
3688                    })?
3689            };
3690            let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3691                DecisionError::Conflict(format!(
3692                    "cluster Commit head epoch {} points to non-outcome authority record {}",
3693                    current.epoch, current.sequence
3694                ))
3695            })?;
3696            if !outcome.is_commit()
3697                || record.commit_head != Some(current)
3698                || outcome.epoch != current.epoch
3699                || outcome.checkpoint_id != current.checkpoint_id
3700            {
3701                return Err(DecisionError::Conflict(format!(
3702                    "cluster Commit link epoch {} sequence {} does not match its authority record",
3703                    current.epoch, current.sequence
3704                ))
3705                .into());
3706            }
3707            commit_newest_first.push(outcome);
3708            commit_links.push(current);
3709            link = record.previous_commit;
3710        }
3711
3712        if let Some(floor) = floor {
3713            match (
3714                link,
3715                floor.committed_anchor_link,
3716                floor.committed_anchor.as_ref(),
3717            ) {
3718                (Some(link), Some(anchor_link), Some(anchor))
3719                    if link == anchor_link
3720                        && link.epoch == anchor.epoch
3721                        && link.checkpoint_id == anchor.checkpoint_id => {}
3722                (None, None, None) => {}
3723                _ => {
3724                    return Err(DecisionError::Conflict(format!(
3725                        "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3726                        floor.artifact_before_epoch
3727                    ))
3728                    .into());
3729                }
3730            }
3731        } else if link.is_some() {
3732            return Err(DecisionError::Conflict(
3733                "cluster Commit chain stopped without a durable retention floor".into(),
3734            )
3735            .into());
3736        }
3737
3738        terminal_newest_first.reverse();
3739        terminal_links.reverse();
3740        commit_newest_first.reverse();
3741        commit_links.reverse();
3742        let boundary_commit_head = floor
3743            .and_then(|floor| floor.terminal_anchor.as_ref())
3744            .and_then(|anchor| {
3745                commit_links
3746                    .iter()
3747                    .rev()
3748                    .find(|link| link.epoch <= anchor.epoch)
3749                    .copied()
3750                    .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3751            });
3752        if expected_commit_head != boundary_commit_head {
3753            return Err(DecisionError::Conflict(
3754                "cluster terminal chain does not preserve Commit continuity at its durable floor"
3755                    .into(),
3756            )
3757            .into());
3758        }
3759        for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3760            if !outcome.is_commit() {
3761                continue;
3762            }
3763            let commit_link = commit_links
3764                .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3765                .ok()
3766                .map(|index| commit_links[index]);
3767            if commit_link != Some(*terminal_link) {
3768                return Err(DecisionError::Conflict(format!(
3769                    "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3770                    outcome.epoch
3771                ))
3772                .into());
3773            }
3774        }
3775        if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3776            floor
3777                .terminal_anchor
3778                .as_ref()
3779                .zip(floor.terminal_anchor_link)
3780        }) {
3781            if terminal_anchor.is_commit() {
3782                let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3783                    commit_links
3784                        .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3785                        .ok()
3786                        .map(|index| commit_links[index])
3787                } else {
3788                    floor.and_then(|floor| {
3789                        (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3790                            .then_some(floor.committed_anchor_link)
3791                            .flatten()
3792                    })
3793                };
3794                if commit_link != Some(terminal_anchor_link) {
3795                    return Err(DecisionError::Conflict(format!(
3796                        "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3797                        terminal_anchor.epoch
3798                    ))
3799                    .into());
3800                }
3801            }
3802        }
3803        let mut outcomes = Vec::with_capacity(
3804            terminal_newest_first
3805                .len()
3806                .saturating_add(commit_newest_first.len())
3807                .saturating_add(2),
3808        );
3809        outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3810        outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3811        outcomes.extend(terminal_newest_first);
3812        outcomes.extend(commit_newest_first);
3813        outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3814        let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3815        for outcome in outcomes {
3816            if let Some(previous) = merged.last() {
3817                if previous.epoch == outcome.epoch {
3818                    if previous != &outcome {
3819                        return Err(DecisionError::Conflict(format!(
3820                            "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3821                            outcome.epoch
3822                        ))
3823                        .into());
3824                    }
3825                    continue;
3826                }
3827            }
3828            merged.push(outcome);
3829        }
3830
3831        let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3832            .load_or_create_deployment_id()
3833            .await?;
3834        for outcome in &merged {
3835            if outcome.deployment_id != expected_deployment {
3836                return Err(DecisionError::Conflict(format!(
3837                    "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3838                    outcome.epoch, outcome.deployment_id, expected_deployment
3839                ))
3840                .into());
3841            }
3842        }
3843        for pair in merged.windows(2) {
3844            let previous = &pair[0];
3845            let current = &pair[1];
3846            if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3847                return Err(DecisionError::Conflict(format!(
3848                    "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3849                    previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3850                ))
3851                .into());
3852            }
3853        }
3854        Ok(ClusterOutcomeAuditSnapshot {
3855            outcomes: Arc::from(merged),
3856            terminal_links: Arc::from(terminal_links),
3857            commit_links: Arc::from(commit_links),
3858        })
3859    }
3860
3861    fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3862        ClusterOutcomeAuditKey {
3863            terminal_head: head.outcome_head,
3864            commit_head: head.commit_head,
3865            floor: head.outcome_floor.clone(),
3866        }
3867    }
3868
3869    fn cached_cluster_outcome_audit(
3870        &self,
3871        key: &ClusterOutcomeAuditKey,
3872        authority_sequence: u64,
3873    ) -> Option<ClusterOutcomeAuditSnapshot> {
3874        let mut cache = self.outcome_audit_cache.lock();
3875        let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3876        cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3877        Some(cached.snapshot.clone())
3878    }
3879
3880    fn install_cluster_outcome_audit(
3881        &self,
3882        key: ClusterOutcomeAuditKey,
3883        authority_sequence: u64,
3884        snapshot: ClusterOutcomeAuditSnapshot,
3885    ) {
3886        let mut cache = self.outcome_audit_cache.lock();
3887        if cache.as_ref().is_none_or(|cached| {
3888            cached.authority_sequence < authority_sequence
3889                || (cached.authority_sequence == authority_sequence && cached.key == key)
3890        }) {
3891            *cache = Some(ClusterOutcomeAuditCache {
3892                key,
3893                authority_sequence,
3894                snapshot,
3895            });
3896        }
3897    }
3898
3899    fn cluster_outcome_audit_gate(
3900        &self,
3901        key: &ClusterOutcomeAuditKey,
3902    ) -> Arc<tokio::sync::Mutex<()>> {
3903        let mut flights = self.outcome_audit_flights.lock();
3904        flights.retain(|flight| flight.gate.strong_count() != 0);
3905        if let Some(gate) = flights
3906            .iter()
3907            .find(|flight| flight.key == *key)
3908            .and_then(|flight| flight.gate.upgrade())
3909        {
3910            return gate;
3911        }
3912        let gate = Arc::new(tokio::sync::Mutex::new(()));
3913        flights.push(ClusterOutcomeAuditFlight {
3914            key: key.clone(),
3915            gate: Arc::downgrade(&gate),
3916        });
3917        gate
3918    }
3919
3920    async fn cached_audited_cluster_outcomes_from(
3921        &self,
3922        head: &LeaderAuthorityRecord,
3923    ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3924        let key = Self::cluster_outcome_audit_key(head);
3925        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3926            return Ok(snapshot);
3927        }
3928
3929        let gate = self.cluster_outcome_audit_gate(&key);
3930        let _guard = gate.lock().await;
3931        if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
3932            return Ok(snapshot);
3933        }
3934
3935        let snapshot = self.audited_cluster_outcomes_from(head).await?;
3936        self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
3937        Ok(snapshot)
3938    }
3939
3940    fn outcomes_retained_by_floor(
3941        floor: &AuthorityOutcomeFloor,
3942        snapshot: &ClusterOutcomeAuditSnapshot,
3943    ) -> ClusterOutcomeAuditSnapshot {
3944        let outcomes = &snapshot.outcomes;
3945        let mut retained = Vec::with_capacity(outcomes.len());
3946        if let Some(anchor) = floor.committed_anchor.as_ref() {
3947            retained.push(anchor.clone());
3948        }
3949        if let Some(anchor) = floor.terminal_anchor.as_ref() {
3950            if retained.last() != Some(anchor) {
3951                retained.push(anchor.clone());
3952            }
3953        }
3954        retained.extend(
3955            outcomes
3956                .iter()
3957                .filter(|outcome| {
3958                    outcome.epoch >= floor.authority_before_epoch
3959                        || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
3960                })
3961                .cloned(),
3962        );
3963        retained.sort_unstable_by_key(|outcome| outcome.epoch);
3964        retained.dedup();
3965        ClusterOutcomeAuditSnapshot {
3966            outcomes: Arc::from(retained),
3967            terminal_links: Arc::from(
3968                snapshot
3969                    .terminal_links
3970                    .iter()
3971                    .filter(|link| link.epoch >= floor.authority_before_epoch)
3972                    .copied()
3973                    .collect::<Vec<_>>(),
3974            ),
3975            commit_links: Arc::from(
3976                snapshot
3977                    .commit_links
3978                    .iter()
3979                    .filter(|link| link.epoch >= floor.artifact_before_epoch)
3980                    .copied()
3981                    .collect::<Vec<_>>(),
3982            ),
3983        }
3984    }
3985
3986    async fn build_cluster_outcome_floor(
3987        &self,
3988        current: &LeaderAuthorityRecord,
3989        snapshot: &ClusterOutcomeAuditSnapshot,
3990        artifact_before_epoch: u64,
3991        authority_before_epoch: u64,
3992    ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
3993        let terminal_anchor = snapshot
3994            .outcomes
3995            .iter()
3996            .rev()
3997            .find(|outcome| outcome.epoch < authority_before_epoch)
3998            .cloned();
3999        let terminal_anchor_link = match terminal_anchor.as_ref() {
4000            Some(anchor)
4001                if current
4002                    .outcome_floor
4003                    .as_ref()
4004                    .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
4005            {
4006                current
4007                    .outcome_floor
4008                    .as_ref()
4009                    .and_then(|floor| floor.terminal_anchor_link)
4010            }
4011            Some(anchor) => {
4012                let index = snapshot
4013                    .terminal_links
4014                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4015                    .map_err(|_| {
4016                        DecisionError::Conflict(format!(
4017                            "cluster outcome epoch {} has no audited authority link",
4018                            anchor.epoch
4019                        ))
4020                    })?;
4021                let link = snapshot.terminal_links[index];
4022                if link.checkpoint_id != anchor.checkpoint_id {
4023                    return Err(DecisionError::Conflict(format!(
4024                        "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
4025                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4026                    ))
4027                    .into());
4028                }
4029                Some(link)
4030            }
4031            None => None,
4032        };
4033        let committed_anchor = snapshot
4034            .outcomes
4035            .iter()
4036            .rev()
4037            .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
4038            .cloned();
4039        let committed_anchor_link = match committed_anchor.as_ref() {
4040            Some(anchor)
4041                if current
4042                    .outcome_floor
4043                    .as_ref()
4044                    .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
4045            {
4046                current
4047                    .outcome_floor
4048                    .as_ref()
4049                    .and_then(|floor| floor.committed_anchor_link)
4050            }
4051            Some(anchor) => {
4052                let index = snapshot
4053                    .commit_links
4054                    .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4055                    .map_err(|_| {
4056                        DecisionError::Conflict(format!(
4057                            "cluster Commit epoch {} has no audited Commit link",
4058                            anchor.epoch
4059                        ))
4060                    })?;
4061                let link = snapshot.commit_links[index];
4062                if link.checkpoint_id != anchor.checkpoint_id {
4063                    return Err(DecisionError::Conflict(format!(
4064                        "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
4065                        anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4066                    ))
4067                    .into());
4068                }
4069                Some(link)
4070            }
4071            None => None,
4072        };
4073        let floor = AuthorityOutcomeFloor {
4074            deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
4075                .load_or_create_deployment_id()
4076                .await?,
4077            artifact_before_epoch,
4078            authority_before_epoch,
4079            terminal_anchor,
4080            terminal_anchor_link,
4081            committed_anchor,
4082            committed_anchor_link,
4083        };
4084        floor.validate()?;
4085        Ok(floor)
4086    }
4087
4088    async fn compact_cluster_outcome_history_before_append(
4089        &self,
4090        proof: &LeaderProof,
4091        current: &LeaderAuthorityRecord,
4092        snapshot: &ClusterOutcomeAuditSnapshot,
4093    ) -> Result<bool, ClusterCheckpointAuthorityError> {
4094        if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
4095            return Ok(false);
4096        }
4097        let authority_before_epoch = snapshot.terminal_links
4098            [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
4099            .epoch;
4100        let artifact_before_epoch = current
4101            .outcome_floor
4102            .as_ref()
4103            .map_or(0, |floor| floor.artifact_before_epoch);
4104        let floor = self
4105            .build_cluster_outcome_floor(
4106                current,
4107                snapshot,
4108                artifact_before_epoch,
4109                authority_before_epoch,
4110            )
4111            .await?;
4112        // Floor construction may perform remote reads. Publish only if the audited chain and both
4113        // horizons are still the exact authority snapshot used to select the anchors.
4114        let published = self
4115            .load_published_authority_head()
4116            .await?
4117            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4118        let rechecked = &published.record;
4119        if !rechecked.lease.matches_proof(proof) {
4120            return Err(ClusterCheckpointAuthorityError::Fenced);
4121        }
4122        if rechecked.outcome_head != current.outcome_head
4123            || rechecked.commit_head != current.commit_head
4124            || rechecked.outcome_floor != current.outcome_floor
4125        {
4126            return Ok(true);
4127        }
4128
4129        let sequence = rechecked
4130            .lease
4131            .seq
4132            .checked_add(1)
4133            .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4134        let mut lease = rechecked.lease.clone();
4135        lease.seq = sequence;
4136        let mut next = rechecked.preserve_with_lease(lease);
4137        next.outcome_floor = Some(floor.clone());
4138        next.validate()?;
4139        match self
4140            .create_authority_record(Some(&published), &next)
4141            .await?
4142        {
4143            AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4144                self.install_cluster_outcome_audit(
4145                    Self::cluster_outcome_audit_key(&next),
4146                    next.lease.seq,
4147                    Self::outcomes_retained_by_floor(&floor, snapshot),
4148                );
4149                Ok(true)
4150            }
4151            AuthorityCreateOutcome::Contended(winner) => {
4152                if !winner.lease.matches_proof(proof) {
4153                    return Err(ClusterCheckpointAuthorityError::Fenced);
4154                }
4155                if winner.lease.seq <= current.lease.seq {
4156                    return Err(LeaseError::Invalid(
4157                        "outcome history compaction contention did not advance the authority sequence"
4158                            .into(),
4159                    )
4160                    .into());
4161                }
4162                Ok(true)
4163            }
4164        }
4165    }
4166
4167    async fn audited_cluster_outcomes(
4168        &self,
4169    ) -> Result<
4170        (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
4171        ClusterCheckpointAuthorityError,
4172    > {
4173        const AUDIT_RETRIES: usize = 3;
4174        for attempt in 0..AUDIT_RETRIES {
4175            let Some(head) = self.load_record().await? else {
4176                *self.outcome_audit_cache.lock() = None;
4177                return Ok((None, Arc::from([])));
4178            };
4179            match self.cached_audited_cluster_outcomes_from(&head).await {
4180                Err(ClusterCheckpointAuthorityError::Decision(
4181                    DecisionError::InventoryChanged(_),
4182                )) if attempt + 1 < AUDIT_RETRIES => {
4183                    tokio::task::yield_now().await;
4184                }
4185                Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
4186                Err(error) => return Err(error),
4187            }
4188        }
4189        Err(DecisionError::InventoryChanged(
4190            "cluster outcome audit exhausted stability retries".into(),
4191        )
4192        .into())
4193    }
4194
4195    async fn audited_assignment_decisions_from(
4196        &self,
4197        head: &LeaderAuthorityRecord,
4198    ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4199        let mut newest_first = Vec::new();
4200        let floor = head.assignment_decision_floor.as_ref();
4201        let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
4202        let mut stopped_at_anchor = false;
4203        let mut link = head.assignment_decision_head;
4204        let mut traversed = 0;
4205        while let Some(current) = link {
4206            if !consume_live_authority_link(&mut traversed) {
4207                return Err(DecisionError::Conflict(format!(
4208                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
4209                ))
4210                .into());
4211            }
4212            if current.target_version < before_target_version {
4213                if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
4214                    return Err(DecisionError::Conflict(format!(
4215                        "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
4216                    ))
4217                    .into());
4218                }
4219                stopped_at_anchor = true;
4220                break;
4221            }
4222            let record = read_authority_record(self.store.as_ref(), current.sequence)
4223                .await?
4224                .ok_or_else(|| {
4225                    DecisionError::InventoryChanged(format!(
4226                        "assignment decision authority record {} disappeared during audit",
4227                        current.sequence
4228                    ))
4229                })?;
4230            let decision = record.assignment_decision.clone().ok_or_else(|| {
4231                DecisionError::Conflict(format!(
4232                    "assignment decision head version {} points to non-decision authority record {}",
4233                    current.target_version, current.sequence
4234                ))
4235            })?;
4236            if record.assignment_decision_head != Some(current)
4237                || decision.target_version() != current.target_version
4238            {
4239                return Err(DecisionError::Conflict(format!(
4240                    "assignment decision link version {} sequence {} does not match its authority record",
4241                    current.target_version, current.sequence
4242                ))
4243                .into());
4244            }
4245            newest_first.push(decision);
4246            link = record.previous_assignment_decision;
4247        }
4248        if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4249            return Err(DecisionError::Conflict(
4250                "assignment decision chain stopped without its durable retention anchor".into(),
4251            )
4252            .into());
4253        }
4254        newest_first.reverse();
4255        if let Some(pair) = newest_first
4256            .windows(2)
4257            .find(|pair| pair[0].target_version() >= pair[1].target_version())
4258        {
4259            return Err(DecisionError::Conflict(format!(
4260                "assignment decisions regress from version {} to {}",
4261                pair[0].target_version(),
4262                pair[1].target_version()
4263            ))
4264            .into());
4265        }
4266        Ok(newest_first)
4267    }
4268
4269    async fn exact_assignment_decision_link(
4270        &self,
4271        head: &LeaderAuthorityRecord,
4272        decision: &AuthorityAssignmentDecision,
4273    ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4274        if let Some(floor) = head.assignment_decision_floor.as_ref() {
4275            if floor.terminal_anchor.as_ref() == Some(decision) {
4276                return floor.terminal_anchor_link.ok_or_else(|| {
4277                    DecisionError::Conflict(
4278                        "assignment decision floor lost its terminal authority link".into(),
4279                    )
4280                    .into()
4281                });
4282            }
4283        }
4284        let before_target_version = head
4285            .assignment_decision_floor
4286            .as_ref()
4287            .map_or(0, |floor| floor.before_target_version);
4288        let mut link = head.assignment_decision_head;
4289        let mut traversed = 0;
4290        while let Some(current) = link {
4291            if !consume_live_authority_link(&mut traversed) {
4292                return Err(DecisionError::Conflict(format!(
4293                    "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4294                ))
4295                .into());
4296            }
4297            if current.target_version < before_target_version {
4298                break;
4299            }
4300            let record = read_authority_record(self.store.as_ref(), current.sequence)
4301                .await?
4302                .ok_or_else(|| {
4303                    DecisionError::InventoryChanged(format!(
4304                        "assignment decision authority record {} disappeared during exact lookup",
4305                        current.sequence
4306                    ))
4307                })?;
4308            if record.assignment_decision.as_ref() == Some(decision) {
4309                return Ok(current);
4310            }
4311            link = record.previous_assignment_decision;
4312        }
4313        Err(DecisionError::Conflict(format!(
4314            "assignment decision version {} is not linked from the durable authority head",
4315            decision.target_version()
4316        ))
4317        .into())
4318    }
4319
4320    async fn reject_consumed_checkpoint_assignment(
4321        &self,
4322        current: &LeaderAuthorityRecord,
4323        assignment_fence: &CheckpointAssignmentFence,
4324    ) -> Result<(), ClusterCheckpointAuthorityError> {
4325        let decisions = self.audited_assignment_decisions_from(current).await?;
4326        if let Some(floor) = current.assignment_decision_floor.as_ref() {
4327            // The floor is half-open: only target versions below it were compacted. Equality is
4328            // not evidence that the fence was consumed (notably for a bootstrap assignment),
4329            // while any decision at the boundary remains in `decisions` and is checked below.
4330            if assignment_fence.assignment_version < floor.before_target_version {
4331                return Err(DecisionError::Conflict(format!(
4332                    "checkpoint assignment version {} is below durable assignment-decision floor {}",
4333                    assignment_fence.assignment_version, floor.before_target_version
4334                ))
4335                .into());
4336            }
4337        }
4338        let Some(newest) = decisions.last() else {
4339            return Ok(());
4340        };
4341        if newest.target_version() > assignment_fence.assignment_version {
4342            return Err(DecisionError::Conflict(format!(
4343                "checkpoint assignment version {} was consumed by assignment decision version {}",
4344                assignment_fence.assignment_version,
4345                newest.target_version()
4346            ))
4347            .into());
4348        }
4349        if newest.target_version() == assignment_fence.assignment_version
4350            && newest.materialized_target() != *assignment_fence
4351        {
4352            return Err(DecisionError::Conflict(format!(
4353                "checkpoint assignment version {} does not match its materialized assignment decision",
4354                assignment_fence.assignment_version
4355            ))
4356            .into());
4357        }
4358        Ok(())
4359    }
4360
4361    /// Admit one exact cluster checkpoint attempt before any participant writes artifacts.
4362    ///
4363    /// An identical retry returns the durable inventory. A later attempt cannot begin until the
4364    /// current attempt commits or its aborted artifacts are cleaned exactly.
4365    ///
4366    /// # Errors
4367    /// Fails for a stale proof, foreign deployment, malformed inventory, another active attempt,
4368    /// or object-store failure.
4369    pub async fn begin_cluster_checkpoint_artifacts(
4370        &self,
4371        proof: &LeaderProof,
4372        inventory: CheckpointArtifactInventory,
4373    ) -> Result<CheckpointArtifactInventory, ClusterCheckpointAuthorityError> {
4374        if !proof.is_canonical() {
4375            return Err(ClusterCheckpointAuthorityError::Fenced);
4376        }
4377        inventory.validate().map_err(DecisionError::Conflict)?;
4378        let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
4379            DecisionError::Conflict(
4380                "cluster checkpoint artifact inventory requires an assignment fence".into(),
4381            )
4382        })?;
4383        if assignment_fence.participant_incarnation(proof.owner.node_id)
4384            != Some(proof.owner.boot_id)
4385        {
4386            return Err(DecisionError::Conflict(
4387                "checkpoint artifact admitting leader is outside the assignment fence".into(),
4388            )
4389            .into());
4390        }
4391        let initial = self
4392            .load_record()
4393            .await?
4394            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4395        if !initial.lease.matches_proof(proof) {
4396            return Err(ClusterCheckpointAuthorityError::Fenced);
4397        }
4398        let deployment_id = CheckpointDecisionStore::new(Arc::clone(&self.store))
4399            .load_or_create_deployment_id()
4400            .await?;
4401        if inventory.deployment_id != deployment_id {
4402            return Err(DecisionError::Conflict(format!(
4403                "checkpoint artifact inventory deployment {} does not match authority deployment {deployment_id}",
4404                inventory.deployment_id
4405            ))
4406            .into());
4407        }
4408
4409        loop {
4410            let published = self
4411                .load_published_authority_head()
4412                .await?
4413                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4414            let current = &published.record;
4415            if !current.lease.matches_proof(proof) {
4416                return Err(ClusterCheckpointAuthorityError::Fenced);
4417            }
4418            self.reject_consumed_checkpoint_assignment(current, assignment_fence)
4419                .await?;
4420            if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
4421                if active == &inventory
4422                    && current.active_checkpoint_artifact_leader_proof.as_ref() == Some(proof)
4423                    && current.outcome_head.is_none_or(|head| {
4424                        head.epoch != inventory.attempt.epoch
4425                            || head.checkpoint_id != inventory.attempt.checkpoint_id
4426                    })
4427                {
4428                    return Ok(active.clone());
4429                }
4430                return Err(DecisionError::Conflict(format!(
4431                    "checkpoint {} still has unresolved or inherited cluster artifacts",
4432                    active.attempt.checkpoint_id
4433                ))
4434                .into());
4435            }
4436            if let Some(head) = current.outcome_head {
4437                let terminal =
4438                    crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
4439                if inventory.attempt.relation_to(terminal)
4440                    != crate::checkpoint::CheckpointAttemptRelation::Newer
4441                {
4442                    return Err(DecisionError::Conflict(format!(
4443                        "checkpoint {} does not advance terminal checkpoint {}",
4444                        inventory.attempt.checkpoint_id, head.checkpoint_id
4445                    ))
4446                    .into());
4447                }
4448            }
4449
4450            let sequence =
4451                current.lease.seq.checked_add(1).ok_or_else(|| {
4452                    LeaseError::Invalid("leader authority sequence exhausted".into())
4453                })?;
4454            let mut lease = current.lease.clone();
4455            lease.seq = sequence;
4456            let mut next = current.preserve_with_lease(lease);
4457            next.active_checkpoint_artifacts = Some(inventory.clone());
4458            next.active_checkpoint_artifact_leader_proof = Some(proof.clone());
4459            next.validate()?;
4460            match self
4461                .create_authority_record(Some(&published), &next)
4462                .await?
4463            {
4464                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4465                    return Ok(inventory);
4466                }
4467                AuthorityCreateOutcome::Contended(winner) => {
4468                    if !winner.lease.matches_proof(proof) {
4469                        return Err(ClusterCheckpointAuthorityError::Fenced);
4470                    }
4471                    tokio::task::yield_now().await;
4472                }
4473            }
4474        }
4475    }
4476
4477    /// Clear one exact retained inventory after its durable Abort artifact paths are sealed.
4478    ///
4479    /// # Errors
4480    /// Fails for a stale proof, a different active attempt, a missing matching Abort, malformed
4481    /// inventory, or object-store failure.
4482    pub async fn finish_cluster_checkpoint_artifact_cleanup(
4483        &self,
4484        proof: &LeaderProof,
4485        expected: &CheckpointArtifactInventory,
4486    ) -> Result<(), ClusterCheckpointAuthorityError> {
4487        if !proof.is_canonical() {
4488            return Err(ClusterCheckpointAuthorityError::Fenced);
4489        }
4490        expected.validate().map_err(DecisionError::Conflict)?;
4491        if expected.assignment_fence.is_none() {
4492            return Err(DecisionError::Conflict(
4493                "cluster checkpoint artifact inventory requires an assignment fence".into(),
4494            )
4495            .into());
4496        }
4497
4498        loop {
4499            let published = self
4500                .load_published_authority_head()
4501                .await?
4502                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4503            let current = &published.record;
4504            if !current.lease.matches_proof(proof) {
4505                return Err(ClusterCheckpointAuthorityError::Fenced);
4506            }
4507            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4508            let matching_abort = snapshot.outcomes.iter().any(|outcome| {
4509                matches!(outcome.verdict, CheckpointVerdict::Abort)
4510                    && outcome.deployment_id == expected.deployment_id
4511                    && outcome.epoch == expected.attempt.epoch
4512                    && outcome.checkpoint_id == expected.attempt.checkpoint_id
4513                    && outcome.assignment_fence.as_ref() == expected.assignment_fence.as_ref()
4514            });
4515            match current.active_checkpoint_artifacts.as_ref() {
4516                None if matching_abort => return Ok(()),
4517                None => {
4518                    return Err(DecisionError::Conflict(format!(
4519                        "checkpoint {} has no retained cluster artifact inventory",
4520                        expected.attempt.checkpoint_id
4521                    ))
4522                    .into());
4523                }
4524                Some(active) if active != expected => {
4525                    return Err(DecisionError::Conflict(format!(
4526                        "checkpoint {} still has unresolved cluster artifacts",
4527                        active.attempt.checkpoint_id
4528                    ))
4529                    .into());
4530                }
4531                Some(_) if !matching_abort => {
4532                    return Err(DecisionError::Conflict(format!(
4533                        "checkpoint {} has no matching durable Abort",
4534                        expected.attempt.checkpoint_id
4535                    ))
4536                    .into());
4537                }
4538                Some(_) => {}
4539            }
4540
4541            let sequence =
4542                current.lease.seq.checked_add(1).ok_or_else(|| {
4543                    LeaseError::Invalid("leader authority sequence exhausted".into())
4544                })?;
4545            let mut lease = current.lease.clone();
4546            lease.seq = sequence;
4547            let mut next = current.preserve_with_lease(lease);
4548            next.active_checkpoint_artifacts = None;
4549            next.active_checkpoint_artifact_leader_proof = None;
4550            next.validate()?;
4551            match self
4552                .create_authority_record(Some(&published), &next)
4553                .await?
4554            {
4555                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4556                    return Ok(());
4557                }
4558                AuthorityCreateOutcome::Contended(winner) => {
4559                    if !winner.lease.matches_proof(proof) {
4560                        return Err(ClusterCheckpointAuthorityError::Fenced);
4561                    }
4562                    tokio::task::yield_now().await;
4563                }
4564            }
4565        }
4566    }
4567
4568    /// Admit one cluster terminal outcome through the exact next leader-authority sequence.
4569    ///
4570    /// Renewals, takeovers, catalog seals, floor advances, and other decisions all contend on the
4571    /// same create-only object. An identical retry converges on the durable winner.
4572    ///
4573    /// # Errors
4574    /// Fails closed for a stale proof, non-monotonic or conflicting outcome, malformed committed
4575    /// checkpoint index, or object-store failure.
4576    pub async fn record_cluster_outcome(
4577        &self,
4578        proof: &LeaderProof,
4579        epoch: u64,
4580        checkpoint_id: u64,
4581        assignment_fence: CheckpointAssignmentFence,
4582        verdict: CheckpointVerdict,
4583        committed_checkpoint: Option<CommittedCheckpointRef>,
4584    ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4585        if !proof.is_canonical() {
4586            return Err(ClusterCheckpointAuthorityError::Fenced);
4587        }
4588        let attempt = crate::checkpoint::CheckpointAttempt::new(epoch, checkpoint_id);
4589        if !attempt.is_canonical() {
4590            return Err(DecisionError::Conflict(
4591                "cluster checkpoint outcomes require one nonzero canonical checkpoint ID".into(),
4592            )
4593            .into());
4594        }
4595        let initial = self
4596            .load_record()
4597            .await?
4598            .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4599        if !initial.lease.matches_proof(proof) {
4600            return Err(ClusterCheckpointAuthorityError::Fenced);
4601        }
4602        let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
4603        let (candidate, committed_index) = decisions
4604            .canonical_outcome_with_index(
4605                epoch,
4606                checkpoint_id,
4607                CheckpointScope::Cluster,
4608                Some(assignment_fence),
4609                Some(proof.clone()),
4610                verdict,
4611                committed_checkpoint,
4612            )
4613            .await?;
4614
4615        loop {
4616            let published = self
4617                .load_published_authority_head()
4618                .await?
4619                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4620            let current = &published.record;
4621            if !current.lease.matches_proof(proof) {
4622                return Err(ClusterCheckpointAuthorityError::Fenced);
4623            }
4624            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4625            let outcomes = &snapshot.outcomes;
4626            if let Some(winner) = outcomes
4627                .iter()
4628                .find(|outcome| outcome.epoch == candidate.epoch)
4629            {
4630                return if winner == &candidate {
4631                    Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4632                } else {
4633                    Ok(RecordOutcomeResult::Conflict {
4634                        winner: winner.clone(),
4635                    })
4636                };
4637            }
4638            let active = match current.active_checkpoint_artifacts.as_ref() {
4639                Some(active)
4640                    if active.deployment_id == candidate.deployment_id
4641                        && active.attempt.epoch == candidate.epoch
4642                        && active.attempt.checkpoint_id == candidate.checkpoint_id
4643                        && active.assignment_fence.as_ref()
4644                            == candidate.assignment_fence.as_ref() =>
4645                {
4646                    Some(active)
4647                }
4648                Some(_) => {
4649                    return Err(DecisionError::Conflict(format!(
4650                        "cluster checkpoint {} does not match the active artifact inventory",
4651                        candidate.checkpoint_id
4652                    ))
4653                    .into());
4654                }
4655                None if candidate.is_commit() => {
4656                    return Err(DecisionError::Conflict(format!(
4657                        "cluster Commit checkpoint {} has no admitted artifact inventory",
4658                        candidate.checkpoint_id
4659                    ))
4660                    .into());
4661                }
4662                None => None,
4663            };
4664            if candidate.is_commit()
4665                && current.active_checkpoint_artifact_leader_proof.as_ref() != Some(proof)
4666            {
4667                return Err(DecisionError::Conflict(format!(
4668                    "takeover leader cannot Commit checkpoint {} admitted by an older leader term",
4669                    candidate.checkpoint_id
4670                ))
4671                .into());
4672            }
4673            if let Some(last) = outcomes.last() {
4674                if candidate.checkpoint_id <= last.checkpoint_id {
4675                    return Err(DecisionError::Conflict(format!(
4676                        "cluster checkpoint {} does not advance durable checkpoint {}",
4677                        candidate.checkpoint_id, last.checkpoint_id
4678                    ))
4679                    .into());
4680                }
4681            }
4682            if candidate.is_commit()
4683                && current
4684                    .assignment_handoff_pin
4685                    .as_ref()
4686                    .is_some_and(|pin| candidate.assignment_fence.as_ref() != Some(&pin.target))
4687            {
4688                return Err(DecisionError::Conflict(
4689                    "cluster Commit does not bind the active assignment handoff target".into(),
4690                )
4691                .into());
4692            }
4693            let (commit_index, expected_predecessor) = if candidate.is_commit() {
4694                let index = committed_index.as_ref().ok_or_else(|| {
4695                    DecisionError::Conflict(
4696                        "canonical cluster Commit is missing its committed checkpoint index".into(),
4697                    )
4698                })?;
4699                let expected_predecessor = outcomes
4700                    .iter()
4701                    .rev()
4702                    .find(|outcome| outcome.is_commit())
4703                    .and_then(|outcome| outcome.committed_checkpoint.clone());
4704                if index.predecessor != expected_predecessor {
4705                    return Err(DecisionError::Conflict(format!(
4706                        "cluster Commit checkpoint {} does not extend the authoritative Commit head",
4707                        candidate.checkpoint_id
4708                    ))
4709                    .into());
4710                }
4711                if active.is_none_or(|active| index.pipeline_identity != active.pipeline_identity) {
4712                    return Err(DecisionError::Conflict(format!(
4713                        "cluster Commit checkpoint {} does not match its admitted pipeline identity",
4714                        candidate.checkpoint_id
4715                    ))
4716                    .into());
4717                }
4718                (Some(index), expected_predecessor)
4719            } else {
4720                (None, None)
4721            };
4722            if candidate.is_commit() && snapshot.commit_links.len() >= MAX_LIVE_AUTHORITY_LINKS {
4723                return Err(DecisionError::Conflict(format!(
4724                    "live Commit retention reached the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound; advance the artifact-retention horizon before admitting another Commit"
4725                ))
4726                .into());
4727            }
4728            if let Some(floor) = current.outcome_floor.as_ref() {
4729                if candidate.deployment_id != floor.deployment_id
4730                    || candidate.epoch < floor.authority_before_epoch
4731                {
4732                    return Err(DecisionError::Conflict(format!(
4733                        "cluster outcome epoch {} is below or outside authority floor {}",
4734                        candidate.epoch, floor.authority_before_epoch
4735                    ))
4736                    .into());
4737                }
4738            }
4739            if Box::pin(
4740                self.compact_cluster_outcome_history_before_append(proof, current, &snapshot),
4741            )
4742            .await?
4743            {
4744                tokio::task::yield_now().await;
4745                continue;
4746            }
4747            if let (Some(index), Some(predecessor_ref)) =
4748                (commit_index, expected_predecessor.as_ref())
4749            {
4750                let predecessor = decisions.load_committed_checkpoint(predecessor_ref).await?;
4751                index
4752                    .validate_predecessor_index(&predecessor)
4753                    .map_err(DecisionError::Conflict)?;
4754            }
4755
4756            let base_sequence = current.lease.seq;
4757            let sequence = base_sequence
4758                .checked_add(1)
4759                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4760            let mut next = current.preserve_with_lease(LeaderLease {
4761                seq: sequence,
4762                renewal_sequence: current.lease.renewal_sequence,
4763                token: current.lease.token,
4764                owner: current.lease.owner.clone(),
4765                expires_at_ms: current.lease.expires_at_ms,
4766                catalog_manifest: current.lease.catalog_manifest.clone(),
4767            });
4768            next.checkpoint_outcome = Some(candidate.clone());
4769            next.previous_outcome = current.outcome_head;
4770            let new_link = OutcomeLink {
4771                sequence,
4772                epoch: candidate.epoch,
4773                checkpoint_id: candidate.checkpoint_id,
4774            };
4775            next.outcome_head = Some(new_link);
4776            if candidate.is_commit() {
4777                next.previous_commit = current.commit_head;
4778                next.commit_head = Some(new_link);
4779                next.active_checkpoint_artifacts = None;
4780                next.active_checkpoint_artifact_leader_proof = None;
4781                if next
4782                    .assignment_handoff_pin
4783                    .as_ref()
4784                    .is_some_and(|pin| candidate.assignment_fence.as_ref() == Some(&pin.target))
4785                {
4786                    next.assignment_handoff_pin = None;
4787                }
4788            }
4789            next.validate()?;
4790            match self
4791                .create_authority_record(Some(&published), &next)
4792                .await?
4793            {
4794                AuthorityCreateOutcome::Created => {
4795                    let mut appended = outcomes.to_vec();
4796                    appended.push(candidate.clone());
4797                    let mut terminal_links = snapshot.terminal_links.to_vec();
4798                    terminal_links.push(new_link);
4799                    let mut commit_links = snapshot.commit_links.to_vec();
4800                    if candidate.is_commit() {
4801                        commit_links.push(new_link);
4802                    }
4803                    self.install_cluster_outcome_audit(
4804                        Self::cluster_outcome_audit_key(&next),
4805                        next.lease.seq,
4806                        ClusterOutcomeAuditSnapshot {
4807                            outcomes: Arc::from(appended),
4808                            terminal_links: Arc::from(terminal_links),
4809                            commit_links: Arc::from(commit_links),
4810                        },
4811                    );
4812                    return Ok(RecordOutcomeResult::Created(candidate));
4813                }
4814                AuthorityCreateOutcome::ExistingIdentical => {
4815                    return Ok(RecordOutcomeResult::Unchanged(candidate));
4816                }
4817                AuthorityCreateOutcome::Contended(winner_head) => {
4818                    let winners = self
4819                        .cached_audited_cluster_outcomes_from(&winner_head)
4820                        .await?;
4821                    if let Some(winner) = winners
4822                        .outcomes
4823                        .iter()
4824                        .find(|outcome| outcome.epoch == candidate.epoch)
4825                    {
4826                        return if winner == &candidate {
4827                            Ok(RecordOutcomeResult::Unchanged(winner.clone()))
4828                        } else {
4829                            Ok(RecordOutcomeResult::Conflict {
4830                                winner: winner.clone(),
4831                            })
4832                        };
4833                    }
4834                    if !winner_head.lease.matches_proof(proof) {
4835                        return Err(ClusterCheckpointAuthorityError::Fenced);
4836                    }
4837                    if winner_head.lease.seq <= base_sequence {
4838                        return Err(LeaseError::Invalid(
4839                            "cluster outcome contention did not advance the authority sequence"
4840                                .into(),
4841                        )
4842                        .into());
4843                    }
4844                    tokio::task::yield_now().await;
4845                }
4846            }
4847        }
4848    }
4849
4850    async fn validate_assignment_recovery_snapshot(
4851        &self,
4852        decision: &AssignmentRecoveryDecision,
4853    ) -> Result<(), ClusterCheckpointAuthorityError> {
4854        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4855        let predecessor = snapshots
4856            .load()
4857            .await
4858            .map_err(|error| {
4859                assignment_snapshot_error("load assignment recovery predecessor", error)
4860            })?
4861            .ok_or_else(|| {
4862                LeaseError::Invalid(
4863                    "assignment recovery requires a durable predecessor snapshot".into(),
4864                )
4865            })?;
4866        let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4867            LeaseError::Invalid(format!(
4868                "assignment recovery predecessor has no valid fence: {error}"
4869            ))
4870        })?;
4871        if predecessor.draining || predecessor_fence != decision.predecessor {
4872            return Err(LeaseError::Invalid(
4873                "assignment recovery predecessor is not the exact committed assignment head".into(),
4874            )
4875            .into());
4876        }
4877        let proposal = snapshots
4878            .load_recovery_proposal(&decision.proposal)
4879            .await
4880            .map_err(|error| {
4881                assignment_snapshot_error("load assignment recovery proposal", error)
4882            })?;
4883        let proposal_fence = proposal.assignment_fence().map_err(|error| {
4884            LeaseError::Invalid(format!(
4885                "assignment recovery proposal has no valid fence: {error}"
4886            ))
4887        })?;
4888        if proposal_fence != decision.target {
4889            return Err(LeaseError::Invalid(
4890                "assignment recovery proposal does not match its authorized target fence".into(),
4891            )
4892            .into());
4893        }
4894        Ok(())
4895    }
4896
4897    async fn current_assignment_checkpoint(
4898        &self,
4899        current: &LeaderAuthorityRecord,
4900        predecessor: &CheckpointAssignmentFence,
4901        purpose: &str,
4902    ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
4903        let Some(commit_head) = current.commit_head else {
4904            return Ok(None);
4905        };
4906        let outcome = self
4907            .cluster_outcome_from_snapshot(current, commit_head.epoch)
4908            .await?
4909            .ok_or_else(|| {
4910                DecisionError::Conflict(format!(
4911                    "{purpose} checkpoint {} is not live",
4912                    commit_head.checkpoint_id
4913                ))
4914            })?;
4915        if !outcome.is_commit()
4916            || outcome.checkpoint_id != commit_head.checkpoint_id
4917            || outcome.assignment_fence.as_ref() != Some(predecessor)
4918        {
4919            return Err(DecisionError::Conflict(format!(
4920                "{purpose} checkpoint {} does not bind its predecessor fence",
4921                commit_head.checkpoint_id
4922            ))
4923            .into());
4924        }
4925        let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
4926            DecisionError::Conflict(format!(
4927                "{purpose} checkpoint {} has no committed index reference",
4928                commit_head.checkpoint_id
4929            ))
4930        })?;
4931        let index = CheckpointDecisionStore::new(Arc::clone(&self.store))
4932            .validate_committed_checkpoint_for_outcome(&outcome)
4933            .await?;
4934        if index.assignment_fence.as_ref() != Some(predecessor) {
4935            return Err(DecisionError::Conflict(format!(
4936                "{purpose} checkpoint index {} does not bind its predecessor fence",
4937                commit_head.checkpoint_id
4938            ))
4939            .into());
4940        }
4941        Ok(Some(reference.clone()))
4942    }
4943
4944    async fn validate_current_assignment_checkpoint(
4945        &self,
4946        current: &LeaderAuthorityRecord,
4947        reference: &CommittedCheckpointRef,
4948        predecessor: &CheckpointAssignmentFence,
4949        purpose: &str,
4950    ) -> Result<(), ClusterCheckpointAuthorityError> {
4951        let current_reference = self
4952            .current_assignment_checkpoint(current, predecessor, purpose)
4953            .await?
4954            .ok_or_else(|| {
4955                DecisionError::Conflict(format!("{purpose} has no authoritative checkpoint Commit"))
4956            })?;
4957        if current_reference != *reference {
4958            return Err(DecisionError::Conflict(format!(
4959                "{purpose} checkpoint {} is not the current Commit head {}",
4960                reference.checkpoint_id, current_reference.checkpoint_id
4961            ))
4962            .into());
4963        }
4964        Ok(())
4965    }
4966
4967    fn aborted_handoff_target(
4968        transition: &AssignmentDrainTransition,
4969    ) -> Result<CheckpointAssignmentFence, ClusterCheckpointAuthorityError> {
4970        let mut target = transition.predecessor.clone();
4971        target.assignment_version = transition.target.assignment_version;
4972        if !target.is_canonical() {
4973            return Err(LeaseError::Invalid(
4974                "assignment drain abort produced an invalid handoff target".into(),
4975            )
4976            .into());
4977        }
4978        Ok(target)
4979    }
4980
4981    async fn next_assignment_handoff_pin(
4982        &self,
4983        current: &LeaderAuthorityRecord,
4984        decision: &AuthorityAssignmentDecision,
4985    ) -> Result<Option<AssignmentHandoffPin>, ClusterCheckpointAuthorityError> {
4986        match decision {
4987            AuthorityAssignmentDecision::Drain(drain) => match drain.verdict {
4988                AssignmentDrainVerdict::Commit => {
4989                    if current.assignment_handoff_pin.is_some() {
4990                        return Err(DecisionError::Conflict(
4991                            "assignment decision cannot overtake an unresolved state handoff"
4992                                .into(),
4993                        )
4994                        .into());
4995                    }
4996                    let reference = drain.handoff_checkpoint.as_ref().ok_or_else(|| {
4997                        LeaseError::Invalid(
4998                            "committed assignment drain has no handoff checkpoint".into(),
4999                        )
5000                    })?;
5001                    self.validate_current_assignment_checkpoint(
5002                        current,
5003                        reference,
5004                        &drain.transition.predecessor,
5005                        "assignment drain handoff",
5006                    )
5007                    .await?;
5008                    Ok(Some(AssignmentHandoffPin {
5009                        target: drain.transition.target.clone(),
5010                        checkpoint: reference.clone(),
5011                    }))
5012                }
5013                AssignmentDrainVerdict::Abort => match current.assignment_handoff_pin.as_ref() {
5014                    None => {
5015                        let checkpoint = self
5016                            .current_assignment_checkpoint(
5017                                current,
5018                                &drain.transition.predecessor,
5019                                "assignment drain abort",
5020                            )
5021                            .await?;
5022                        match checkpoint {
5023                            Some(checkpoint) => Ok(Some(AssignmentHandoffPin {
5024                                target: Self::aborted_handoff_target(&drain.transition)?,
5025                                checkpoint,
5026                            })),
5027                            None => Ok(None),
5028                        }
5029                    }
5030                    Some(pin) if pin.target == drain.transition.predecessor => {
5031                        Ok(Some(AssignmentHandoffPin {
5032                            target: Self::aborted_handoff_target(&drain.transition)?,
5033                            checkpoint: pin.checkpoint.clone(),
5034                        }))
5035                    }
5036                    Some(_) => Err(DecisionError::Conflict(
5037                        "assignment decision cannot overtake an unresolved state handoff".into(),
5038                    )
5039                    .into()),
5040                },
5041            },
5042            AuthorityAssignmentDecision::Recovery(recovery) => {
5043                if let Some(pin) = current.assignment_handoff_pin.as_ref() {
5044                    if pin.target != recovery.predecessor
5045                        || pin.checkpoint != recovery.recovery_checkpoint
5046                    {
5047                        return Err(DecisionError::Conflict(
5048                            "assignment recovery does not continue the exact unresolved state handoff"
5049                                .into(),
5050                        )
5051                        .into());
5052                    }
5053                } else {
5054                    self.validate_current_assignment_checkpoint(
5055                        current,
5056                        &recovery.recovery_checkpoint,
5057                        &recovery.predecessor,
5058                        "assignment recovery",
5059                    )
5060                    .await?;
5061                }
5062                Ok(Some(AssignmentHandoffPin {
5063                    target: recovery.target.clone(),
5064                    checkpoint: recovery.recovery_checkpoint.clone(),
5065                }))
5066            }
5067        }
5068    }
5069
5070    async fn record_assignment_decision(
5071        &self,
5072        proof: &LeaderProof,
5073        decision: AuthorityAssignmentDecision,
5074    ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
5075        decision.validate()?;
5076        if decision.leader_proof() != proof || !proof.is_canonical() {
5077            return Err(ClusterCheckpointAuthorityError::Fenced);
5078        }
5079
5080        loop {
5081            let published = self
5082                .load_published_authority_head()
5083                .await?
5084                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5085            let current = &published.record;
5086            if !current.lease.matches_proof(proof) {
5087                return Err(ClusterCheckpointAuthorityError::Fenced);
5088            }
5089            if let Some(floor) = current.assignment_decision_floor.as_ref() {
5090                if decision.target_version() < floor.before_target_version {
5091                    return Err(DecisionError::Conflict(format!(
5092                        "assignment decision version {} is below durable retention floor {}",
5093                        decision.target_version(),
5094                        floor.before_target_version
5095                    ))
5096                    .into());
5097                }
5098            }
5099            let decisions = self.audited_assignment_decisions_from(current).await?;
5100            if let Some(winner) = decisions
5101                .iter()
5102                .find(|winner| winner.target_version() == decision.target_version())
5103            {
5104                return if winner == &decision {
5105                    Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5106                        winner.clone(),
5107                    ))
5108                } else {
5109                    Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5110                        winner: winner.clone(),
5111                    })
5112                };
5113            }
5114            if matches!(&decision, AuthorityAssignmentDecision::Drain(_)) {
5115                let predecessor = decision.predecessor();
5116                if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
5117                    let active_fence = active.assignment_fence.as_ref().ok_or_else(|| {
5118                        DecisionError::Conflict(
5119                            "active checkpoint artifact inventory lost its assignment fence".into(),
5120                        )
5121                    })?;
5122                    if active_fence.assignment_version == predecessor.assignment_version {
5123                        let detail = if active_fence == predecessor {
5124                            "matches"
5125                        } else {
5126                            "conflicts with"
5127                        };
5128                        return Err(DecisionError::Conflict(format!(
5129                            "assignment drain decision cannot overtake checkpoint {} whose assignment fence {detail} predecessor version {}",
5130                            active.attempt.checkpoint_id,
5131                            predecessor.assignment_version
5132                        ))
5133                        .into());
5134                    }
5135                }
5136            }
5137            if let Some(last) = decisions.last() {
5138                if decision.target_version() <= last.target_version() {
5139                    return Err(DecisionError::Conflict(format!(
5140                        "assignment decision version {} does not advance durable version {}",
5141                        decision.target_version(),
5142                        last.target_version()
5143                    ))
5144                    .into());
5145                }
5146            }
5147            if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
5148                self.validate_assignment_recovery_snapshot(recovery).await?;
5149            }
5150            let assignment_handoff_pin =
5151                self.next_assignment_handoff_pin(current, &decision).await?;
5152
5153            let base_sequence = current.lease.seq;
5154            let sequence = base_sequence
5155                .checked_add(1)
5156                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5157            let mut candidate = current.preserve_with_lease(LeaderLease {
5158                seq: sequence,
5159                renewal_sequence: current.lease.renewal_sequence,
5160                token: current.lease.token,
5161                owner: current.lease.owner.clone(),
5162                expires_at_ms: current.lease.expires_at_ms,
5163                catalog_manifest: current.lease.catalog_manifest.clone(),
5164            });
5165            candidate.assignment_decision = Some(decision.clone());
5166            candidate.previous_assignment_decision = current.assignment_decision_head;
5167            candidate.assignment_decision_head = Some(AssignmentDecisionLink {
5168                sequence,
5169                target_version: decision.target_version(),
5170            });
5171            candidate.assignment_handoff_pin = assignment_handoff_pin;
5172            candidate.validate()?;
5173
5174            match self
5175                .create_authority_record(Some(&published), &candidate)
5176                .await?
5177            {
5178                AuthorityCreateOutcome::Created => {
5179                    return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
5180                }
5181                AuthorityCreateOutcome::ExistingIdentical => {
5182                    return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
5183                }
5184                AuthorityCreateOutcome::Contended(winner_head) => {
5185                    let winners = self.audited_assignment_decisions_from(&winner_head).await?;
5186                    if let Some(winner) = winners
5187                        .iter()
5188                        .find(|winner| winner.target_version() == decision.target_version())
5189                    {
5190                        return if winner == &decision {
5191                            Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5192                                winner.clone(),
5193                            ))
5194                        } else {
5195                            Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5196                                winner: winner.clone(),
5197                            })
5198                        };
5199                    }
5200                    if !winner_head.lease.matches_proof(proof) {
5201                        return Err(ClusterCheckpointAuthorityError::Fenced);
5202                    }
5203                    if winner_head.lease.seq > base_sequence {
5204                        tokio::task::yield_now().await;
5205                        continue;
5206                    }
5207                    return Err(LeaseError::Invalid(
5208                        "assignment authority contention did not advance the sequence".into(),
5209                    )
5210                    .into());
5211                }
5212            }
5213        }
5214    }
5215
5216    /// Admit one assignment-drain settlement through the exact next authority sequence.
5217    ///
5218    /// Lease renewals, takeovers, checkpoint outcomes, and other decisions contend on that same
5219    /// create-only sequence. An identical retry converges on the durable winner.
5220    ///
5221    /// # Errors
5222    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
5223    pub async fn record_assignment_drain_decision(
5224        &self,
5225        proof: &LeaderProof,
5226        decision: AssignmentDrainDecision,
5227    ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5228        match Box::pin(
5229            self.record_assignment_decision(proof, AuthorityAssignmentDecision::Drain(decision)),
5230        )
5231        .await?
5232        {
5233            RecordAuthorityAssignmentDecisionResult::Created(
5234                AuthorityAssignmentDecision::Drain(decision),
5235            ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
5236            RecordAuthorityAssignmentDecisionResult::Unchanged(
5237                AuthorityAssignmentDecision::Drain(decision),
5238            ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
5239            RecordAuthorityAssignmentDecisionResult::Conflict {
5240                winner: AuthorityAssignmentDecision::Drain(winner),
5241            } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
5242            RecordAuthorityAssignmentDecisionResult::Conflict {
5243                winner: AuthorityAssignmentDecision::Recovery(winner),
5244            } => Err(DecisionError::Conflict(format!(
5245                "assignment recovery decision already settled target version {}",
5246                winner.target_version()
5247            ))
5248            .into()),
5249            RecordAuthorityAssignmentDecisionResult::Created(_)
5250            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5251                "assignment decision recorder returned the wrong decision kind".into(),
5252            )
5253            .into()),
5254        }
5255    }
5256
5257    /// Admit one failure-recovery assignment through the exact next authority sequence.
5258    ///
5259    /// # Errors
5260    /// Fails closed for a stale proof, malformed/non-monotonic decision, or storage failure.
5261    pub(crate) async fn record_assignment_recovery_decision(
5262        &self,
5263        proof: &LeaderProof,
5264        decision: AssignmentRecoveryDecision,
5265    ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
5266        match Box::pin(
5267            self.record_assignment_decision(proof, AuthorityAssignmentDecision::Recovery(decision)),
5268        )
5269        .await?
5270        {
5271            RecordAuthorityAssignmentDecisionResult::Created(
5272                AuthorityAssignmentDecision::Recovery(decision),
5273            ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
5274            RecordAuthorityAssignmentDecisionResult::Unchanged(
5275                AuthorityAssignmentDecision::Recovery(decision),
5276            ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
5277            RecordAuthorityAssignmentDecisionResult::Conflict {
5278                winner: AuthorityAssignmentDecision::Recovery(winner),
5279            } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
5280            RecordAuthorityAssignmentDecisionResult::Conflict {
5281                winner: AuthorityAssignmentDecision::Drain(winner),
5282            } => Err(DecisionError::Conflict(format!(
5283                "assignment drain decision already settled target version {}",
5284                winner.target_version()
5285            ))
5286            .into()),
5287            RecordAuthorityAssignmentDecisionResult::Created(_)
5288            | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5289                "assignment decision recorder returned the wrong decision kind".into(),
5290            )
5291            .into()),
5292        }
5293    }
5294
5295    async fn assignment_decision(
5296        &self,
5297        target_version: u64,
5298    ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
5299        if target_version == 0 {
5300            return Err(LeaseError::Invalid(
5301                "assignment decision target version must be nonzero".into(),
5302            )
5303            .into());
5304        }
5305        let Some(head) = self.load_record().await? else {
5306            return Ok(None);
5307        };
5308        if let Some(floor) = head.assignment_decision_floor.as_ref() {
5309            if target_version < floor.before_target_version {
5310                return Err(DecisionError::Conflict(format!(
5311                    "assignment decision version {target_version} is below durable retention floor {}",
5312                    floor.before_target_version
5313                ))
5314                .into());
5315            }
5316        }
5317        Ok(self
5318            .audited_assignment_decisions_from(&head)
5319            .await?
5320            .into_iter()
5321            .find(|decision| decision.target_version() == target_version))
5322    }
5323
5324    /// Read the immutable settlement for one exact target assignment version.
5325    ///
5326    /// # Errors
5327    /// Fails closed on malformed or incomplete authority history.
5328    pub async fn assignment_drain_decision(
5329        &self,
5330        target_version: u64,
5331    ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
5332        match self.assignment_decision(target_version).await? {
5333            Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
5334            Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
5335                format!("target version {target_version} was settled by assignment recovery"),
5336            )
5337            .into()),
5338            None => Ok(None),
5339        }
5340    }
5341
5342    /// Read the immutable recovery authorization for one exact target assignment version.
5343    ///
5344    /// # Errors
5345    /// Fails closed on malformed, incomplete, or cross-kind authority history.
5346    pub async fn assignment_recovery_decision(
5347        &self,
5348        target_version: u64,
5349    ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
5350        match self.assignment_decision(target_version).await? {
5351            Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
5352            Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
5353                "target version {target_version} was settled by assignment drain"
5354            ))
5355            .into()),
5356            None => Ok(None),
5357        }
5358    }
5359
5360    /// Read the state handoff checkpoint pinned for one exact target assignment.
5361    ///
5362    /// # Errors
5363    /// Fails when `target` is invalid or the durable authority head is unavailable.
5364    pub async fn assignment_handoff_checkpoint(
5365        &self,
5366        target: &CheckpointAssignmentFence,
5367    ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
5368        if !target.is_canonical() {
5369            return Err(LeaseError::Invalid(
5370                "assignment handoff checkpoint lookup requires a canonical target fence".into(),
5371            )
5372            .into());
5373        }
5374        Ok(self.load_record().await?.and_then(|head| {
5375            head.assignment_handoff_pin
5376                .filter(|pin| pin.target == *target)
5377                .map(|pin| pin.checkpoint)
5378        }))
5379    }
5380
5381    /// Materialize the immutable authority winner for one recovery target version.
5382    ///
5383    /// The caller supplies only the version; an arbitrary staged proposal can never bypass the
5384    /// shared leader-decision chain.
5385    ///
5386    /// # Errors
5387    /// Fails closed when no recovery decision exists, its proposal is invalid, or storage fails.
5388    pub async fn materialize_assignment_recovery(
5389        &self,
5390        target_version: u64,
5391    ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
5392        let decision = self
5393            .assignment_recovery_decision(target_version)
5394            .await?
5395            .ok_or_else(|| {
5396                DecisionError::Conflict(format!(
5397                    "assignment recovery target {target_version} has no authority decision"
5398                ))
5399            })?;
5400        let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5401        let proposal = snapshots
5402            .load_recovery_proposal(&decision.proposal)
5403            .await
5404            .map_err(|error| {
5405                assignment_snapshot_error("load authorized assignment recovery proposal", error)
5406            })?;
5407        if proposal.assignment_fence().map_err(|error| {
5408            LeaseError::Invalid(format!(
5409                "authorized assignment recovery proposal has no valid fence: {error}"
5410            ))
5411        })? != decision.target
5412        {
5413            return Err(LeaseError::Invalid(
5414                "authorized assignment recovery proposal does not match its target".into(),
5415            )
5416            .into());
5417        }
5418        snapshots
5419            .materialize_recovery(&decision.proposal)
5420            .await
5421            .map_err(|error| {
5422                assignment_snapshot_error("materialize authorized assignment recovery", error)
5423                    .into()
5424            })
5425    }
5426
5427    /// Advance the shared assignment-decision floor through the exact next authority sequence.
5428    ///
5429    /// The compatibility-shaped API retains both drain and recovery decisions on one ordered
5430    /// chain. The caller must first durably prune assignment snapshots below the same
5431    /// target-version horizon. Decision-bearing authority records below the durable floor then
5432    /// become eligible for best-effort deletion while the exact terminal anchor preserves chain
5433    /// continuity.
5434    ///
5435    /// # Errors
5436    ///
5437    /// Fails for a stale proof, invalid horizon, corrupt history, or storage failure.
5438    pub async fn prune_assignment_drain_decisions_before(
5439        &self,
5440        proof: &LeaderProof,
5441        before_target_version: u64,
5442    ) -> Result<u64, ClusterCheckpointAuthorityError> {
5443        if before_target_version == 0 {
5444            return Ok(self
5445                .load_record()
5446                .await?
5447                .and_then(|head| head.assignment_decision_floor)
5448                .map_or(0, |floor| floor.before_target_version));
5449        }
5450        if !proof.is_canonical() {
5451            return Err(ClusterCheckpointAuthorityError::Fenced);
5452        }
5453        loop {
5454            let published = self
5455                .load_published_authority_head()
5456                .await?
5457                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5458            let current = &published.record;
5459            if !current.lease.matches_proof(proof) {
5460                return Err(ClusterCheckpointAuthorityError::Fenced);
5461            }
5462            if let Some(floor) = current.assignment_decision_floor.as_ref() {
5463                if floor.before_target_version >= before_target_version {
5464                    self.schedule_history_prune();
5465                    return Ok(floor.before_target_version);
5466                }
5467            }
5468
5469            let decisions = self.audited_assignment_decisions_from(current).await?;
5470            let terminal_anchor = decisions
5471                .iter()
5472                .rev()
5473                .find(|decision| decision.target_version() < before_target_version)
5474                .cloned()
5475                .or_else(|| {
5476                    current
5477                        .assignment_decision_floor
5478                        .as_ref()
5479                        .and_then(|floor| floor.terminal_anchor.clone())
5480                });
5481            let terminal_anchor_link = match terminal_anchor.as_ref() {
5482                Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
5483                None => None,
5484            };
5485            let floor = AuthorityAssignmentDecisionFloor {
5486                before_target_version,
5487                terminal_anchor,
5488                terminal_anchor_link,
5489            };
5490            floor.validate()?;
5491
5492            let base_sequence = current.lease.seq;
5493            let sequence = base_sequence
5494                .checked_add(1)
5495                .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5496            let mut next = current.preserve_with_lease(LeaderLease {
5497                seq: sequence,
5498                renewal_sequence: current.lease.renewal_sequence,
5499                token: current.lease.token,
5500                owner: current.lease.owner.clone(),
5501                expires_at_ms: current.lease.expires_at_ms,
5502                catalog_manifest: current.lease.catalog_manifest.clone(),
5503            });
5504            next.assignment_decision_floor = Some(floor);
5505            next.validate()?;
5506
5507            match self
5508                .create_authority_record(Some(&published), &next)
5509                .await?
5510            {
5511                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5512                    return Ok(before_target_version);
5513                }
5514                AuthorityCreateOutcome::Contended(winner) => {
5515                    if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
5516                        if winner_floor.before_target_version >= before_target_version {
5517                            self.schedule_history_prune();
5518                            return Ok(winner_floor.before_target_version);
5519                        }
5520                    }
5521                    if !winner.lease.matches_proof(proof) {
5522                        return Err(ClusterCheckpointAuthorityError::Fenced);
5523                    }
5524                    if winner.lease.seq > base_sequence {
5525                        tokio::task::yield_now().await;
5526                        continue;
5527                    }
5528                    return Err(LeaseError::Invalid(
5529                        "assignment decision floor contention did not advance the sequence".into(),
5530                    )
5531                    .into());
5532                }
5533            }
5534        }
5535    }
5536
5537    async fn cluster_outcome_from_snapshot(
5538        &self,
5539        head: &LeaderAuthorityRecord,
5540        epoch: u64,
5541    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5542        let floor = head.outcome_floor.as_ref();
5543        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5544        if epoch < artifact_before_epoch {
5545            return Ok(None);
5546        }
5547        let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
5548        let cache_key = Self::cluster_outcome_audit_key(head);
5549        if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
5550            return Ok(snapshot
5551                .outcomes
5552                .binary_search_by_key(&epoch, |outcome| outcome.epoch)
5553                .ok()
5554                .map(|index| snapshot.outcomes[index].clone())
5555                .filter(|outcome| {
5556                    outcome.epoch >= authority_before_epoch
5557                        || outcome.is_commit()
5558                        || floor
5559                            .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
5560                }));
5561        }
5562        if let Some(anchor) = floor
5563            .and_then(|floor| floor.terminal_anchor.as_ref())
5564            .filter(|anchor| anchor.epoch == epoch)
5565        {
5566            let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5567                .load_or_create_deployment_id()
5568                .await?;
5569            if anchor.deployment_id != expected_deployment {
5570                return Err(DecisionError::Conflict(format!(
5571                    "cluster terminal anchor epoch {} does not belong to current deployment {}",
5572                    anchor.epoch, expected_deployment
5573                ))
5574                .into());
5575            }
5576            return Ok(Some(anchor.clone()));
5577        }
5578
5579        let commit_only = epoch < authority_before_epoch;
5580        let before_epoch = if commit_only {
5581            artifact_before_epoch
5582        } else {
5583            authority_before_epoch
5584        };
5585        let mut current = if commit_only {
5586            head.commit_head
5587        } else {
5588            head.outcome_head
5589        };
5590        let expected_anchor = if commit_only {
5591            floor.and_then(|floor| floor.committed_anchor_link)
5592        } else {
5593            floor.and_then(|floor| floor.terminal_anchor_link)
5594        };
5595        let expected_anchor_outcome = if commit_only {
5596            floor.and_then(|floor| floor.committed_anchor.as_ref())
5597        } else {
5598            floor.and_then(|floor| floor.terminal_anchor.as_ref())
5599        };
5600        let mut traversed = 0;
5601        while let Some(link) = current {
5602            if link.epoch < before_epoch {
5603                match (expected_anchor, expected_anchor_outcome) {
5604                    (Some(anchor_link), Some(anchor))
5605                        if link == anchor_link
5606                            && link.epoch == anchor.epoch
5607                            && link.checkpoint_id == anchor.checkpoint_id =>
5608                    {
5609                        return Ok(None)
5610                    }
5611                    (None, None) => return Ok(None),
5612                    _ => {
5613                        return Err(DecisionError::Conflict(format!(
5614                            "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
5615                            if commit_only { "Commit" } else { "terminal outcome" }
5616                        ))
5617                        .into());
5618                    }
5619                }
5620            }
5621            if link.epoch < epoch {
5622                return Ok(None);
5623            }
5624            if !consume_live_authority_link(&mut traversed) {
5625                return Err(DecisionError::Conflict(format!(
5626                    "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
5627                    if commit_only { "Commit" } else { "outcome" }
5628                ))
5629                .into());
5630            }
5631
5632            let admission;
5633            let record = if link.sequence == head.lease.seq {
5634                head
5635            } else {
5636                admission = read_authority_record(self.store.as_ref(), link.sequence)
5637                    .await?
5638                    .ok_or_else(|| {
5639                        DecisionError::InventoryChanged(format!(
5640                            "cluster {} authority record {} disappeared during exact lookup",
5641                            if commit_only { "Commit" } else { "outcome" },
5642                            link.sequence
5643                        ))
5644                    })?;
5645                &admission
5646            };
5647            let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
5648                DecisionError::Conflict(format!(
5649                    "cluster {} head epoch {} points to non-outcome authority record {}",
5650                    if commit_only { "Commit" } else { "outcome" },
5651                    link.epoch,
5652                    link.sequence
5653                ))
5654            })?;
5655            let record_head = if commit_only {
5656                record.commit_head
5657            } else {
5658                record.outcome_head
5659            };
5660            if record_head != Some(link)
5661                || (commit_only && !outcome.is_commit())
5662                || outcome.epoch != link.epoch
5663                || outcome.checkpoint_id != link.checkpoint_id
5664            {
5665                return Err(DecisionError::Conflict(format!(
5666                    "cluster {} link epoch {} sequence {} does not match its authority record",
5667                    if commit_only { "Commit" } else { "outcome" },
5668                    link.epoch,
5669                    link.sequence
5670                ))
5671                .into());
5672            }
5673            if outcome.epoch == epoch {
5674                let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5675                    .load_or_create_deployment_id()
5676                    .await?;
5677                if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
5678                    || outcome.deployment_id != expected_deployment
5679                {
5680                    return Err(DecisionError::Conflict(format!(
5681                        "cluster outcome epoch {} does not belong to current deployment {}",
5682                        outcome.epoch, expected_deployment
5683                    ))
5684                    .into());
5685                }
5686                return Ok(Some(outcome.clone()));
5687            }
5688            current = if commit_only {
5689                record.previous_commit
5690            } else {
5691                record.previous_outcome
5692            };
5693        }
5694        if expected_anchor.is_some() {
5695            return Err(DecisionError::Conflict(format!(
5696                "cluster {} chain stopped without its durable floor anchor",
5697                if commit_only { "Commit" } else { "outcome" }
5698            ))
5699            .into());
5700        }
5701        Ok(None)
5702    }
5703
5704    async fn stable_cluster_outcome(
5705        &self,
5706        epoch: u64,
5707    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5708        const LOOKUP_RETRIES: usize = 3;
5709        for attempt in 0..LOOKUP_RETRIES {
5710            let Some(head) = self.load_record().await? else {
5711                return Ok(None);
5712            };
5713            match self.cluster_outcome_from_snapshot(&head, epoch).await {
5714                Err(ClusterCheckpointAuthorityError::Decision(
5715                    DecisionError::InventoryChanged(_),
5716                )) if attempt + 1 < LOOKUP_RETRIES => {
5717                    tokio::task::yield_now().await;
5718                }
5719                result => return result,
5720            }
5721        }
5722        Err(DecisionError::InventoryChanged(
5723            "cluster outcome exact lookup exhausted stability retries".into(),
5724        )
5725        .into())
5726    }
5727
5728    /// Read one live cluster outcome from the shared authority.
5729    ///
5730    /// # Errors
5731    ///
5732    /// Fails when the durable authority history is unavailable or invalid.
5733    pub async fn cluster_outcome(
5734        &self,
5735        epoch: u64,
5736    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5737        self.stable_cluster_outcome(epoch).await
5738    }
5739
5740    /// Store one immutable committed-checkpoint index before publishing its Commit outcome.
5741    ///
5742    /// # Errors
5743    ///
5744    /// Fails when the durable authority cannot create the immutable index.
5745    pub async fn create_committed_checkpoint(
5746        &self,
5747        index: &CommittedCheckpointIndex,
5748    ) -> Result<CommittedCheckpointRef, ClusterCheckpointAuthorityError> {
5749        CheckpointDecisionStore::new(Arc::clone(&self.store))
5750            .create_committed_checkpoint(index)
5751            .await
5752            .map_err(Into::into)
5753    }
5754
5755    /// Load one exact content-addressed committed-checkpoint index.
5756    ///
5757    /// # Errors
5758    ///
5759    /// Fails when the durable authority cannot load or validate the index.
5760    pub async fn load_committed_checkpoint(
5761        &self,
5762        reference: &CommittedCheckpointRef,
5763    ) -> Result<CommittedCheckpointIndex, ClusterCheckpointAuthorityError> {
5764        CheckpointDecisionStore::new(Arc::clone(&self.store))
5765            .load_committed_checkpoint(reference)
5766            .await
5767            .map_err(Into::into)
5768    }
5769
5770    /// Read one live cluster outcome together with its committed checkpoint index.
5771    /// Commit always returns a validated index; Abort returns `None`.
5772    ///
5773    /// # Errors
5774    ///
5775    /// Fails when the authority history or selected committed index is unavailable or invalid.
5776    pub async fn cluster_outcome_with_committed_checkpoint(
5777        &self,
5778        epoch: u64,
5779    ) -> Result<
5780        Option<(CheckpointOutcome, Option<CommittedCheckpointIndex>)>,
5781        ClusterCheckpointAuthorityError,
5782    > {
5783        let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5784        let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5785            return Ok(None);
5786        };
5787        let index = if outcome.is_commit() {
5788            let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
5789                DecisionError::Conflict(format!(
5790                    "cluster Commit epoch {} checkpoint {} has no committed checkpoint index",
5791                    outcome.epoch, outcome.checkpoint_id
5792                ))
5793            })?;
5794            let index = decisions.load_committed_checkpoint(reference).await?;
5795            if index.epoch != outcome.epoch
5796                || index.checkpoint_id != outcome.checkpoint_id
5797                || index.scope != outcome.scope
5798                || index.assignment_fence.as_ref() != outcome.assignment_fence.as_ref()
5799                || index.deployment_id != outcome.deployment_id
5800            {
5801                return Err(DecisionError::Conflict(format!(
5802                    "cluster Commit epoch {} checkpoint {} does not match committed checkpoint '{}'",
5803                    outcome.epoch, outcome.checkpoint_id, reference.sha256
5804                ))
5805                .into());
5806            }
5807            Some(index)
5808        } else {
5809            None
5810        };
5811        Ok(Some((outcome, index)))
5812    }
5813
5814    /// Audit and return every live cluster outcome in ascending epoch order.
5815    ///
5816    /// # Errors
5817    ///
5818    /// Fails when the durable authority history is unavailable or invalid.
5819    pub async fn cluster_outcome_inventory(
5820        &self,
5821    ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5822        let (head, outcomes) = self.audited_cluster_outcomes().await?;
5823        Ok(Self::cluster_outcome_inventory_from_audit(
5824            head.as_ref(),
5825            &outcomes,
5826        ))
5827    }
5828
5829    fn cluster_outcome_inventory_from_audit(
5830        head: Option<&LeaderAuthorityRecord>,
5831        outcomes: &[CheckpointOutcome],
5832    ) -> ClusterOutcomeInventory {
5833        let floor = head.and_then(|head| head.outcome_floor.as_ref());
5834        let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5835        ClusterOutcomeInventory {
5836            outcomes: outcomes
5837                .iter()
5838                .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5839                .cloned()
5840                .collect(),
5841            retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5842        }
5843    }
5844
5845    /// Audit and return every live cluster outcome in ascending epoch order.
5846    ///
5847    /// # Errors
5848    ///
5849    /// Fails when the durable authority history is unavailable or invalid.
5850    pub async fn cluster_outcomes(
5851        &self,
5852    ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5853        Ok(self.cluster_outcome_inventory().await?.outcomes)
5854    }
5855
5856    /// Greatest live cluster commit recovery cut.
5857    ///
5858    /// # Errors
5859    ///
5860    /// Fails when the durable authority history is unavailable or invalid.
5861    pub async fn highest_cluster_committed_outcome(
5862        &self,
5863    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5864        Ok(self
5865            .cluster_outcomes()
5866            .await?
5867            .into_iter()
5868            .rev()
5869            .find(CheckpointOutcome::is_commit))
5870    }
5871
5872    /// Greatest terminal cluster outcome, including the compacted continuity anchor.
5873    ///
5874    /// # Errors
5875    ///
5876    /// Fails when the durable authority history is unavailable or invalid.
5877    pub async fn highest_cluster_terminal_outcome(
5878        &self,
5879    ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5880        Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5881    }
5882
5883    fn cleanup_participant_ids(
5884        index: &CommittedCheckpointIndex,
5885    ) -> Result<Vec<u64>, ClusterCheckpointAuthorityError> {
5886        index.validate().map_err(DecisionError::Conflict)?;
5887        let participant_ids = index
5888            .participants
5889            .iter()
5890            .map(|participant| participant.participant_id)
5891            .collect::<Vec<_>>();
5892        if participant_ids.is_empty()
5893            || participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
5894            || participant_ids[0] == 0
5895            || !participant_ids.windows(2).all(|pair| pair[0] < pair[1])
5896        {
5897            return Err(DecisionError::Conflict(
5898                "committed checkpoint participants are not canonical".into(),
5899            )
5900            .into());
5901        }
5902        Ok(participant_ids)
5903    }
5904
5905    fn cleanup_stop_before(head: &LeaderAuthorityRecord) -> Option<CommittedCheckpointRef> {
5906        head.outcome_floor
5907            .as_ref()
5908            .filter(|floor| floor.artifact_before_epoch != 0)
5909            .and_then(|floor| floor.committed_anchor.as_ref())
5910            .and_then(|outcome| outcome.committed_checkpoint.clone())
5911    }
5912
5913    /// Read the exact cluster checkpoint artifact cleanup position.
5914    ///
5915    /// The returned cursor was admitted through the shared leader-authority sequence. `None`
5916    /// means no destructive checkpoint cleanup is currently authorized.
5917    ///
5918    /// # Errors
5919    /// Fails when the durable authority head is unavailable or invalid.
5920    pub async fn cluster_artifact_cleanup(
5921        &self,
5922    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
5923        Ok(self
5924            .load_record()
5925            .await?
5926            .and_then(|head| head.artifact_cleanup))
5927    }
5928
5929    /// Atomically advance the artifact floor and authorize its first expired checkpoint.
5930    ///
5931    /// `protected` must name a live cluster Commit. Its exact predecessor becomes the first
5932    /// cleanup target. An already covered horizon or a protected genesis cut returns `None`.
5933    /// An identical retry returns the active cursor; a different active segment is rejected.
5934    ///
5935    /// # Errors
5936    /// Fails for a stale proof, invalid committed-index chain, failed protected-artifact
5937    /// validation, concurrent cleanup segment, or object-store failure.
5938    pub async fn begin_cluster_artifact_cleanup<V, Fut>(
5939        &self,
5940        proof: &LeaderProof,
5941        protected: CommittedCheckpointRef,
5942        validate_artifacts: V,
5943    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError>
5944    where
5945        V: Fn(CheckpointOutcome) -> Fut + Send + Sync + 'static,
5946        Fut: std::future::Future<Output = Result<(), String>> + Send + 'static,
5947    {
5948        if !proof.is_canonical() {
5949            return Err(ClusterCheckpointAuthorityError::Fenced);
5950        }
5951        protected.validate().map_err(DecisionError::Conflict)?;
5952        loop {
5953            let published = self
5954                .load_published_authority_head()
5955                .await?
5956                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5957            let current = &published.record;
5958            if !current.lease.matches_proof(proof) {
5959                return Err(ClusterCheckpointAuthorityError::Fenced);
5960            }
5961            if let Some(active) = current.artifact_cleanup.as_ref() {
5962                if active.protected == protected {
5963                    return Ok(Some(active.clone()));
5964                }
5965                return Err(DecisionError::Conflict(format!(
5966                    "cluster artifact cleanup for protected checkpoint {} is still active",
5967                    active.protected.checkpoint_id
5968                ))
5969                .into());
5970            }
5971            if current.assignment_handoff_pin.as_ref().is_some_and(|pin| {
5972                protected.epoch > pin.checkpoint.epoch
5973                    && protected.checkpoint_id > pin.checkpoint.checkpoint_id
5974            }) {
5975                return Ok(None);
5976            }
5977            if current
5978                .outcome_floor
5979                .as_ref()
5980                .is_some_and(|floor| floor.artifact_before_epoch >= protected.epoch)
5981            {
5982                return Ok(None);
5983            }
5984
5985            let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
5986            let protected_outcome = snapshot
5987                .outcomes
5988                .iter()
5989                .find(|outcome| {
5990                    outcome.is_commit() && outcome.committed_checkpoint.as_ref() == Some(&protected)
5991                })
5992                .cloned()
5993                .ok_or_else(|| {
5994                    DecisionError::Conflict(format!(
5995                        "protected checkpoint {} is not a live cluster Commit",
5996                        protected.checkpoint_id
5997                    ))
5998                })?;
5999            let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6000            let protected_index = decisions
6001                .validate_committed_checkpoint_for_outcome(&protected_outcome)
6002                .await?;
6003            validate_artifacts(protected_outcome.clone())
6004                .await
6005                .map_err(|error| {
6006                    DecisionError::Conflict(format!(
6007                        "protected cluster checkpoint {} failed durable artifact preflight: {error}",
6008                        protected.checkpoint_id
6009                    ))
6010                })?;
6011            let Some(expired) = protected_index.predecessor.clone() else {
6012                return Ok(None);
6013            };
6014            let expired_index = decisions.load_committed_checkpoint(&expired).await?;
6015            protected_index
6016                .validate_predecessor_index(&expired_index)
6017                .map_err(DecisionError::Conflict)?;
6018
6019            let authority_before_epoch = current
6020                .outcome_floor
6021                .as_ref()
6022                .map_or(protected.epoch, |floor| {
6023                    floor.authority_before_epoch.max(protected.epoch)
6024                });
6025            let floor = self
6026                .build_cluster_outcome_floor(
6027                    current,
6028                    &snapshot,
6029                    protected.epoch,
6030                    authority_before_epoch,
6031                )
6032                .await?;
6033            if floor
6034                .committed_anchor
6035                .as_ref()
6036                .and_then(|outcome| outcome.committed_checkpoint.as_ref())
6037                != Some(&expired)
6038            {
6039                return Err(DecisionError::Conflict(
6040                    "protected checkpoint predecessor is not the artifact-floor Commit anchor"
6041                        .into(),
6042                )
6043                .into());
6044            }
6045            let cursor = ClusterArtifactCleanupCursor {
6046                protected: protected.clone(),
6047                current: expired,
6048                next: expired_index.predecessor.clone(),
6049                stop_before: Self::cleanup_stop_before(current),
6050                participant_ids: Self::cleanup_participant_ids(&expired_index)?,
6051                phase: ClusterArtifactCleanupPhase::DeleteData,
6052            };
6053            cursor.validate()?;
6054
6055            // Artifact preflight and index reads may race harmless authority mutations. Publish
6056            // only from the exact outcome/floor/cursor snapshot used to derive this segment.
6057            let rechecked = self
6058                .load_published_authority_head()
6059                .await?
6060                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6061            if !rechecked.record.lease.matches_proof(proof) {
6062                return Err(ClusterCheckpointAuthorityError::Fenced);
6063            }
6064            if rechecked.record.outcome_head != current.outcome_head
6065                || rechecked.record.commit_head != current.commit_head
6066                || rechecked.record.outcome_floor != current.outcome_floor
6067                || rechecked.record.artifact_cleanup != current.artifact_cleanup
6068            {
6069                tokio::task::yield_now().await;
6070                continue;
6071            }
6072            let sequence =
6073                rechecked.record.lease.seq.checked_add(1).ok_or_else(|| {
6074                    LeaseError::Invalid("leader authority sequence exhausted".into())
6075                })?;
6076            let mut lease = rechecked.record.lease.clone();
6077            lease.seq = sequence;
6078            let mut next = rechecked.record.preserve_with_lease(lease);
6079            next.outcome_floor = Some(floor.clone());
6080            next.artifact_cleanup = Some(cursor.clone());
6081            next.validate()?;
6082            match self
6083                .create_authority_record(Some(&rechecked), &next)
6084                .await?
6085            {
6086                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6087                    self.install_cluster_outcome_audit(
6088                        Self::cluster_outcome_audit_key(&next),
6089                        next.lease.seq,
6090                        Self::outcomes_retained_by_floor(&floor, &snapshot),
6091                    );
6092                    return Ok(Some(cursor));
6093                }
6094                AuthorityCreateOutcome::Contended(winner) => {
6095                    if !winner.lease.matches_proof(proof) {
6096                        return Err(ClusterCheckpointAuthorityError::Fenced);
6097                    }
6098                    tokio::task::yield_now().await;
6099                }
6100            }
6101        }
6102    }
6103
6104    /// Durably authorize metadata deletion after the exact current data cleanup completes.
6105    ///
6106    /// # Errors
6107    /// Fails for a stale proof, stale or non-`DeleteData` cursor, or object-store failure.
6108    pub async fn mark_cluster_artifact_data_deleted(
6109        &self,
6110        proof: &LeaderProof,
6111        expected: &ClusterArtifactCleanupCursor,
6112    ) -> Result<ClusterArtifactCleanupCursor, ClusterCheckpointAuthorityError> {
6113        if !proof.is_canonical() {
6114            return Err(ClusterCheckpointAuthorityError::Fenced);
6115        }
6116        expected.validate()?;
6117        if expected.phase != ClusterArtifactCleanupPhase::DeleteData {
6118            return Err(DecisionError::Conflict(
6119                "cluster artifact data transition requires a DeleteData cursor".into(),
6120            )
6121            .into());
6122        }
6123        let mut replacement = expected.clone();
6124        replacement.phase = ClusterArtifactCleanupPhase::DeleteMetadata;
6125        loop {
6126            let published = self
6127                .load_published_authority_head()
6128                .await?
6129                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6130            let current = &published.record;
6131            if !current.lease.matches_proof(proof) {
6132                return Err(ClusterCheckpointAuthorityError::Fenced);
6133            }
6134            if current.artifact_cleanup.as_ref() == Some(&replacement) {
6135                return Ok(replacement);
6136            }
6137            if current.artifact_cleanup.as_ref() != Some(expected) {
6138                return Err(DecisionError::Conflict(
6139                    "cluster artifact cleanup cursor changed before data completion".into(),
6140                )
6141                .into());
6142            }
6143            let sequence =
6144                current.lease.seq.checked_add(1).ok_or_else(|| {
6145                    LeaseError::Invalid("leader authority sequence exhausted".into())
6146                })?;
6147            let mut lease = current.lease.clone();
6148            lease.seq = sequence;
6149            let mut next = current.preserve_with_lease(lease);
6150            next.artifact_cleanup = Some(replacement.clone());
6151            next.validate()?;
6152            match self
6153                .create_authority_record(Some(&published), &next)
6154                .await?
6155            {
6156                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6157                    return Ok(replacement);
6158                }
6159                AuthorityCreateOutcome::Contended(winner) => {
6160                    if !winner.lease.matches_proof(proof) {
6161                        return Err(ClusterCheckpointAuthorityError::Fenced);
6162                    }
6163                    tokio::task::yield_now().await;
6164                }
6165            }
6166        }
6167    }
6168
6169    /// Complete metadata deletion and authorize the exact next expired checkpoint, if any.
6170    ///
6171    /// The transition clears the journal at genesis or immediately before the previously
6172    /// reclaimed stop boundary. Otherwise it loads the immutable next index and records its exact
6173    /// predecessor and participants before any deletion of that target is allowed.
6174    ///
6175    /// # Errors
6176    /// Fails for a stale proof, stale or non-`DeleteMetadata` cursor, a broken predecessor chain,
6177    /// or object-store failure.
6178    pub async fn mark_cluster_artifact_metadata_deleted(
6179        &self,
6180        proof: &LeaderProof,
6181        expected: &ClusterArtifactCleanupCursor,
6182    ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
6183        if !proof.is_canonical() {
6184            return Err(ClusterCheckpointAuthorityError::Fenced);
6185        }
6186        expected.validate()?;
6187        if expected.phase != ClusterArtifactCleanupPhase::DeleteMetadata {
6188            return Err(DecisionError::Conflict(
6189                "cluster artifact metadata transition requires a DeleteMetadata cursor".into(),
6190            )
6191            .into());
6192        }
6193        loop {
6194            let published = self
6195                .load_published_authority_head()
6196                .await?
6197                .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6198            let current = &published.record;
6199            if !current.lease.matches_proof(proof) {
6200                return Err(ClusterCheckpointAuthorityError::Fenced);
6201            }
6202            if current.artifact_cleanup.as_ref() != Some(expected) {
6203                let completed = expected.next.is_none()
6204                    || expected.next.as_ref() == expected.stop_before.as_ref();
6205                if completed && current.artifact_cleanup.is_none() {
6206                    return Ok(None);
6207                }
6208                if let (Some(next_reference), Some(actual)) =
6209                    (expected.next.as_ref(), current.artifact_cleanup.as_ref())
6210                {
6211                    if actual.phase == ClusterArtifactCleanupPhase::DeleteData
6212                        && actual.protected == expected.protected
6213                        && &actual.current == next_reference
6214                        && actual.stop_before == expected.stop_before
6215                    {
6216                        return Ok(Some(actual.clone()));
6217                    }
6218                }
6219                return Err(DecisionError::Conflict(
6220                    "cluster artifact cleanup cursor changed before metadata completion".into(),
6221                )
6222                .into());
6223            }
6224
6225            let replacement = match expected.next.as_ref() {
6226                None => None,
6227                Some(next_reference) if Some(next_reference) == expected.stop_before.as_ref() => {
6228                    None
6229                }
6230                Some(next_reference) => {
6231                    let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6232                    let index = decisions.load_committed_checkpoint(next_reference).await?;
6233                    if index.scope != CheckpointScope::Cluster {
6234                        return Err(DecisionError::Conflict(
6235                            "cluster artifact cleanup next checkpoint is not cluster-scoped".into(),
6236                        )
6237                        .into());
6238                    }
6239                    let replacement = ClusterArtifactCleanupCursor {
6240                        protected: expected.protected.clone(),
6241                        current: next_reference.clone(),
6242                        next: index.predecessor.clone(),
6243                        stop_before: expected.stop_before.clone(),
6244                        participant_ids: Self::cleanup_participant_ids(&index)?,
6245                        phase: ClusterArtifactCleanupPhase::DeleteData,
6246                    };
6247                    replacement.validate()?;
6248                    Some(replacement)
6249                }
6250            };
6251
6252            let sequence =
6253                current.lease.seq.checked_add(1).ok_or_else(|| {
6254                    LeaseError::Invalid("leader authority sequence exhausted".into())
6255                })?;
6256            let mut lease = current.lease.clone();
6257            lease.seq = sequence;
6258            let mut next = current.preserve_with_lease(lease);
6259            next.artifact_cleanup = replacement.clone();
6260            next.validate()?;
6261            match self
6262                .create_authority_record(Some(&published), &next)
6263                .await?
6264            {
6265                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6266                    return Ok(replacement);
6267                }
6268                AuthorityCreateOutcome::Contended(winner) => {
6269                    if !winner.lease.matches_proof(proof) {
6270                        return Err(ClusterCheckpointAuthorityError::Fenced);
6271                    }
6272                    tokio::task::yield_now().await;
6273                }
6274            }
6275        }
6276    }
6277
6278    /// Exact continuity boundary for cluster outcomes compacted from the authority history.
6279    ///
6280    /// # Errors
6281    ///
6282    /// Fails when the durable authority history is unavailable or invalid.
6283    pub async fn cluster_outcome_retention_boundary(
6284        &self,
6285    ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
6286        let head = self.load_record().await?;
6287        Ok(ClusterOutcomeRetentionBoundary::from_floor(
6288            head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
6289        ))
6290    }
6291
6292    /// Acquire an empty authority as a new term, or rotate the fencing token when this exact
6293    /// process incarnation already owns the durable head. Rival wall clocks never authorize
6294    /// takeover.
6295    ///
6296    /// # Errors
6297    /// Fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
6298    pub async fn begin_new_term(
6299        &self,
6300        owner: &LeaderLeaseOwner,
6301        now_ms: i64,
6302    ) -> Result<LeaseOutcome, LeaseError> {
6303        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6304            owner,
6305            now_ms,
6306            SameOwnerToken::Rotate,
6307        ))
6308        .await
6309    }
6310
6311    /// Renew only the exact durable authority term identified by `token`.
6312    ///
6313    /// # Errors
6314    /// Returns [`LeaseError::Fenced`] if the durable head is absent or belongs to another owner or
6315    /// term. Also fails closed on invalid input, object-store I/O, or arithmetic exhaustion.
6316    pub async fn renew_exact(
6317        &self,
6318        owner: &LeaderLeaseOwner,
6319        token: u64,
6320        now_ms: i64,
6321    ) -> Result<LeaseOutcome, LeaseError> {
6322        Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6323            owner,
6324            now_ms,
6325            SameOwnerToken::Exact(token),
6326        ))
6327        .await
6328    }
6329
6330    #[cfg(test)]
6331    pub(crate) async fn acquire_or_renew_current_term_for_test(
6332        &self,
6333        owner: &LeaderLeaseOwner,
6334        now_ms: i64,
6335    ) -> Result<LeaseOutcome, LeaseError> {
6336        self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
6337            .await
6338    }
6339
6340    async fn acquire_or_renew_current_term_for_test_inner(
6341        &self,
6342        owner: &LeaderLeaseOwner,
6343        now_ms: i64,
6344        same_owner_token: SameOwnerToken,
6345    ) -> Result<LeaseOutcome, LeaseError> {
6346        owner.validate()?;
6347        self.ttl()?;
6348        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6349        loop {
6350            let published = self.load_published_authority_head().await?;
6351            let candidate = match published.as_ref() {
6352                None => {
6353                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6354                        return Err(LeaseError::Fenced(
6355                            "exact leader renewal lost the durable authority head".into(),
6356                        ));
6357                    }
6358                    LeaderAuthorityRecord::initial(LeaderLease {
6359                        seq: 1,
6360                        renewal_sequence: 1,
6361                        token: 1,
6362                        owner: owner.clone(),
6363                        expires_at_ms,
6364                        catalog_manifest: None,
6365                    })
6366                }
6367                Some(head) if head.record.lease.owner == *owner => {
6368                    let record = &head.record;
6369                    let token = match same_owner_token {
6370                        #[cfg(test)]
6371                        SameOwnerToken::Preserve => record.lease.token,
6372                        SameOwnerToken::Rotate => {
6373                            record.lease.token.checked_add(1).ok_or_else(|| {
6374                                LeaseError::Invalid("leader fencing token exhausted".into())
6375                            })?
6376                        }
6377                        SameOwnerToken::Exact(expected) if expected == record.lease.token => {
6378                            expected
6379                        }
6380                        SameOwnerToken::Exact(_) => {
6381                            return Err(LeaseError::Fenced(
6382                                "exact leader renewal was superseded by a newer local term".into(),
6383                            ));
6384                        }
6385                    };
6386                    let lease = LeaderLease {
6387                        seq: record.lease.seq.checked_add(1).ok_or_else(|| {
6388                            LeaseError::Invalid("lease sequence exhausted".into())
6389                        })?,
6390                        renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
6391                            || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6392                        )?,
6393                        token,
6394                        owner: owner.clone(),
6395                        expires_at_ms,
6396                        catalog_manifest: record.lease.catalog_manifest.clone(),
6397                    };
6398                    record.preserve_with_lease(lease)
6399                }
6400                Some(head) => {
6401                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6402                        return Err(LeaseError::Fenced(
6403                            "exact leader renewal was superseded by a rival owner".into(),
6404                        ));
6405                    }
6406                    return Ok(LeaseOutcome::Held(head.record.lease.clone()));
6407                }
6408            };
6409            match self
6410                .create_authority_record(published.as_ref(), &candidate)
6411                .await?
6412            {
6413                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6414                    return Ok(LeaseOutcome::Acquired(candidate.lease));
6415                }
6416                AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
6417                    if matches!(
6418                        same_owner_token,
6419                        SameOwnerToken::Exact(expected) if winner.lease.token != expected
6420                    ) {
6421                        return Err(LeaseError::Fenced(
6422                            "exact leader renewal lost a same-owner CAS race".into(),
6423                        ));
6424                    }
6425                    tokio::task::yield_now().await;
6426                }
6427                AuthorityCreateOutcome::Contended(winner) => {
6428                    if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6429                        return Err(LeaseError::Fenced(
6430                            "exact leader renewal lost a rival-owner CAS race".into(),
6431                        ));
6432                    }
6433                    return Ok(LeaseOutcome::Held(winner.lease));
6434                }
6435            }
6436        }
6437    }
6438
6439    /// Start a candidate-local observation of a rival durable liveness identity.
6440    ///
6441    /// # Errors
6442    /// Rejects malformed state or an observation of the candidate itself.
6443    pub fn observe_rival(
6444        &self,
6445        owner: &LeaderLeaseOwner,
6446        lease: &LeaderLease,
6447    ) -> Result<LeaderLeaseObservation, LeaseError> {
6448        owner.validate()?;
6449        lease.validate()?;
6450        self.ttl()?;
6451        if lease.owner == *owner {
6452            return Err(LeaseError::Invalid(
6453                "leader takeover observation must belong to a rival".into(),
6454            ));
6455        }
6456        Ok(LeaderLeaseObservation {
6457            lease: lease.clone(),
6458            started: Instant::now(),
6459        })
6460    }
6461
6462    /// Take over only after the rival's owner, fencing token, and renewal sequence remained
6463    /// current for a full TTL on the candidate's monotonic clock.
6464    ///
6465    /// # Errors
6466    /// Fails closed on early observation, invalid state, I/O, or arithmetic exhaustion.
6467    pub async fn try_takeover(
6468        &self,
6469        owner: &LeaderLeaseOwner,
6470        observation: &LeaderLeaseObservation,
6471        now_ms: i64,
6472    ) -> Result<LeaseOutcome, LeaseError> {
6473        owner.validate()?;
6474        observation.lease.validate()?;
6475        let ttl = self.ttl()?;
6476        if observation.lease.owner == *owner {
6477            return Err(LeaseError::Invalid(
6478                "leader takeover observation must belong to a rival".into(),
6479            ));
6480        }
6481        if observation.started.elapsed() < ttl {
6482            return Ok(LeaseOutcome::Held(observation.lease.clone()));
6483        }
6484        let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6485        loop {
6486            let published = self
6487                .load_published_authority_head()
6488                .await?
6489                .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
6490            let current = &published.record;
6491            if !current.lease.has_same_liveness_identity(&observation.lease) {
6492                return Ok(LeaseOutcome::Held(current.lease.clone()));
6493            }
6494            let candidate_lease = LeaderLease {
6495                seq: current
6496                    .lease
6497                    .seq
6498                    .checked_add(1)
6499                    .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
6500                renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
6501                    || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6502                )?,
6503                token: current
6504                    .lease
6505                    .token
6506                    .checked_add(1)
6507                    .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
6508                owner: owner.clone(),
6509                expires_at_ms,
6510                catalog_manifest: current.lease.catalog_manifest.clone(),
6511            };
6512            let candidate = current.preserve_with_lease(candidate_lease);
6513            match self
6514                .create_authority_record(Some(&published), &candidate)
6515                .await?
6516            {
6517                AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6518                    return Ok(LeaseOutcome::Acquired(candidate.lease));
6519                }
6520                AuthorityCreateOutcome::Contended(winner)
6521                    if winner.lease.has_same_liveness_identity(&observation.lease) =>
6522                {
6523                    tokio::task::yield_now().await;
6524                }
6525                AuthorityCreateOutcome::Contended(winner) => {
6526                    return Ok(LeaseOutcome::Held(winner.lease));
6527                }
6528            }
6529        }
6530    }
6531}
6532
6533/// Renewal timings for the leader lease.
6534#[derive(Debug, Clone, Copy)]
6535pub struct LeaderLeaseConfig {
6536    /// Lease lifetime.
6537    pub ttl: Duration,
6538    /// Renewal cadence, strictly below the lifetime.
6539    pub renew_interval: Duration,
6540}
6541
6542impl Default for LeaderLeaseConfig {
6543    fn default() -> Self {
6544        Self {
6545            // Keep one full five-second control-I/O window after the renewal tick. Native cloud
6546            // stores can legitimately consume that window while applying their bounded retry
6547            // policy; expiring sooner would turn a transport tail into a needless leader term.
6548            ttl: Duration::from_secs(15),
6549            renew_interval: Duration::from_secs(5),
6550        }
6551    }
6552}
6553
6554/// Whether the exact owner has a durable record and a live process-local deadline.
6555#[must_use]
6556pub fn lease_grants_leadership(
6557    lease: &Option<LeaderLease>,
6558    owner: &LeaderLeaseOwner,
6559    deadline: &LeaseDeadline,
6560) -> bool {
6561    deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
6562}
6563
6564/// Whether a captured proof still matches the current exact grant and local deadline.
6565#[must_use]
6566pub fn lease_grants_proof(
6567    lease: &Option<LeaderLease>,
6568    owner: &LeaderLeaseOwner,
6569    deadline: &LeaseDeadline,
6570    proof: &LeaderProof,
6571) -> bool {
6572    deadline.is_live()
6573        && proof.owner == owner.proof_owner()
6574        && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
6575}
6576
6577/// Acquires and renews leadership while candidacy remains true.
6578pub struct LeaderLeaseManager {
6579    store: Arc<LeaderLeaseStore>,
6580    owner: LeaderLeaseOwner,
6581    config: LeaderLeaseConfig,
6582    lease_tx: watch::Sender<Option<LeaderLease>>,
6583    deadline_tx: watch::Sender<Arc<LeaseDeadline>>,
6584}
6585
6586#[cfg(feature = "cluster")]
6587enum LeaseOperationEvent {
6588    Shutdown,
6589    Candidacy(Result<(), watch::error::RecvError>),
6590    Deadline,
6591    Completed {
6592        result: Result<LeaseOutcome, LeaseError>,
6593        valid_until: tokio::time::Instant,
6594    },
6595}
6596
6597#[cfg(feature = "cluster")]
6598struct LeaderLeaseExitGuard {
6599    shutdown: tokio_util::sync::CancellationToken,
6600    unexpected_exit: tokio_util::sync::CancellationToken,
6601    lease_tx: watch::Sender<Option<LeaderLease>>,
6602    deadline_tx: watch::Sender<Arc<LeaseDeadline>>,
6603}
6604
6605#[cfg(feature = "cluster")]
6606impl Drop for LeaderLeaseExitGuard {
6607    fn drop(&mut self) {
6608        self.deadline_tx.borrow().fence();
6609        self.lease_tx.send_replace(None);
6610        if !self.shutdown.is_cancelled() {
6611            tracing::error!(
6612                "leader lease manager exited without intentional shutdown; fencing process authority"
6613            );
6614            self.unexpected_exit.cancel();
6615        }
6616    }
6617}
6618
6619#[cfg(feature = "cluster")]
6620async fn wait_for_deadline(deadline: Option<tokio::time::Instant>) {
6621    if let Some(deadline) = deadline {
6622        tokio::time::sleep_until(deadline).await;
6623    } else {
6624        std::future::pending::<()>().await;
6625    }
6626}
6627
6628impl std::fmt::Debug for LeaderLeaseManager {
6629    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
6630        formatter
6631            .debug_struct("LeaderLeaseManager")
6632            .field("owner", &self.owner)
6633            .field("config", &self.config)
6634            .finish_non_exhaustive()
6635    }
6636}
6637
6638impl LeaderLeaseManager {
6639    /// Construct a manager bound to this boot's acquired stable-node process lease.
6640    ///
6641    /// # Errors
6642    /// Rejects invalid ownership or inconsistent renewal timings.
6643    pub fn new(
6644        store: Arc<LeaderLeaseStore>,
6645        process_lease: &ProcessLease,
6646        config: LeaderLeaseConfig,
6647    ) -> Result<Self, LeaseError> {
6648        let owner = LeaderLeaseOwner::from_process_lease(process_lease)?;
6649        let ttl_ms = i64::try_from(config.ttl.as_millis())
6650            .map_err(|_| LeaseError::Invalid("lease TTL exceeds diagnostic range".into()))?;
6651        let exact_ttl = u64::try_from(ttl_ms)
6652            .ok()
6653            .map(Duration::from_millis)
6654            .is_some_and(|ttl| ttl == config.ttl);
6655        if ttl_ms <= 0
6656            || !exact_ttl
6657            || config.renew_interval.is_zero()
6658            || config.renew_interval >= config.ttl
6659            || ttl_ms != store.ttl_ms
6660        {
6661            return Err(LeaseError::Invalid(
6662                "manager requires a renewal interval below the store's matching TTL".into(),
6663            ));
6664        }
6665        let (lease_tx, _lease_rx) = watch::channel(None);
6666        let (deadline_tx, _deadline_rx) = watch::channel(Arc::new(LeaseDeadline::uninitialized()));
6667        Ok(Self {
6668            store,
6669            owner,
6670            config,
6671            lease_tx,
6672            deadline_tx,
6673        })
6674    }
6675
6676    /// Exact process incarnation this manager may publish.
6677    #[must_use]
6678    pub fn owner(&self) -> &LeaderLeaseOwner {
6679        &self.owner
6680    }
6681
6682    /// Subscribe to the locally authorized leader record.
6683    #[must_use]
6684    pub fn lease_watch(&self) -> watch::Receiver<Option<LeaderLease>> {
6685        self.lease_tx.subscribe()
6686    }
6687
6688    /// Snapshot the current local-monotonic deadline generation.
6689    ///
6690    /// This is useful before the manager starts and in fixed-generation tests. A running manager
6691    /// may replace it after any discontinuous grant; runtime consumers must subscribe through
6692    /// [`Self::deadline_watch`].
6693    #[must_use]
6694    pub fn deadline(&self) -> Arc<LeaseDeadline> {
6695        self.deadline_tx.borrow().clone()
6696    }
6697
6698    /// Subscribe to generation-scoped local leader deadlines.
6699    ///
6700    /// A discontinuous grant permanently fences the previous deadline and publishes a fresh
6701    /// inactive one before another durable fencing term can become locally authoritative.
6702    #[must_use]
6703    pub fn deadline_watch(&self) -> watch::Receiver<Arc<LeaseDeadline>> {
6704        self.deadline_tx.subscribe()
6705    }
6706
6707    #[cfg(feature = "cluster")]
6708    fn withdraw(&self) -> bool {
6709        let deadline = self.deadline_tx.borrow().clone();
6710        let had_grant = self.lease_tx.borrow().is_some() || deadline.is_live();
6711        // Always rotate the generation. An acquisition may have crossed its deadline after the
6712        // durable response but before local publication, leaving an expired, never-published
6713        // deadline that `is_live()` cannot distinguish from a fresh inactive one.
6714        deadline.fence();
6715        self.lease_tx.send_replace(None);
6716        self.deadline_tx
6717            .send_replace(Arc::new(LeaseDeadline::uninitialized()));
6718        had_grant
6719    }
6720
6721    #[cfg(feature = "cluster")]
6722    fn fence(&self) {
6723        self.deadline_tx.borrow().fence();
6724        self.lease_tx.send_replace(None);
6725    }
6726
6727    #[cfg(feature = "cluster")]
6728    fn withdraw_for_new_term(
6729        &self,
6730        ticker: &mut tokio::time::Interval,
6731        valid_until: &mut Option<tokio::time::Instant>,
6732        observation: &mut Option<LeaderLeaseObservation>,
6733        held_token: &mut Option<u64>,
6734        rotate_not_before: &mut Option<tokio::time::Instant>,
6735    ) {
6736        // Permanently fence the process-local grant before forgetting its fencing token. A later
6737        // acquisition runs through `begin_new_term` and a fresh deadline generation, so neither
6738        // half of the expired proof can become live again.
6739        let had_grant = self.withdraw();
6740        *valid_until = None;
6741        *observation = None;
6742        *held_token = None;
6743        let now = tokio::time::Instant::now();
6744        if had_grant {
6745            let retry_at = now
6746                .checked_add(self.config.ttl)
6747                .unwrap_or_else(tokio::time::Instant::now);
6748            // An old-proof checkpoint tail is still allowed to win the authority CAS while no
6749            // newer term exists. Preserve one full TTL for that bounded settlement before the
6750            // same process rotates the durable token.
6751            if rotate_not_before.is_none_or(|current| retry_at > current) {
6752                *rotate_not_before = Some(retry_at);
6753            }
6754        }
6755        if let Some(retry_at) = *rotate_not_before {
6756            if retry_at > now {
6757                ticker.reset_at(retry_at);
6758            } else {
6759                *rotate_not_before = None;
6760                ticker.reset_immediately();
6761            }
6762        } else {
6763            ticker.reset_immediately();
6764        }
6765    }
6766
6767    #[cfg(feature = "cluster")]
6768    async fn attempt_lease(
6769        &self,
6770        shutdown: &tokio_util::sync::CancellationToken,
6771        candidate: &mut watch::Receiver<LeaderCandidacy>,
6772        valid_until: Option<tokio::time::Instant>,
6773        observation: Option<&LeaderLeaseObservation>,
6774        held_token: Option<u64>,
6775    ) -> LeaseOperationEvent {
6776        let Some(attempt_valid_until) = tokio::time::Instant::now().checked_add(self.config.ttl)
6777        else {
6778            return LeaseOperationEvent::Deadline;
6779        };
6780        let operation_deadline = valid_until.map_or(attempt_valid_until, |current| {
6781            current.min(attempt_valid_until)
6782        });
6783        let operation = async {
6784            if let Some(observation) = observation {
6785                Box::pin(
6786                    self.store
6787                        .try_takeover(&self.owner, observation, now_millis()),
6788                )
6789                .await
6790            } else if let Some(token) = held_token {
6791                Box::pin(self.store.renew_exact(&self.owner, token, now_millis())).await
6792            } else {
6793                Box::pin(self.store.begin_new_term(&self.owner, now_millis())).await
6794            }
6795        };
6796        tokio::select! {
6797            biased;
6798            () = shutdown.cancelled() => LeaseOperationEvent::Shutdown,
6799            changed = candidate.changed() => LeaseOperationEvent::Candidacy(changed),
6800            () = wait_for_deadline(Some(operation_deadline)) => LeaseOperationEvent::Deadline,
6801            result = operation => LeaseOperationEvent::Completed {
6802                result,
6803                valid_until: attempt_valid_until,
6804            },
6805        }
6806    }
6807
6808    #[cfg(feature = "cluster")]
6809    async fn wait_for_candidacy_change(
6810        &self,
6811        shutdown: &tokio_util::sync::CancellationToken,
6812        candidate: &mut watch::Receiver<LeaderCandidacy>,
6813    ) -> bool {
6814        tokio::select! {
6815            biased;
6816            () = shutdown.cancelled() => {
6817                self.fence();
6818                false
6819            }
6820            changed = candidate.changed() => {
6821                if changed.is_err() {
6822                    self.fence();
6823                    return false;
6824                }
6825                true
6826            }
6827        }
6828    }
6829
6830    #[cfg(feature = "cluster")]
6831    async fn run(
6832        self,
6833        shutdown: tokio_util::sync::CancellationToken,
6834        mut candidate: watch::Receiver<LeaderCandidacy>,
6835    ) {
6836        let mut ticker = tokio::time::interval(self.config.renew_interval);
6837        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6838        let mut valid_until: Option<tokio::time::Instant> = None;
6839        let mut observation: Option<LeaderLeaseObservation> = None;
6840        // A newly constructed manager and every locally withdrawn grant need a new durable
6841        // fencing token. Only uninterrupted renewals may preserve the current token.
6842        let mut held_token = None;
6843        let mut rotate_not_before = None;
6844        let mut candidacy_generation = candidate.borrow().generation;
6845
6846        loop {
6847            let candidacy = *candidate.borrow_and_update();
6848            if candidacy.generation != candidacy_generation {
6849                self.withdraw_for_new_term(
6850                    &mut ticker,
6851                    &mut valid_until,
6852                    &mut observation,
6853                    &mut held_token,
6854                    &mut rotate_not_before,
6855                );
6856                candidacy_generation = candidacy.generation;
6857            }
6858            if !candidacy.eligible {
6859                self.withdraw_for_new_term(
6860                    &mut ticker,
6861                    &mut valid_until,
6862                    &mut observation,
6863                    &mut held_token,
6864                    &mut rotate_not_before,
6865                );
6866                if !self
6867                    .wait_for_candidacy_change(&shutdown, &mut candidate)
6868                    .await
6869                {
6870                    return;
6871                }
6872                continue;
6873            }
6874
6875            tokio::select! {
6876                biased;
6877                () = shutdown.cancelled() => {
6878                    self.fence();
6879                    return;
6880                }
6881                changed = candidate.changed() => {
6882                    if changed.is_err() {
6883                        self.fence();
6884                        return;
6885                    }
6886                    continue;
6887                }
6888                () = wait_for_deadline(valid_until) => {
6889                    tracing::warn!(
6890                        owner = ?self.owner,
6891                        "leader lease local deadline expired; withdrawing and rotating the fencing token"
6892                    );
6893                    self.withdraw_for_new_term(
6894                        &mut ticker,
6895                        &mut valid_until,
6896                        &mut observation,
6897                        &mut held_token,
6898                        &mut rotate_not_before,
6899                    );
6900                    continue;
6901                }
6902                _ = ticker.tick() => {}
6903            }
6904
6905            let (result, attempt_valid_until) = match Box::pin(self.attempt_lease(
6906                &shutdown,
6907                &mut candidate,
6908                valid_until,
6909                observation.as_ref(),
6910                held_token,
6911            ))
6912            .await
6913            {
6914                LeaseOperationEvent::Shutdown => {
6915                    self.fence();
6916                    return;
6917                }
6918                LeaseOperationEvent::Deadline => {
6919                    tracing::warn!(
6920                        owner = ?self.owner,
6921                        "leader lease operation exceeded its local deadline; withdrawing and rotating the fencing token"
6922                    );
6923                    self.withdraw_for_new_term(
6924                        &mut ticker,
6925                        &mut valid_until,
6926                        &mut observation,
6927                        &mut held_token,
6928                        &mut rotate_not_before,
6929                    );
6930                    continue;
6931                }
6932                LeaseOperationEvent::Candidacy(changed) => {
6933                    if changed.is_err() {
6934                        self.fence();
6935                        return;
6936                    }
6937                    continue;
6938                }
6939                LeaseOperationEvent::Completed {
6940                    result,
6941                    valid_until: attempt_valid_until,
6942                } => {
6943                    let response_at = tokio::time::Instant::now();
6944                    if response_at >= attempt_valid_until
6945                        || valid_until.is_some_and(|current| response_at >= current)
6946                    {
6947                        tracing::warn!(
6948                            owner = ?self.owner,
6949                            "leader lease response arrived after its local deadline; withdrawing and rotating the fencing token"
6950                        );
6951                        self.withdraw_for_new_term(
6952                            &mut ticker,
6953                            &mut valid_until,
6954                            &mut observation,
6955                            &mut held_token,
6956                            &mut rotate_not_before,
6957                        );
6958                        continue;
6959                    }
6960                    (result, attempt_valid_until)
6961                }
6962            };
6963
6964            match result {
6965                Ok(LeaseOutcome::Acquired(lease)) if lease.owner == self.owner => {
6966                    let publication_at = tokio::time::Instant::now();
6967                    if publication_at >= attempt_valid_until
6968                        || valid_until.is_some_and(|current| publication_at >= current)
6969                    {
6970                        tracing::warn!(
6971                            owner = ?self.owner,
6972                            "leader lease publication crossed its local deadline; withdrawing and rotating the fencing token"
6973                        );
6974                        self.withdraw_for_new_term(
6975                            &mut ticker,
6976                            &mut valid_until,
6977                            &mut observation,
6978                            &mut held_token,
6979                            &mut rotate_not_before,
6980                        );
6981                        continue;
6982                    }
6983                    observation = None;
6984                    valid_until = Some(attempt_valid_until);
6985                    held_token = Some(lease.token);
6986                    rotate_not_before = None;
6987                    let deadline = self.deadline_tx.borrow().clone();
6988                    deadline.extend_until(attempt_valid_until.into_std());
6989                    if !deadline.is_live() {
6990                        tracing::warn!(
6991                            owner = ?self.owner,
6992                            "leader lease deadline expired during local publication; withdrawing and rotating the fencing token"
6993                        );
6994                        self.withdraw_for_new_term(
6995                            &mut ticker,
6996                            &mut valid_until,
6997                            &mut observation,
6998                            &mut held_token,
6999                            &mut rotate_not_before,
7000                        );
7001                        continue;
7002                    }
7003                    self.lease_tx.send_replace(Some(lease));
7004                }
7005                Ok(LeaseOutcome::Acquired(_)) => {
7006                    self.fence();
7007                    return;
7008                }
7009                Ok(LeaseOutcome::Held(rival)) => {
7010                    self.withdraw();
7011                    valid_until = None;
7012                    held_token = None;
7013                    let unchanged = observation
7014                        .as_ref()
7015                        .is_some_and(|observed| observed.lease.has_same_liveness_identity(&rival));
7016                    if !unchanged {
7017                        match self.store.observe_rival(&self.owner, &rival) {
7018                            Ok(new_observation) => observation = Some(new_observation),
7019                            Err(error) => {
7020                                tracing::warn!(%error, "leader lease observation rejected");
7021                                observation = None;
7022                            }
7023                        }
7024                    }
7025                }
7026                Err(LeaseError::Fenced(error)) => {
7027                    tracing::warn!(%error, "leader lease renewal was fenced");
7028                    self.fence();
7029                    return;
7030                }
7031                Err(error) => {
7032                    tracing::warn!(%error, "leader lease operation failed");
7033                }
7034            }
7035        }
7036    }
7037
7038    /// Spawn the renewal loop. Loss of candidacy withdraws the current local grant so this
7039    /// manager can contend again later. A missed local deadline withdraws the grant and rotates
7040    /// its fencing token before reacquisition. Shutdown or an invalid lease outcome terminally
7041    /// fences the manager.
7042    #[cfg(feature = "cluster")]
7043    #[must_use]
7044    pub fn spawn(
7045        self,
7046        shutdown: tokio_util::sync::CancellationToken,
7047        candidate: watch::Receiver<LeaderCandidacy>,
7048    ) -> tokio::task::JoinHandle<()> {
7049        tokio::spawn(self.run(shutdown, candidate))
7050    }
7051
7052    /// Spawn the renewal loop and signal any exit not preceded by intentional shutdown.
7053    ///
7054    /// The guard lives inside the same task as the manager future, so panic and forced task abort
7055    /// cannot detach a still-running inner lease writer.
7056    #[cfg(feature = "cluster")]
7057    #[must_use]
7058    pub fn spawn_supervised(
7059        self,
7060        shutdown: tokio_util::sync::CancellationToken,
7061        candidate: watch::Receiver<LeaderCandidacy>,
7062        unexpected_exit: tokio_util::sync::CancellationToken,
7063    ) -> tokio::task::JoinHandle<()> {
7064        tokio::spawn(async move {
7065            let run_shutdown = shutdown.clone();
7066            let _exit_guard = LeaderLeaseExitGuard {
7067                shutdown,
7068                unexpected_exit,
7069                lease_tx: self.lease_tx.clone(),
7070                deadline_tx: self.deadline_tx.clone(),
7071            };
7072            self.run(run_shutdown, candidate).await;
7073        })
7074    }
7075}
7076
7077#[cfg(test)]
7078mod tests;