1mod artifact_admission;
4mod assignment_drain;
5mod attempt_status;
6mod subscription_replay;
7mod topology;
8mod topology_activation;
9mod topology_admission;
10mod topology_checkpoint;
11mod topology_commit;
12mod topology_committed_restore;
13mod topology_cut;
14mod topology_migration_root;
15mod topology_preparation;
16mod topology_recovery;
17mod topology_recovery_round;
18mod topology_restore;
19mod topology_retention;
20mod topology_subscription;
21mod topology_target_preparation;
22
23mod checkpoint_publication;
24mod history_retention;
25mod manager;
26mod release;
27
28pub use attempt_status::{ClusterAttemptStatus, ClusterCommitHead};
29pub use manager::LeaderLeaseManager;
30pub use subscription_replay::{
31 SubscriptionReplayPin, SubscriptionReplayPinAcquire, SUBSCRIPTION_REPLAY_PIN_RENEW_INTERVAL,
32};
33
34use std::collections::{BTreeMap, BTreeSet};
35use std::sync::atomic::{AtomicBool, Ordering};
36use std::sync::{Arc, Weak};
37use std::time::{Duration, Instant};
38
39use bytes::Bytes;
40use object_store::path::Path as OsPath;
41use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload, UpdateVersion};
42use parking_lot::Mutex;
43use serde::{Deserialize, Serialize};
44use sha2::{Digest as _, Sha256};
45use tokio::sync::watch;
46use tokio_stream::StreamExt;
47use uuid::Uuid;
48
49use crate::checkpoint::{
50 probe_object_store_conditional_update, AssignmentDrainId, AssignmentDrainTransition,
51 CheckpointAssignmentFence, CheckpointStoreError, CommittedCheckpointIndex,
52 CommittedCheckpointRef, LeaderProof, LeaderProofOwner, MAX_CHECKPOINT_PARTICIPANTS,
53};
54use crate::checkpoint_decision::{
55 CheckpointArtifactInventory, CheckpointDecisionStore, CheckpointOutcome, CheckpointScope,
56 CheckpointVerdict, DecisionError, RecordOutcomeResult,
57};
58use crate::cluster::discovery::NodeId;
59
60use super::catalog_manifest::{
61 CatalogManifest, CatalogManifestError, CatalogManifestRef, CatalogSealOutcome,
62};
63use super::controller::{
64 RecoverPhase, RecoveryAdmissionSnapshot, RecoveryAnnouncement, RecoveryFault,
65 RecoveryFaultDisposition, RecoveryFaultInventory, RecoveryFaultPublisher, RecoveryReleaseId,
66 MAX_RECOVERY_ANNOUNCEMENT_BYTES,
67};
68use super::lease_deadline::LeaseDeadline;
69use super::process_lease::{ProcessLease, ProcessLeaseFence};
70use super::snapshot::{
71 AssignmentSnapshotRef, AssignmentSnapshotStore, RotateOutcome, SnapshotError,
72};
73
74const LEASE_PREFIX: &str = "control/leader-lease/";
75const AUTHORITY_HEAD_PATH: &str = "control/leader-lease-head/v1.json";
76const STORE_CONTRACT_PROBE_PREFIX: &str = "control/object-store-contract-probes/v1/";
77const RECOVERY_RELEASE_TERMINAL_PREFIX: &str = "control/recovery-release-terminals/v2/";
78const AUTHORITY_RECORD_VERSION: u32 = 12;
79const TOPOLOGY_AUTHORITY_RECORD_VERSION: u32 = 13;
80const TOPOLOGY_ADMISSION_RECORD_VERSION: u32 = 14;
81const TOPOLOGY_CUT_RECORD_VERSION: u32 = 15;
82const TOPOLOGY_PREPARATION_RECORD_VERSION: u32 = 16;
83const TOPOLOGY_MIGRATION_ROOT_RECORD_VERSION: u32 = 17;
84const TOPOLOGY_SOURCE_ROOT_RECORD_VERSION: u32 = 18;
85const TOPOLOGY_TARGET_PREPARATION_RECORD_VERSION: u32 = 19;
86const TOPOLOGY_COMMIT_RECORD_VERSION: u32 = 20;
87const TOPOLOGY_INSTALLATION_RECORD_VERSION: u32 = 21;
88const TOPOLOGY_RECOVERY_RECORD_VERSION: u32 = 22;
89const TOPOLOGY_SUBMISSION_RECORD_VERSION: u32 = 23;
90const AUTHORITY_HEAD_VERSION: u32 = 1;
91const MAX_AUTHORITY_RECORD_BYTES: u64 = 256 * 1024;
92const MAX_AUTHORITY_HEAD_BYTES: u64 = 128;
93const MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS: usize = MAX_LIVE_AUTHORITY_LINKS * 3
94 + LEADER_LEASE_PRUNE_BATCH_RECORDS * LEADER_LEASE_MAX_PRUNE_BATCHES
95 + 2;
96const MAX_RECOVERY_FAULT_SLOTS: usize = MAX_CHECKPOINT_PARTICIPANTS * 4;
97const RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES: u64 = 32 * 1024;
98const MAX_RECOVERY_RELEASE_TERMINAL_BYTES: u64 = MAX_RECOVERY_ANNOUNCEMENT_BYTES as u64;
99const MAX_LEASE_HEAD_READ_ATTEMPTS: usize = 4;
100const MAX_LIVE_AUTHORITY_LINKS: usize = 4096;
101const OUTCOME_HISTORY_COMPACTION_TRIGGER: usize = 64;
102const OUTCOME_HISTORY_RETAINED_LINKS: usize = 16;
103const LEADER_LEASE_PRUNE_BATCH_RECORDS: usize = 256;
104const LEADER_LEASE_MAX_PRUNE_BATCHES: usize = 4;
105const RECOVERY_RELEASE_GC_BATCH_RECORDS: usize = 64;
106const RECOVERY_RELEASE_GC_MAX_BATCHES: usize = 4;
107const LEADER_LEASE_PRUNE_TIMEOUT: Duration = Duration::from_secs(30);
108#[cfg(test)]
109const MAX_TEST_LEADER_LEASE_RECORDS: usize = 4096;
110
111fn assignment_snapshot_error(context: &str, error: SnapshotError) -> LeaseError {
112 match error {
113 SnapshotError::Io(message) => LeaseError::Io(format!("{context}: {message}")),
114 error => LeaseError::Invalid(format!("{context}: {error}")),
115 }
116}
117
118fn lease_path(sequence: u64) -> OsPath {
119 OsPath::from(format!("{LEASE_PREFIX}v{sequence:016}.json"))
120}
121
122fn authority_head_path() -> OsPath {
123 OsPath::from(AUTHORITY_HEAD_PATH)
124}
125
126fn recovery_release_terminal_path(reference: &RecoveryReleaseTerminalRef) -> OsPath {
127 OsPath::from(format!(
128 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={:020}/sha256={}.json",
129 reference.release.generation(),
130 reference.sha256
131 ))
132}
133
134fn recovery_release_terminal_coordinates(path: &OsPath) -> Result<(u64, String), LeaseError> {
135 let raw = path
136 .as_ref()
137 .strip_prefix(RECOVERY_RELEASE_TERMINAL_PREFIX)
138 .and_then(|value| value.strip_prefix("generation="))
139 .ok_or_else(|| {
140 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
141 })?;
142 let (generation, digest) = raw.split_once("/sha256=").ok_or_else(|| {
143 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
144 })?;
145 let digest = digest.strip_suffix(".json").ok_or_else(|| {
146 LeaseError::Invalid(format!("invalid recovery release terminal path {path}"))
147 })?;
148 if generation.len() != 20
149 || !generation.bytes().all(|byte| byte.is_ascii_digit())
150 || digest.len() != 64
151 || !digest
152 .bytes()
153 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
154 {
155 return Err(LeaseError::Invalid(format!(
156 "invalid recovery release terminal path {path}"
157 )));
158 }
159 let generation = generation.parse::<u64>().map_err(|error| {
160 LeaseError::Invalid(format!(
161 "invalid recovery release terminal generation in {path}: {error}"
162 ))
163 })?;
164 if generation == 0 {
165 return Err(LeaseError::Invalid(format!(
166 "invalid recovery release terminal path {path}"
167 )));
168 }
169 Ok((generation, digest.to_owned()))
170}
171
172fn lease_sequence_from_path(path: &OsPath) -> Result<u64, LeaseError> {
173 let raw = path
174 .as_ref()
175 .strip_prefix(LEASE_PREFIX)
176 .and_then(|file| file.strip_prefix('v'))
177 .and_then(|file| file.strip_suffix(".json"))
178 .ok_or_else(|| LeaseError::Invalid(format!("invalid leader authority path {path}")))?;
179 if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
180 return Err(LeaseError::Invalid(format!(
181 "invalid leader authority sequence in {path}"
182 )));
183 }
184 let sequence = raw.parse::<u64>().map_err(|error| {
185 LeaseError::Invalid(format!(
186 "invalid leader authority sequence in {path}: {error}"
187 ))
188 })?;
189 if sequence == 0 || lease_path(sequence) != *path {
190 return Err(LeaseError::Invalid(format!(
191 "noncanonical leader authority path {path}"
192 )));
193 }
194 Ok(sequence)
195}
196
197fn consume_live_authority_link(traversed: &mut usize) -> bool {
198 if *traversed == MAX_LIVE_AUTHORITY_LINKS {
199 return false;
200 }
201 *traversed += 1;
202 true
203}
204
205fn now_millis() -> i64 {
206 std::time::SystemTime::now()
207 .duration_since(std::time::UNIX_EPOCH)
208 .ok()
209 .and_then(|duration| i64::try_from(duration.as_millis()).ok())
210 .unwrap_or(i64::MAX)
211}
212
213async fn read_authority_record(
214 store: &dyn ObjectStore,
215 sequence: u64,
216) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
217 let result = match store.get(&lease_path(sequence)).await {
218 Ok(result) => result,
219 Err(object_store::Error::NotFound { .. }) => return Ok(None),
220 Err(error) => return Err(LeaseError::Io(error.to_string())),
221 };
222 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_RECORD_BYTES {
223 return Err(LeaseError::Invalid(format!(
224 "leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
225 result.meta.size
226 )));
227 }
228 let bytes = result
229 .bytes()
230 .await
231 .map_err(|error| LeaseError::Io(error.to_string()))?;
232 let record: LeaderAuthorityRecord = serde_json::from_slice(&bytes)?;
233 record.validate()?;
234 if record.lease.seq != sequence {
235 return Err(LeaseError::Invalid(
236 "authority record sequence does not match its object name".into(),
237 ));
238 }
239 let canonical = serde_json::to_vec(&record)?;
240 if canonical.as_slice() != bytes.as_ref() {
241 return Err(LeaseError::Invalid(format!(
242 "leader authority record {sequence} does not use its canonical body"
243 )));
244 }
245 Ok(Some(record))
246}
247
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
249#[serde(deny_unknown_fields)]
250struct AuthorityHeadPointer {
251 version: u32,
252 sequence: u64,
253 nonce: Uuid,
254}
255
256impl AuthorityHeadPointer {
257 fn new(sequence: u64) -> Result<Self, LeaseError> {
258 let pointer = Self {
259 version: AUTHORITY_HEAD_VERSION,
260 sequence,
261 nonce: Uuid::new_v4(),
262 };
263 pointer.validate()?;
264 Ok(pointer)
265 }
266
267 fn validate(self) -> Result<(), LeaseError> {
268 if self.version != AUTHORITY_HEAD_VERSION || self.sequence == 0 || self.nonce.is_nil() {
269 return Err(LeaseError::Invalid(
270 "leader authority head has an unsupported version, zero sequence, or nil nonce"
271 .into(),
272 ));
273 }
274 Ok(())
275 }
276}
277
278#[derive(Clone)]
279struct VersionedAuthorityHeadPointer {
280 pointer: AuthorityHeadPointer,
281 update_version: UpdateVersion,
282}
283
284struct PublishedAuthorityHead {
285 record: LeaderAuthorityRecord,
286 pointer: VersionedAuthorityHeadPointer,
287}
288
289async fn read_authority_head_pointer(
290 store: &dyn ObjectStore,
291) -> Result<Option<VersionedAuthorityHeadPointer>, LeaseError> {
292 let result = match store.get(&authority_head_path()).await {
293 Ok(result) => result,
294 Err(object_store::Error::NotFound { .. }) => return Ok(None),
295 Err(error) => return Err(LeaseError::Io(error.to_string())),
296 };
297 if result.meta.size == 0 || result.meta.size > MAX_AUTHORITY_HEAD_BYTES {
298 return Err(LeaseError::Invalid(format!(
299 "leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
300 result.meta.size
301 )));
302 }
303 let update_version = UpdateVersion {
304 e_tag: result.meta.e_tag.clone(),
305 version: result.meta.version.clone(),
306 };
307 let bytes = result
308 .bytes()
309 .await
310 .map_err(|error| LeaseError::Io(error.to_string()))?;
311 let pointer: AuthorityHeadPointer = serde_json::from_slice(&bytes)?;
312 pointer.validate()?;
313 let canonical = serde_json::to_vec(&pointer)?;
314 if canonical.as_slice() != bytes.as_ref() {
315 return Err(LeaseError::Invalid(
316 "leader authority head does not use its canonical body".into(),
317 ));
318 }
319 Ok(Some(VersionedAuthorityHeadPointer {
320 pointer,
321 update_version,
322 }))
323}
324
325fn encode_authority_head_pointer(sequence: u64) -> Result<Bytes, LeaseError> {
326 let encoded = serde_json::to_vec(&AuthorityHeadPointer::new(sequence)?)?;
327 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
328 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_HEAD_BYTES {
329 return Err(LeaseError::Invalid(format!(
330 "encoded leader authority head is {} bytes; maximum is {MAX_AUTHORITY_HEAD_BYTES}",
331 encoded.len()
332 )));
333 }
334 Ok(Bytes::from(encoded))
335}
336
337fn encode_authority_record(record: &LeaderAuthorityRecord) -> Result<Bytes, LeaseError> {
338 record.validate()?;
339 let encoded = serde_json::to_vec(record)?;
340 let encoded_len = u64::try_from(encoded.len()).unwrap_or(u64::MAX);
341 if encoded.is_empty() || encoded_len > MAX_AUTHORITY_RECORD_BYTES {
342 return Err(LeaseError::Invalid(format!(
343 "encoded leader authority record is {} bytes; maximum is {MAX_AUTHORITY_RECORD_BYTES}",
344 encoded.len()
345 )));
346 }
347 if record.recovery_fault_slots.iter().any(|slot| slot.active) {
348 ensure_recovery_fault_authority_headroom(encoded_len)?;
349 }
350 Ok(Bytes::from(encoded))
351}
352
353fn ensure_recovery_fault_authority_headroom(encoded_len: u64) -> Result<(), LeaseError> {
354 let limit = MAX_AUTHORITY_RECORD_BYTES
355 .checked_sub(RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES)
356 .expect("recovery fault headroom must fit the authority record");
357 if encoded_len > limit {
358 return Err(LeaseError::Invalid(format!(
359 "recovery fault inventory leaves {RECOVERY_FAULT_AUTHORITY_HEADROOM_BYTES} bytes of mandatory authority headroom only up to {limit} bytes; candidate is {encoded_len} bytes"
360 )));
361 }
362 Ok(())
363}
364
365#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
367pub struct LeaderLeaseOwner {
368 pub node: NodeId,
370 pub boot: Uuid,
372 pub process_term: u64,
374}
375
376impl LeaderLeaseOwner {
377 fn from_process_lease(process: &ProcessLease) -> Result<Self, LeaseError> {
378 process
379 .validate(process.node)
380 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
381 Ok(Self {
382 node: process.node,
383 boot: process.owner,
384 process_term: process.term,
385 })
386 }
387
388 fn validate(&self) -> Result<(), LeaseError> {
389 if self.node.is_unassigned() || self.boot.is_nil() || self.process_term == 0 {
390 return Err(LeaseError::Invalid(
391 "leader owner node, boot identity, and process term must be nonzero".into(),
392 ));
393 }
394 Ok(())
395 }
396
397 fn proof_owner(&self) -> LeaderProofOwner {
398 LeaderProofOwner {
399 node_id: self.node.0,
400 boot_id: self.boot,
401 process_term: self.process_term,
402 }
403 }
404}
405
406#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
408pub struct LeaderLease {
409 pub seq: u64,
411 pub renewal_sequence: u64,
413 pub token: u64,
415 pub owner: LeaderLeaseOwner,
417 pub expires_at_ms: i64,
419 pub catalog_manifest: Option<CatalogManifestRef>,
421}
422
423#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
424#[serde(deny_unknown_fields)]
425struct OutcomeLink {
426 sequence: u64,
427 epoch: u64,
428 checkpoint_id: u64,
429}
430
431impl OutcomeLink {
432 fn validate(self) -> Result<(), LeaseError> {
433 let attempt = crate::checkpoint::CheckpointAttempt::new(self.epoch, self.checkpoint_id);
434 if self.sequence == 0 || !attempt.is_canonical() {
435 return Err(LeaseError::Invalid(
436 "checkpoint outcome link requires a nonzero authority sequence and one canonical checkpoint ID"
437 .into(),
438 ));
439 }
440 Ok(())
441 }
442}
443
444#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
445#[serde(deny_unknown_fields)]
446struct AssignmentDecisionLink {
447 sequence: u64,
448 target_version: u64,
449}
450
451#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
452#[serde(deny_unknown_fields)]
453struct AuthorityRecoveryFaultSlot {
454 publisher: RecoveryFaultPublisher,
455 request_sequence: u64,
456 fault_sequence: u64,
457 #[serde(
458 default,
459 skip_serializing_if = "RecoveryFaultDisposition::is_recoverable"
460 )]
461 disposition: RecoveryFaultDisposition,
462 active: bool,
463}
464
465impl AuthorityRecoveryFaultSlot {
466 fn validate(&self) -> Result<(), LeaseError> {
467 self.publisher.validate().map_err(LeaseError::Invalid)?;
468 if self.request_sequence == 0 || self.fault_sequence == 0 {
469 return Err(LeaseError::Invalid(
470 "recovery fault request and authority sequence must be nonzero".into(),
471 ));
472 }
473 if self.disposition == RecoveryFaultDisposition::Terminal && !self.active {
474 return Err(LeaseError::Invalid(
475 "terminal recovery fault authority cannot be tombstoned".into(),
476 ));
477 }
478 Ok(())
479 }
480
481 fn fault(&self) -> RecoveryFault {
482 RecoveryFault {
483 reporter: NodeId(self.publisher.participant.node_id),
484 sequence: self.fault_sequence,
485 disposition: self.disposition,
486 }
487 }
488
489 fn matches_request(
490 &self,
491 publisher: RecoveryFaultPublisher,
492 request_sequence: u64,
493 disposition: RecoveryFaultDisposition,
494 ) -> bool {
495 self.publisher == publisher
496 && self.request_sequence == request_sequence
497 && self.disposition == disposition
498 }
499}
500
501#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
502#[serde(deny_unknown_fields)]
503pub(crate) struct RecoveryReleaseTerminalRef {
504 release: RecoveryReleaseId,
505 sha256: String,
506 encoded_len: u64,
507}
508
509impl RecoveryReleaseTerminalRef {
510 fn validate(&self) -> Result<(), LeaseError> {
511 if !self.release.is_canonical()
512 || self.sha256.len() != 64
513 || !self
514 .sha256
515 .bytes()
516 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
517 || self.encoded_len == 0
518 || self.encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES
519 {
520 return Err(LeaseError::Invalid(
521 "recovery release terminal reference is not canonical".into(),
522 ));
523 }
524 Ok(())
525 }
526
527 pub(crate) fn generation(&self) -> u64 {
528 self.release.generation()
529 }
530}
531
532#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
533#[serde(deny_unknown_fields)]
534struct AuthorityRecoveryReleaseCommit {
535 terminal: RecoveryReleaseTerminalRef,
536 leader_proof: LeaderProof,
537}
538
539impl AuthorityRecoveryReleaseCommit {
540 fn validate(&self) -> Result<(), LeaseError> {
541 self.terminal.validate()?;
542 if !self.leader_proof.is_canonical() {
543 return Err(LeaseError::Invalid(
544 "recovery release commit has no canonical leader proof".into(),
545 ));
546 }
547 Ok(())
548 }
549}
550
551#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
552#[serde(deny_unknown_fields)]
553pub(crate) struct RecoveryReleaseLink {
554 sequence: u64,
555 terminal: RecoveryReleaseTerminalRef,
556}
557
558impl RecoveryReleaseLink {
559 fn validate(&self) -> Result<(), LeaseError> {
560 if self.sequence == 0 {
561 return Err(LeaseError::Invalid(
562 "recovery release authority link has a zero sequence".into(),
563 ));
564 }
565 self.terminal.validate()
566 }
567}
568
569#[derive(Debug, Clone, PartialEq, Eq)]
570pub(crate) enum RecordRecoveryReleaseCommitResult {
571 Created(RecoveryReleaseTerminalRef),
572 Unchanged(RecoveryReleaseTerminalRef),
573 Conflict { winner: RecoveryReleaseTerminalRef },
574 FaultsChanged,
575}
576
577#[derive(Debug, Clone, Copy, PartialEq, Eq)]
578pub(crate) enum RecordRecoveryFaultResult {
579 Active,
580 AlreadyCleared,
581 CoveredByNewerRequest,
582 TerminalFenceActive,
583 Superseded,
584}
585
586fn recovery_release_id_for_terminal(
587 terminal: &RecoveryAnnouncement,
588) -> Result<RecoveryReleaseId, LeaseError> {
589 terminal.validate().map_err(LeaseError::Invalid)?;
590 let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
591 return Err(LeaseError::Invalid(
592 "recovery release terminal must carry ReleaseCommitted".into(),
593 ));
594 };
595 RecoveryReleaseId::for_pending(&RecoveryAnnouncement {
596 round: terminal.round.clone(),
597 phase: RecoverPhase::Release { epoch },
598 })
599 .map_err(LeaseError::Invalid)
600}
601
602fn encode_recovery_release_terminal(
603 terminal: &RecoveryAnnouncement,
604) -> Result<(Bytes, RecoveryReleaseTerminalRef), LeaseError> {
605 let release = recovery_release_id_for_terminal(terminal)?;
606 let encoded = serde_json::to_vec(terminal)?;
607 let encoded_len = u64::try_from(encoded.len())
608 .map_err(|_| LeaseError::Invalid("recovery release terminal is too large".into()))?;
609 if encoded_len == 0 || encoded_len > MAX_RECOVERY_RELEASE_TERMINAL_BYTES {
610 return Err(LeaseError::Invalid(format!(
611 "recovery release terminal is {encoded_len} bytes; maximum is {MAX_RECOVERY_RELEASE_TERMINAL_BYTES}"
612 )));
613 }
614 let reference = RecoveryReleaseTerminalRef {
615 release,
616 sha256: format!("{:x}", Sha256::digest(&encoded)),
617 encoded_len,
618 };
619 reference.validate()?;
620 Ok((Bytes::from(encoded), reference))
621}
622
623#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
625#[serde(deny_unknown_fields)]
626pub struct AssignmentDrainDecision {
627 pub transition: AssignmentDrainTransition,
629 pub leader_proof: LeaderProof,
631 pub verdict: AssignmentDrainVerdict,
633 pub handoff_checkpoint: Option<CommittedCheckpointRef>,
636}
637
638impl AssignmentDrainDecision {
639 pub fn commit(
644 transition: &AssignmentDrainTransition,
645 leader_proof: LeaderProof,
646 handoff_checkpoint: CommittedCheckpointRef,
647 ) -> Result<Self, String> {
648 handoff_checkpoint.validate()?;
649 if !transition.is_canonical() || leader_proof != transition.leader {
650 return Err("assignment drain decision requires a canonical transition".into());
651 }
652 Ok(Self {
653 transition: transition.clone(),
654 leader_proof,
655 verdict: AssignmentDrainVerdict::Commit,
656 handoff_checkpoint: Some(handoff_checkpoint),
657 })
658 }
659
660 pub fn abort(
665 transition: &AssignmentDrainTransition,
666 leader_proof: LeaderProof,
667 ) -> Result<Self, String> {
668 if !transition.is_canonical() || !leader_proof.is_canonical() {
669 return Err("assignment drain decision requires a canonical transition".into());
670 }
671 Ok(Self {
672 transition: transition.clone(),
673 leader_proof,
674 verdict: AssignmentDrainVerdict::Abort,
675 handoff_checkpoint: None,
676 })
677 }
678
679 #[must_use]
681 pub fn round(&self) -> AssignmentDrainId {
682 self.transition.id()
683 }
684
685 #[must_use]
687 pub fn target_version(&self) -> u64 {
688 self.transition.target.assignment_version
689 }
690
691 fn validate(&self) -> Result<(), LeaseError> {
692 if !self.transition.is_canonical()
693 || !self.leader_proof.is_canonical()
694 || (self.verdict == AssignmentDrainVerdict::Commit
695 && self.leader_proof != self.transition.leader)
696 {
697 return Err(LeaseError::Invalid(
698 "assignment drain decision is not canonical".into(),
699 ));
700 }
701 match (self.verdict, self.handoff_checkpoint.as_ref()) {
702 (AssignmentDrainVerdict::Commit, Some(reference)) => {
703 reference.validate().map_err(LeaseError::Invalid)?;
704 }
705 (AssignmentDrainVerdict::Abort, None) => {}
706 _ => {
707 return Err(LeaseError::Invalid(
708 "assignment drain decision has an invalid handoff checkpoint".into(),
709 ));
710 }
711 }
712 Ok(())
713 }
714}
715
716#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
718#[serde(rename_all = "snake_case")]
719pub enum AssignmentDrainVerdict {
720 Commit,
722 Abort,
724}
725
726#[derive(Debug, Clone, PartialEq, Eq)]
728pub enum RecordAssignmentDrainDecisionResult {
729 Created(AssignmentDrainDecision),
731 Unchanged(AssignmentDrainDecision),
733 Conflict {
735 winner: AssignmentDrainDecision,
737 },
738}
739
740#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
742#[serde(deny_unknown_fields)]
743pub struct AssignmentRecoveryDecision {
744 pub predecessor: CheckpointAssignmentFence,
746 pub target: CheckpointAssignmentFence,
748 pub proposal: AssignmentSnapshotRef,
750 pub removed_process_fences: Vec<ProcessLeaseFence>,
752 pub recovery_checkpoint: CommittedCheckpointRef,
754 pub leader_proof: LeaderProof,
756}
757
758impl AssignmentRecoveryDecision {
759 pub fn new(
764 predecessor: CheckpointAssignmentFence,
765 target: CheckpointAssignmentFence,
766 proposal: AssignmentSnapshotRef,
767 removed_process_fences: Vec<ProcessLeaseFence>,
768 recovery_checkpoint: CommittedCheckpointRef,
769 leader_proof: LeaderProof,
770 ) -> Result<Self, String> {
771 let decision = Self {
772 predecessor,
773 target,
774 proposal,
775 removed_process_fences,
776 recovery_checkpoint,
777 leader_proof,
778 };
779 decision.validate().map_err(|error| error.to_string())?;
780 Ok(decision)
781 }
782
783 #[must_use]
785 pub fn target_version(&self) -> u64 {
786 self.target.assignment_version
787 }
788
789 fn validate(&self) -> Result<(), LeaseError> {
790 if !self.predecessor.is_canonical()
791 || !self.target.is_canonical()
792 || !self.leader_proof.is_canonical()
793 || self.predecessor.vnode_count != self.target.vnode_count
794 || self.predecessor.assignment_version.checked_add(1)
795 != Some(self.target.assignment_version)
796 || self.proposal.validate().is_err()
797 || self.proposal.version != self.target.assignment_version
798 || self.recovery_checkpoint.validate().is_err()
799 {
800 return Err(LeaseError::Invalid(
801 "assignment recovery decision is not a canonical successor".into(),
802 ));
803 }
804
805 let removed: Vec<_> = self
806 .predecessor
807 .participants
808 .iter()
809 .filter(|participant| {
810 self.target.participant_incarnation(participant.node_id)
811 != Some(participant.boot_incarnation)
812 })
813 .collect();
814 if removed.is_empty() || removed.len() != self.removed_process_fences.len() {
815 return Err(LeaseError::Invalid(
816 "assignment recovery decision requires the exact removed predecessor process set"
817 .into(),
818 ));
819 }
820 for (participant, fence) in removed.iter().zip(&self.removed_process_fences) {
821 if !fence.is_canonical()
822 || fence.predecessor.node.0 != participant.node_id
823 || fence.predecessor.owner != participant.boot_incarnation
824 {
825 return Err(LeaseError::Invalid(
826 "assignment recovery process fences are incomplete, unsorted, or unrelated"
827 .into(),
828 ));
829 }
830 }
831 Ok(())
832 }
833}
834
835#[derive(Debug, Clone, PartialEq, Eq)]
837pub enum RecordAssignmentRecoveryDecisionResult {
838 Created(AssignmentRecoveryDecision),
840 Unchanged(AssignmentRecoveryDecision),
842 Conflict {
844 winner: AssignmentRecoveryDecision,
846 },
847}
848
849#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
850#[serde(
851 tag = "kind",
852 content = "decision",
853 rename_all = "snake_case",
854 deny_unknown_fields
855)]
856enum AuthorityAssignmentDecision {
857 Drain(AssignmentDrainDecision),
858 Recovery(AssignmentRecoveryDecision),
859}
860
861impl AuthorityAssignmentDecision {
862 fn target_version(&self) -> u64 {
863 match self {
864 Self::Drain(decision) => decision.target_version(),
865 Self::Recovery(decision) => decision.target_version(),
866 }
867 }
868
869 fn predecessor(&self) -> &CheckpointAssignmentFence {
870 match self {
871 Self::Drain(decision) => &decision.transition.predecessor,
872 Self::Recovery(decision) => &decision.predecessor,
873 }
874 }
875
876 fn materialized_target(&self) -> CheckpointAssignmentFence {
877 match self {
878 Self::Drain(decision) => match decision.verdict {
879 AssignmentDrainVerdict::Commit => decision.transition.target.clone(),
880 AssignmentDrainVerdict::Abort => {
881 let mut target = decision.transition.predecessor.clone();
882 target.assignment_version = decision.transition.target.assignment_version;
883 target
884 }
885 },
886 Self::Recovery(decision) => decision.target.clone(),
887 }
888 }
889
890 fn leader_proof(&self) -> &LeaderProof {
891 match self {
892 Self::Drain(decision) => &decision.leader_proof,
893 Self::Recovery(decision) => &decision.leader_proof,
894 }
895 }
896
897 fn validate(&self) -> Result<(), LeaseError> {
898 match self {
899 Self::Drain(decision) => decision.validate(),
900 Self::Recovery(decision) => decision.validate(),
901 }
902 }
903}
904
905enum RecordAuthorityAssignmentDecisionResult {
906 Created(AuthorityAssignmentDecision),
907 Unchanged(AuthorityAssignmentDecision),
908 Conflict { winner: AuthorityAssignmentDecision },
909}
910
911#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
913#[serde(rename_all = "snake_case")]
914pub enum ClusterArtifactCleanupPhase {
915 DeleteData,
917 DeleteMetadata,
919}
920
921#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
923#[serde(deny_unknown_fields)]
924pub struct ClusterArtifactCleanupCursor {
925 pub protected: CommittedCheckpointRef,
927 pub current: CommittedCheckpointRef,
929 pub next: Option<CommittedCheckpointRef>,
931 pub stop_before: Option<CommittedCheckpointRef>,
933 pub participant_ids: Vec<u64>,
935 pub phase: ClusterArtifactCleanupPhase,
937}
938
939impl ClusterArtifactCleanupCursor {
940 fn validate(&self) -> Result<(), LeaseError> {
941 for (name, reference) in [
942 ("protected", Some(&self.protected)),
943 ("current", Some(&self.current)),
944 ("next", self.next.as_ref()),
945 ("stop-before", self.stop_before.as_ref()),
946 ] {
947 if let Some(reference) = reference {
948 reference.validate().map_err(|error| {
949 LeaseError::Invalid(format!(
950 "cluster artifact cleanup {name} reference: {error}"
951 ))
952 })?;
953 }
954 }
955 if self.current.epoch >= self.protected.epoch
956 || self.current.checkpoint_id >= self.protected.checkpoint_id
957 {
958 return Err(LeaseError::Invalid(
959 "cluster artifact cleanup current checkpoint is not older than its protected cut"
960 .into(),
961 ));
962 }
963 if let Some(next) = self.next.as_ref() {
964 if next.epoch >= self.current.epoch || next.checkpoint_id >= self.current.checkpoint_id
965 {
966 return Err(LeaseError::Invalid(
967 "cluster artifact cleanup next checkpoint is not older than its current checkpoint"
968 .into(),
969 ));
970 }
971 }
972 if let Some(stop) = self.stop_before.as_ref() {
973 if stop.epoch >= self.current.epoch || stop.checkpoint_id >= self.current.checkpoint_id
974 {
975 return Err(LeaseError::Invalid(
976 "cluster artifact cleanup stop boundary is not older than its current checkpoint"
977 .into(),
978 ));
979 }
980 let next = self.next.as_ref().ok_or_else(|| {
981 LeaseError::Invalid(
982 "cluster artifact cleanup lost the path to its stop boundary".into(),
983 )
984 })?;
985 if next.epoch < stop.epoch || next.checkpoint_id < stop.checkpoint_id {
986 return Err(LeaseError::Invalid(
987 "cluster artifact cleanup next checkpoint crosses its stop boundary".into(),
988 ));
989 }
990 }
991 if self.participant_ids.is_empty()
992 || self.participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
993 || self.participant_ids[0] == 0
994 || !self
995 .participant_ids
996 .windows(2)
997 .all(|pair| pair[0] < pair[1])
998 {
999 return Err(LeaseError::Invalid(
1000 "cluster artifact cleanup participants are not canonical".into(),
1001 ));
1002 }
1003 Ok(())
1004 }
1005}
1006
1007#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1008#[serde(deny_unknown_fields)]
1009struct AuthorityOutcomeFloor {
1010 deployment_id: String,
1011 artifact_before_epoch: u64,
1012 authority_before_epoch: u64,
1013 terminal_anchor: Option<CheckpointOutcome>,
1014 terminal_anchor_link: Option<OutcomeLink>,
1015 committed_anchor: Option<CheckpointOutcome>,
1016 committed_anchor_link: Option<OutcomeLink>,
1017}
1018
1019#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1020#[serde(deny_unknown_fields)]
1021struct AuthorityAssignmentDecisionFloor {
1022 before_target_version: u64,
1023 terminal_anchor: Option<AuthorityAssignmentDecision>,
1024 terminal_anchor_link: Option<AssignmentDecisionLink>,
1025}
1026
1027#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1028#[serde(deny_unknown_fields)]
1029struct AssignmentHandoffPin {
1030 target: CheckpointAssignmentFence,
1031 checkpoint: CommittedCheckpointRef,
1032}
1033
1034impl AssignmentHandoffPin {
1035 fn validate(&self) -> Result<(), LeaseError> {
1036 if !self.target.is_canonical() {
1037 return Err(LeaseError::Invalid(
1038 "assignment handoff pin has an invalid target fence".into(),
1039 ));
1040 }
1041 self.checkpoint.validate().map_err(LeaseError::Invalid)
1042 }
1043}
1044
1045#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1047#[serde(deny_unknown_fields)]
1048struct LeaderAuthorityRecord {
1049 version: u32,
1050 lease: LeaderLease,
1051 checkpoint_outcome: Option<CheckpointOutcome>,
1053 previous_outcome: Option<OutcomeLink>,
1055 outcome_head: Option<OutcomeLink>,
1057 previous_commit: Option<OutcomeLink>,
1059 commit_head: Option<OutcomeLink>,
1061 outcome_floor: Option<AuthorityOutcomeFloor>,
1063 artifact_cleanup: Option<ClusterArtifactCleanupCursor>,
1065 #[serde(default, skip_serializing_if = "Option::is_none")]
1067 subscription_cleanup_commit: Option<subscription_replay::SubscriptionCleanupCommit>,
1068 active_checkpoint_artifacts: Option<CheckpointArtifactInventory>,
1070 active_checkpoint_artifact_leader_proof: Option<LeaderProof>,
1072 assignment_decision: Option<AuthorityAssignmentDecision>,
1074 previous_assignment_decision: Option<AssignmentDecisionLink>,
1076 assignment_decision_head: Option<AssignmentDecisionLink>,
1078 assignment_decision_floor: Option<AuthorityAssignmentDecisionFloor>,
1080 assignment_handoff_pin: Option<AssignmentHandoffPin>,
1082 recovery_fault_revision: u64,
1084 recovery_fault_slots: Vec<AuthorityRecoveryFaultSlot>,
1086 recovery_release_commit: Option<AuthorityRecoveryReleaseCommit>,
1088 recovery_release_head: Option<RecoveryReleaseLink>,
1090 #[serde(default, skip_serializing_if = "Option::is_none")]
1092 topology_baseline: Option<super::topology::LegacyTopologyBaseline>,
1093 #[serde(default, skip_serializing_if = "Option::is_none")]
1095 assignment_drain_reservation: Option<AssignmentDrainReservation>,
1096 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1098 topology_operations: Vec<super::topology::TopologyAdmissionStatus>,
1099}
1100
1101#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1102#[serde(deny_unknown_fields)]
1103struct AssignmentDrainReservation {
1104 proposal: AssignmentSnapshotRef,
1105 transition: AssignmentDrainTransition,
1106 authority_sequence: u64,
1107}
1108
1109impl LeaderLease {
1110 fn validate(&self) -> Result<(), LeaseError> {
1111 self.owner.validate()?;
1112 if self.seq == 0
1113 || self.renewal_sequence == 0
1114 || self.renewal_sequence > self.seq
1115 || self.token == 0
1116 {
1117 return Err(LeaseError::Invalid(
1118 "leader lease sequence and token must be nonzero and renewal sequence must be within 1..=sequence"
1119 .into(),
1120 ));
1121 }
1122 if let Some(reference) = &self.catalog_manifest {
1123 reference
1124 .validate()
1125 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1126 }
1127 Ok(())
1128 }
1129
1130 #[must_use]
1132 pub fn proof(&self) -> LeaderProof {
1133 LeaderProof {
1134 owner: self.owner.proof_owner(),
1135 fencing_token: self.token,
1136 }
1137 }
1138
1139 #[must_use]
1141 pub fn matches_proof(&self, proof: &LeaderProof) -> bool {
1142 proof.is_canonical()
1143 && proof.owner == self.owner.proof_owner()
1144 && proof.fencing_token == self.token
1145 }
1146
1147 fn has_same_liveness_identity(&self, other: &Self) -> bool {
1148 self.owner == other.owner
1149 && self.token == other.token
1150 && self.renewal_sequence == other.renewal_sequence
1151 }
1152}
1153
1154impl AuthorityOutcomeFloor {
1155 fn validate(&self) -> Result<(), LeaseError> {
1156 let deployment = Uuid::parse_str(&self.deployment_id)
1157 .map_err(|error| LeaseError::Invalid(format!("outcome floor deployment: {error}")))?;
1158 if deployment.is_nil()
1159 || deployment.to_string() != self.deployment_id
1160 || self.authority_before_epoch == 0
1161 || self.artifact_before_epoch > self.authority_before_epoch
1162 {
1163 return Err(LeaseError::Invalid(
1164 "outcome floor requires a canonical deployment and ordered authority/artifact horizons"
1165 .into(),
1166 ));
1167 }
1168 for link in [self.terminal_anchor_link, self.committed_anchor_link]
1169 .into_iter()
1170 .flatten()
1171 {
1172 link.validate()?;
1173 }
1174 for (name, anchor) in [
1175 ("terminal", self.terminal_anchor.as_ref()),
1176 ("committed", self.committed_anchor.as_ref()),
1177 ] {
1178 let Some(anchor) = anchor else { continue };
1179 anchor
1180 .validate_shape(anchor.epoch)
1181 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1182 if anchor.scope != CheckpointScope::Cluster
1183 || anchor.deployment_id != self.deployment_id
1184 {
1185 return Err(LeaseError::Invalid(format!(
1186 "outcome floor has an invalid {name} anchor"
1187 )));
1188 }
1189 }
1190 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1191 (Some(anchor), Some(link))
1192 if anchor.epoch < self.authority_before_epoch
1193 && link.sequence != 0
1194 && link.epoch == anchor.epoch
1195 && link.checkpoint_id == anchor.checkpoint_id => {}
1196 (None, None) => {}
1197 _ => {
1198 return Err(LeaseError::Invalid(
1199 "outcome floor terminal anchor does not match its exact authority link".into(),
1200 ));
1201 }
1202 }
1203 match (self.committed_anchor.as_ref(), self.committed_anchor_link) {
1204 (Some(anchor), Some(link))
1205 if self.artifact_before_epoch != 0
1206 && anchor.is_commit()
1207 && anchor.epoch < self.artifact_before_epoch
1208 && link.sequence != 0
1209 && link.epoch == anchor.epoch
1210 && link.checkpoint_id == anchor.checkpoint_id => {}
1211 (None, None) => {}
1212 _ => {
1213 return Err(LeaseError::Invalid(
1214 "outcome floor committed anchor does not match its exact Commit link".into(),
1215 ));
1216 }
1217 }
1218 if let (Some(committed), Some(terminal)) = (
1219 self.committed_anchor.as_ref(),
1220 self.terminal_anchor.as_ref(),
1221 ) {
1222 let ordered = if committed.epoch == terminal.epoch {
1223 committed == terminal
1224 } else {
1225 committed.epoch < terminal.epoch && committed.checkpoint_id < terminal.checkpoint_id
1226 };
1227 if !ordered {
1228 return Err(LeaseError::Invalid(
1229 "outcome floor anchors are not monotonically ordered".into(),
1230 ));
1231 }
1232 let links_ordered = match (self.committed_anchor_link, self.terminal_anchor_link) {
1233 (Some(committed_link), Some(terminal_link)) if committed == terminal => {
1234 committed_link == terminal_link
1235 }
1236 (Some(committed_link), Some(terminal_link)) => {
1237 committed_link.sequence < terminal_link.sequence
1238 }
1239 _ => false,
1240 };
1241 if !links_ordered {
1242 return Err(LeaseError::Invalid(
1243 "outcome floor anchor links are not ordered with their exact outcomes".into(),
1244 ));
1245 }
1246 } else if self.committed_anchor.is_some() {
1247 return Err(LeaseError::Invalid(
1248 "outcome floor has a committed anchor without terminal continuity".into(),
1249 ));
1250 }
1251 Ok(())
1252 }
1253}
1254
1255impl AuthorityAssignmentDecisionFloor {
1256 fn validate(&self) -> Result<(), LeaseError> {
1257 if self.before_target_version == 0 {
1258 return Err(LeaseError::Invalid(
1259 "assignment decision floor requires a nonzero target version".into(),
1260 ));
1261 }
1262 match (self.terminal_anchor.as_ref(), self.terminal_anchor_link) {
1263 (Some(anchor), Some(link)) => {
1264 anchor.validate()?;
1265 if link.sequence == 0
1266 || link.target_version != anchor.target_version()
1267 || anchor.target_version() >= self.before_target_version
1268 {
1269 return Err(LeaseError::Invalid(
1270 "assignment decision floor anchor does not match its exact authority link"
1271 .into(),
1272 ));
1273 }
1274 }
1275 (None, None) => {}
1276 _ => {
1277 return Err(LeaseError::Invalid(
1278 "assignment decision floor has an incomplete terminal anchor".into(),
1279 ));
1280 }
1281 }
1282 Ok(())
1283 }
1284}
1285
1286impl LeaderAuthorityRecord {
1287 fn initial(lease: LeaderLease) -> Self {
1288 Self {
1289 version: AUTHORITY_RECORD_VERSION,
1290 lease,
1291 checkpoint_outcome: None,
1292 previous_outcome: None,
1293 outcome_head: None,
1294 previous_commit: None,
1295 commit_head: None,
1296 outcome_floor: None,
1297 artifact_cleanup: None,
1298 subscription_cleanup_commit: None,
1299 active_checkpoint_artifacts: None,
1300 active_checkpoint_artifact_leader_proof: None,
1301 assignment_decision: None,
1302 previous_assignment_decision: None,
1303 assignment_decision_head: None,
1304 assignment_decision_floor: None,
1305 assignment_handoff_pin: None,
1306 recovery_fault_revision: 0,
1307 recovery_fault_slots: Vec::new(),
1308 recovery_release_commit: None,
1309 recovery_release_head: None,
1310 topology_baseline: None,
1311 assignment_drain_reservation: None,
1312 topology_operations: Vec::new(),
1313 }
1314 }
1315
1316 fn preserve_with_lease(&self, lease: LeaderLease) -> Self {
1317 let leader_changed = self.lease.owner != lease.owner || self.lease.token != lease.token;
1318 let mut preserved = Self {
1319 version: self.version,
1320 lease,
1321 checkpoint_outcome: None,
1322 previous_outcome: None,
1323 outcome_head: self.outcome_head,
1324 previous_commit: None,
1325 commit_head: self.commit_head,
1326 outcome_floor: self.outcome_floor.clone(),
1327 artifact_cleanup: self.artifact_cleanup.clone(),
1328 subscription_cleanup_commit: self.subscription_cleanup_commit.clone(),
1329 active_checkpoint_artifacts: self.active_checkpoint_artifacts.clone(),
1330 active_checkpoint_artifact_leader_proof: self
1331 .active_checkpoint_artifact_leader_proof
1332 .clone(),
1333 assignment_decision: None,
1334 previous_assignment_decision: None,
1335 assignment_decision_head: self.assignment_decision_head,
1336 assignment_decision_floor: self.assignment_decision_floor.clone(),
1337 assignment_handoff_pin: self.assignment_handoff_pin.clone(),
1338 recovery_fault_revision: self.recovery_fault_revision,
1339 recovery_fault_slots: self.recovery_fault_slots.clone(),
1340 recovery_release_commit: None,
1341 recovery_release_head: self.recovery_release_head.clone(),
1342 topology_baseline: self.topology_baseline.clone(),
1343 assignment_drain_reservation: self.assignment_drain_reservation.clone(),
1344 topology_operations: self.topology_operations.clone(),
1345 };
1346 if leader_changed {
1347 preserved
1348 .abort_topology_preparation(super::topology::TopologyAbortReason::LeaderChanged);
1349 }
1350 preserved
1351 }
1352
1353 fn validate(&self) -> Result<(), LeaseError> {
1354 if self.version != AUTHORITY_RECORD_VERSION
1355 && self.version != TOPOLOGY_AUTHORITY_RECORD_VERSION
1356 && self.version != TOPOLOGY_ADMISSION_RECORD_VERSION
1357 && self.version != TOPOLOGY_CUT_RECORD_VERSION
1358 && self.version != TOPOLOGY_PREPARATION_RECORD_VERSION
1359 && self.version != TOPOLOGY_MIGRATION_ROOT_RECORD_VERSION
1360 && self.version != TOPOLOGY_SOURCE_ROOT_RECORD_VERSION
1361 && self.version != TOPOLOGY_TARGET_PREPARATION_RECORD_VERSION
1362 && self.version != TOPOLOGY_COMMIT_RECORD_VERSION
1363 && self.version != TOPOLOGY_INSTALLATION_RECORD_VERSION
1364 && self.version != TOPOLOGY_RECOVERY_RECORD_VERSION
1365 && self.version != TOPOLOGY_SUBMISSION_RECORD_VERSION
1366 {
1367 return Err(LeaseError::Invalid(format!(
1368 "authority record version {} is unsupported",
1369 self.version
1370 )));
1371 }
1372 self.lease.validate()?;
1373 self.validate_topology_baseline()?;
1374 self.validate_topology_admission()?;
1375 for link in [
1376 self.previous_outcome,
1377 self.outcome_head,
1378 self.previous_commit,
1379 self.commit_head,
1380 ]
1381 .into_iter()
1382 .flatten()
1383 {
1384 link.validate()?;
1385 }
1386 self.subscription_cleanup_commit
1387 .iter()
1388 .try_for_each(subscription_replay::SubscriptionCleanupCommit::validate)?;
1389 self.validate_recovery_fault_inventory()?;
1390 self.validate_checkpoint_outcome_chain()?;
1391 self.validate_outcome_floor()?;
1392 self.validate_artifact_cleanup()?;
1393 self.validate_checkpoint_artifact_inventory()?;
1394 self.validate_assignment_decision_chain()?;
1395 self.validate_assignment_decision_floor()?;
1396 self.validate_assignment_handoff_pin()?;
1397 self.validate_recovery_release()?;
1398 Ok(())
1399 }
1400
1401 fn validate_recovery_fault_inventory(&self) -> Result<(), LeaseError> {
1402 if self.recovery_fault_slots.windows(2).any(|pair| {
1403 pair[0].publisher.participant.node_id >= pair[1].publisher.participant.node_id
1404 }) || (self.recovery_fault_revision == 0 && !self.recovery_fault_slots.is_empty())
1405 || (self.recovery_fault_revision != 0
1406 && self.recovery_fault_slots.is_empty()
1407 && self
1408 .recovery_release_head
1409 .as_ref()
1410 .map(|head| head.sequence)
1411 != Some(self.recovery_fault_revision))
1412 || self.recovery_fault_revision > self.lease.seq
1413 || self.recovery_fault_slots.len() > MAX_RECOVERY_FAULT_SLOTS
1414 {
1415 return Err(LeaseError::Invalid(
1416 "leader authority recovery-fault inventory is not canonical".into(),
1417 ));
1418 }
1419 let mut fault_sequences = BTreeSet::new();
1420 for slot in &self.recovery_fault_slots {
1421 slot.validate()?;
1422 if slot.fault_sequence > self.recovery_fault_revision
1423 || slot.fault_sequence > self.lease.seq
1424 || !fault_sequences.insert(slot.fault_sequence)
1425 {
1426 return Err(LeaseError::Invalid(
1427 "leader authority recovery-fault sequences are not canonical".into(),
1428 ));
1429 }
1430 }
1431 if [
1432 self.checkpoint_outcome.is_some(),
1433 self.assignment_decision.is_some(),
1434 self.recovery_release_commit.is_some(),
1435 ]
1436 .into_iter()
1437 .filter(|present| *present)
1438 .count()
1439 > 1
1440 {
1441 return Err(LeaseError::Invalid(
1442 "one authority sequence cannot admit two terminal domains".into(),
1443 ));
1444 }
1445 Ok(())
1446 }
1447
1448 fn validate_checkpoint_outcome_chain(&self) -> Result<(), LeaseError> {
1449 if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1450 outcome
1451 .validate_shape(outcome.epoch)
1452 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
1453 if outcome.scope != CheckpointScope::Cluster {
1454 return Err(LeaseError::Invalid(
1455 "leader authority can only admit cluster checkpoint outcomes".into(),
1456 ));
1457 }
1458 let proof = outcome
1459 .leader_proof
1460 .as_ref()
1461 .ok_or_else(|| LeaseError::Invalid("cluster outcome has no leader proof".into()))?;
1462 let current_link = OutcomeLink {
1463 sequence: self.lease.seq,
1464 epoch: outcome.epoch,
1465 checkpoint_id: outcome.checkpoint_id,
1466 };
1467 if !self.lease.matches_proof(proof) || self.outcome_head != Some(current_link) {
1468 return Err(LeaseError::Invalid(
1469 "cluster outcome is not bound to its exact authority sequence and term".into(),
1470 ));
1471 }
1472 if let Some(previous) = self.previous_outcome {
1473 if previous.sequence >= self.lease.seq
1474 || previous.epoch >= outcome.epoch
1475 || previous.checkpoint_id >= outcome.checkpoint_id
1476 {
1477 return Err(LeaseError::Invalid(
1478 "cluster outcome link does not move backward in sequence and epoch".into(),
1479 ));
1480 }
1481 }
1482 if outcome.is_commit() {
1483 if self.commit_head != Some(current_link) {
1484 return Err(LeaseError::Invalid(
1485 "cluster Commit is not bound to its exact Commit-chain head".into(),
1486 ));
1487 }
1488 if let Some(previous) = self.previous_commit {
1489 if previous.sequence >= self.lease.seq
1490 || previous.epoch >= outcome.epoch
1491 || previous.checkpoint_id >= outcome.checkpoint_id
1492 {
1493 return Err(LeaseError::Invalid(
1494 "cluster Commit link does not move backward in sequence and epoch"
1495 .into(),
1496 ));
1497 }
1498 if self.outcome_floor.as_ref().is_some_and(|floor| {
1499 previous.epoch < floor.artifact_before_epoch
1500 && Some(previous) != floor.committed_anchor_link
1501 }) {
1502 return Err(LeaseError::Invalid(
1503 "cluster Commit link crosses the artifact floor without its exact anchor"
1504 .into(),
1505 ));
1506 }
1507 }
1508 } else if self.previous_commit.is_some() {
1509 return Err(LeaseError::Invalid(
1510 "cluster Abort carries a previous-Commit link".into(),
1511 ));
1512 }
1513 if let Some(floor) = &self.outcome_floor {
1514 if outcome.deployment_id != floor.deployment_id
1515 || outcome.epoch < floor.authority_before_epoch
1516 {
1517 return Err(LeaseError::Invalid(
1518 "cluster outcome is below or outside its durable authority floor".into(),
1519 ));
1520 }
1521 }
1522 } else {
1523 if self.previous_outcome.is_some() || self.previous_commit.is_some() {
1524 return Err(LeaseError::Invalid(
1525 "non-outcome authority record carries an outcome-chain link".into(),
1526 ));
1527 }
1528 if self.outcome_head.is_some_and(|head| {
1529 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1530 }) {
1531 return Err(LeaseError::Invalid(
1532 "authority outcome head is outside the durable sequence".into(),
1533 ));
1534 }
1535 }
1536 if self.commit_head.is_some_and(|head| {
1537 head.sequence > self.lease.seq || head.epoch == 0 || head.checkpoint_id == 0
1538 }) {
1539 return Err(LeaseError::Invalid(
1540 "authority Commit head is outside the durable sequence".into(),
1541 ));
1542 }
1543 match (self.commit_head, self.outcome_head) {
1544 (Some(commit), Some(terminal)) => {
1545 let equal = commit == terminal;
1546 let strictly_older = commit.sequence < terminal.sequence
1547 && commit.epoch < terminal.epoch
1548 && commit.checkpoint_id < terminal.checkpoint_id;
1549 if !equal && !strictly_older {
1550 return Err(LeaseError::Invalid(
1551 "authority Commit head is not ordered behind the terminal head".into(),
1552 ));
1553 }
1554 if self
1555 .checkpoint_outcome
1556 .as_ref()
1557 .is_some_and(|outcome| !outcome.is_commit())
1558 && !strictly_older
1559 {
1560 return Err(LeaseError::Invalid(
1561 "cluster Abort did not preserve a strictly older Commit head".into(),
1562 ));
1563 }
1564 }
1565 (Some(_), None) => {
1566 return Err(LeaseError::Invalid(
1567 "authority Commit head exists without terminal continuity".into(),
1568 ));
1569 }
1570 (None, _) => {}
1571 }
1572 Ok(())
1573 }
1574
1575 fn validate_outcome_floor(&self) -> Result<(), LeaseError> {
1576 if let Some(floor) = self.outcome_floor.as_ref() {
1577 floor.validate()?;
1578 if floor
1579 .terminal_anchor_link
1580 .is_some_and(|link| link.sequence >= self.lease.seq)
1581 {
1582 return Err(LeaseError::Invalid(
1583 "outcome floor anchor is outside the authority sequence".into(),
1584 ));
1585 }
1586 if floor
1587 .committed_anchor_link
1588 .is_some_and(|link| link.sequence >= self.lease.seq)
1589 {
1590 return Err(LeaseError::Invalid(
1591 "outcome floor Commit anchor is outside the authority sequence".into(),
1592 ));
1593 }
1594 if let Some(head) = self.outcome_head {
1595 if head.epoch < floor.authority_before_epoch
1596 && Some(head) != floor.terminal_anchor_link
1597 {
1598 return Err(LeaseError::Invalid(
1599 "authority outcome head does not meet its durable compaction floor".into(),
1600 ));
1601 }
1602 } else if floor.terminal_anchor_link.is_some() {
1603 return Err(LeaseError::Invalid(
1604 "outcome floor anchor is disconnected from the authority head".into(),
1605 ));
1606 }
1607 if let Some(head) = self.commit_head {
1608 if head.epoch < floor.artifact_before_epoch
1609 && Some(head) != floor.committed_anchor_link
1610 {
1611 return Err(LeaseError::Invalid(
1612 "authority Commit head does not meet its durable artifact floor".into(),
1613 ));
1614 }
1615 } else if floor.committed_anchor_link.is_some() {
1616 return Err(LeaseError::Invalid(
1617 "outcome floor Commit anchor is disconnected from the Commit head".into(),
1618 ));
1619 }
1620 }
1621 Ok(())
1622 }
1623
1624 fn validate_artifact_cleanup(&self) -> Result<(), LeaseError> {
1625 let Some(cursor) = self.artifact_cleanup.as_ref() else {
1626 return Ok(());
1627 };
1628 cursor.validate()?;
1629 let floor = self.outcome_floor.as_ref().ok_or_else(|| {
1630 LeaseError::Invalid(
1631 "cluster artifact cleanup exists without an outcome retention floor".into(),
1632 )
1633 })?;
1634 if floor.artifact_before_epoch != cursor.protected.epoch
1635 || self
1636 .commit_head
1637 .is_none_or(|head| head.epoch < cursor.protected.epoch)
1638 {
1639 return Err(LeaseError::Invalid(
1640 "cluster artifact cleanup is not bound to its protected retention cut".into(),
1641 ));
1642 }
1643 Ok(())
1644 }
1645
1646 fn validate_checkpoint_artifact_inventory(&self) -> Result<(), LeaseError> {
1647 let (Some(inventory), Some(admitting_proof)) = (
1648 self.active_checkpoint_artifacts.as_ref(),
1649 self.active_checkpoint_artifact_leader_proof.as_ref(),
1650 ) else {
1651 if self.active_checkpoint_artifacts.is_some()
1652 || self.active_checkpoint_artifact_leader_proof.is_some()
1653 {
1654 return Err(LeaseError::Invalid(
1655 "cluster checkpoint artifact inventory has incomplete leader authority".into(),
1656 ));
1657 }
1658 return Ok(());
1659 };
1660
1661 inventory.validate().map_err(|error| {
1662 LeaseError::Invalid(format!("cluster checkpoint artifact inventory: {error}"))
1663 })?;
1664 let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
1665 LeaseError::Invalid(
1666 "cluster checkpoint artifact inventory has no assignment fence".into(),
1667 )
1668 })?;
1669 if !admitting_proof.is_canonical()
1670 || assignment_fence.participant_incarnation(admitting_proof.owner.node_id)
1671 != Some(admitting_proof.owner.boot_id)
1672 {
1673 return Err(LeaseError::Invalid(
1674 "cluster checkpoint artifact inventory has no canonical admitting leader".into(),
1675 ));
1676 }
1677 if self
1678 .outcome_floor
1679 .as_ref()
1680 .is_some_and(|floor| floor.deployment_id != inventory.deployment_id)
1681 {
1682 return Err(LeaseError::Invalid(
1683 "cluster checkpoint artifact inventory belongs to a foreign deployment".into(),
1684 ));
1685 }
1686 if let Some(head) = self.outcome_head {
1687 let attempt = crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
1688 if matches!(
1689 inventory.attempt.relation_to(attempt),
1690 crate::checkpoint::CheckpointAttemptRelation::Older
1691 | crate::checkpoint::CheckpointAttemptRelation::Conflict
1692 ) {
1693 return Err(LeaseError::Invalid(
1694 "cluster checkpoint artifact inventory is behind the terminal outcome head"
1695 .into(),
1696 ));
1697 }
1698 }
1699 if let Some(outcome) = self.checkpoint_outcome.as_ref() {
1700 let exact = outcome.deployment_id == inventory.deployment_id
1701 && outcome.epoch == inventory.attempt.epoch
1702 && outcome.checkpoint_id == inventory.attempt.checkpoint_id
1703 && outcome.assignment_fence.as_ref() == inventory.assignment_fence.as_ref();
1704 if outcome.is_commit() || !exact {
1705 return Err(LeaseError::Invalid(
1706 "terminal cluster outcome has inconsistent checkpoint artifact inventory"
1707 .into(),
1708 ));
1709 }
1710 }
1711 Ok(())
1712 }
1713
1714 fn validate_assignment_decision_chain(&self) -> Result<(), LeaseError> {
1715 if let Some(decision) = self.assignment_decision.as_ref() {
1716 decision.validate()?;
1717 if self
1718 .assignment_decision_floor
1719 .as_ref()
1720 .is_some_and(|floor| decision.target_version() < floor.before_target_version)
1721 {
1722 return Err(LeaseError::Invalid(
1723 "assignment decision is below its durable authority floor".into(),
1724 ));
1725 }
1726 if !self.lease.matches_proof(decision.leader_proof())
1727 || self.assignment_decision_head
1728 != Some(AssignmentDecisionLink {
1729 sequence: self.lease.seq,
1730 target_version: decision.target_version(),
1731 })
1732 {
1733 return Err(LeaseError::Invalid(
1734 "assignment decision is not bound to its exact authority sequence and term"
1735 .into(),
1736 ));
1737 }
1738 if let Some(previous) = self.previous_assignment_decision {
1739 if previous.sequence >= self.lease.seq
1740 || previous.target_version >= decision.target_version()
1741 {
1742 return Err(LeaseError::Invalid(
1743 "assignment decision link does not move backward".into(),
1744 ));
1745 }
1746 }
1747 } else {
1748 if self.previous_assignment_decision.is_some() {
1749 return Err(LeaseError::Invalid(
1750 "non-decision authority record carries a previous assignment-decision link"
1751 .into(),
1752 ));
1753 }
1754 if self
1755 .assignment_decision_head
1756 .is_some_and(|head| head.sequence > self.lease.seq || head.target_version == 0)
1757 {
1758 return Err(LeaseError::Invalid(
1759 "authority assignment-decision head is outside the durable sequence".into(),
1760 ));
1761 }
1762 }
1763 Ok(())
1764 }
1765
1766 fn validate_assignment_decision_floor(&self) -> Result<(), LeaseError> {
1767 if let Some(floor) = self.assignment_decision_floor.as_ref() {
1768 floor.validate()?;
1769 if floor
1770 .terminal_anchor_link
1771 .is_some_and(|link| link.sequence >= self.lease.seq)
1772 {
1773 return Err(LeaseError::Invalid(
1774 "assignment decision floor anchor is outside the authority sequence".into(),
1775 ));
1776 }
1777 if let Some(head) = self.assignment_decision_head {
1778 if head.target_version < floor.before_target_version
1779 && Some(head) != floor.terminal_anchor_link
1780 {
1781 return Err(LeaseError::Invalid(
1782 "authority assignment-decision head does not meet its durable floor".into(),
1783 ));
1784 }
1785 } else if floor.terminal_anchor_link.is_some() {
1786 return Err(LeaseError::Invalid(
1787 "assignment decision floor anchor is disconnected from the authority head"
1788 .into(),
1789 ));
1790 }
1791 }
1792 Ok(())
1793 }
1794
1795 fn validate_assignment_handoff_pin(&self) -> Result<(), LeaseError> {
1796 let Some(pin) = self.assignment_handoff_pin.as_ref() else {
1797 return Ok(());
1798 };
1799 pin.validate()?;
1800 if self.commit_head.is_none_or(|head| {
1801 head.epoch != pin.checkpoint.epoch || head.checkpoint_id != pin.checkpoint.checkpoint_id
1802 }) || self
1803 .outcome_floor
1804 .as_ref()
1805 .is_some_and(|floor| floor.artifact_before_epoch > pin.checkpoint.epoch)
1806 || self
1807 .artifact_cleanup
1808 .as_ref()
1809 .is_some_and(|cursor| cursor.protected.epoch > pin.checkpoint.epoch)
1810 {
1811 return Err(LeaseError::Invalid(
1812 "assignment handoff pin is outside its live checkpoint retention range".into(),
1813 ));
1814 }
1815 Ok(())
1816 }
1817
1818 fn validate_recovery_release(&self) -> Result<(), LeaseError> {
1819 match self.recovery_release_commit.as_ref() {
1820 Some(commit) => {
1821 commit.validate()?;
1822 let expected = RecoveryReleaseLink {
1823 sequence: self.lease.seq,
1824 terminal: commit.terminal.clone(),
1825 };
1826 if !self.lease.matches_proof(&commit.leader_proof)
1827 || self.recovery_release_head.as_ref() != Some(&expected)
1828 || self.recovery_fault_revision != self.lease.seq
1829 || self.recovery_fault_slots.iter().any(|slot| {
1830 slot.active || slot.disposition == RecoveryFaultDisposition::Terminal
1831 })
1832 {
1833 return Err(LeaseError::Invalid(
1834 "recovery release commit is not bound to its exact authority sequence, term, and settled fault inventory"
1835 .into(),
1836 ));
1837 }
1838 }
1839 None => {
1840 if let Some(head) = self.recovery_release_head.as_ref() {
1841 head.validate()?;
1842 if head.sequence >= self.lease.seq {
1843 return Err(LeaseError::Invalid(
1844 "recovery release head is outside the durable authority sequence"
1845 .into(),
1846 ));
1847 }
1848 if !self.recovery_fault_slots.iter().any(|slot| slot.active)
1849 && self.recovery_fault_revision != head.sequence
1850 {
1851 return Err(LeaseError::Invalid(
1852 "settled recovery fault inventory is not bound to its release head"
1853 .into(),
1854 ));
1855 }
1856 }
1857 }
1858 }
1859 Ok(())
1860 }
1861}
1862
1863#[derive(Debug, Clone, PartialEq, Eq)]
1865pub enum LeaseOutcome {
1866 Acquired(LeaderLease),
1868 Held(LeaderLease),
1870}
1871
1872enum AuthorityCreateOutcome {
1873 Created,
1874 ExistingIdentical,
1875 Contended(Box<LeaderAuthorityRecord>),
1876}
1877
1878#[derive(Clone, Copy)]
1879enum SameOwnerToken {
1880 #[cfg(test)]
1881 Preserve,
1882 Rotate,
1883 Exact(u64),
1884}
1885
1886#[derive(Debug)]
1888pub struct LeaderLeaseObservation {
1889 lease: LeaderLease,
1890 started: Instant,
1891}
1892
1893#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1896pub struct LeaderCandidacy {
1897 eligible: bool,
1898 generation: u64,
1899}
1900
1901impl LeaderCandidacy {
1902 pub(crate) const fn initial(eligible: bool) -> Self {
1903 Self {
1904 eligible,
1905 generation: 1,
1906 }
1907 }
1908
1909 pub(crate) fn transition(self, eligible: bool) -> Option<Self> {
1910 let generation = if self.eligible && !eligible {
1911 self.generation.checked_add(1)?
1912 } else {
1913 self.generation
1914 };
1915 Some(Self {
1916 eligible,
1917 generation,
1918 })
1919 }
1920
1921 pub(crate) const fn terminal() -> Self {
1922 Self {
1923 eligible: false,
1924 generation: u64::MAX,
1925 }
1926 }
1927
1928 #[must_use]
1930 pub const fn is_eligible(self) -> bool {
1931 self.eligible
1932 }
1933}
1934
1935#[derive(Debug, thiserror::Error)]
1937pub enum LeaseError {
1938 #[error("object store I/O: {0}")]
1940 Io(String),
1941 #[error("invalid leader lease: {0}")]
1943 Invalid(String),
1944 #[error("leader lease fenced: {0}")]
1946 Fenced(String),
1947 #[error("JSON: {0}")]
1949 Json(#[from] serde_json::Error),
1950}
1951
1952#[derive(Debug, thiserror::Error)]
1954pub enum ClusterCheckpointAuthorityError {
1955 #[error("cluster checkpoint authority is not installed")]
1957 NotConfigured,
1958 #[error("leader authority: {0}")]
1960 Authority(#[from] LeaseError),
1961 #[error("checkpoint decision: {0}")]
1963 Decision(#[from] DecisionError),
1964 #[error("cluster checkpoint decision was fenced by a different durable leader term")]
1966 Fenced,
1967}
1968
1969#[derive(Debug, Clone, PartialEq, Eq)]
1971pub struct ClusterOutcomeRetentionBoundary {
1972 pub artifact_before_epoch: u64,
1974 pub terminal_before_epoch: u64,
1976 pub committed_anchor: Option<CheckpointOutcome>,
1978 pub terminal_anchor: Option<CheckpointOutcome>,
1980}
1981
1982impl ClusterOutcomeRetentionBoundary {
1983 fn from_floor(floor: Option<&AuthorityOutcomeFloor>) -> Self {
1984 floor.map_or(
1985 Self {
1986 artifact_before_epoch: 0,
1987 terminal_before_epoch: 0,
1988 committed_anchor: None,
1989 terminal_anchor: None,
1990 },
1991 |floor| Self {
1992 artifact_before_epoch: floor.artifact_before_epoch,
1993 terminal_before_epoch: floor.authority_before_epoch,
1994 committed_anchor: floor.committed_anchor.clone(),
1995 terminal_anchor: floor.terminal_anchor.clone(),
1996 },
1997 )
1998 }
1999}
2000
2001#[derive(Debug, Clone, PartialEq, Eq)]
2003pub struct ClusterOutcomeInventory {
2004 pub outcomes: Vec<CheckpointOutcome>,
2006 pub retention_boundary: ClusterOutcomeRetentionBoundary,
2008}
2009
2010pub struct LeaderLeaseStore {
2012 store: Arc<dyn ObjectStore>,
2013 ttl_ms: i64,
2014 prune_running: Arc<AtomicBool>,
2015 outcome_audit_cache: Mutex<Option<ClusterOutcomeAuditCache>>,
2016 outcome_audit_flights: Mutex<Vec<ClusterOutcomeAuditFlight>>,
2017}
2018
2019#[derive(Clone, PartialEq, Eq)]
2020struct ClusterOutcomeAuditKey {
2021 terminal_head: Option<OutcomeLink>,
2022 commit_head: Option<OutcomeLink>,
2023 floor: Option<AuthorityOutcomeFloor>,
2024}
2025
2026struct ClusterOutcomeAuditCache {
2027 key: ClusterOutcomeAuditKey,
2028 authority_sequence: u64,
2029 snapshot: ClusterOutcomeAuditSnapshot,
2030}
2031
2032#[derive(Clone)]
2033struct ClusterOutcomeAuditSnapshot {
2034 outcomes: Arc<[CheckpointOutcome]>,
2035 terminal_links: Arc<[OutcomeLink]>,
2036 commit_links: Arc<[OutcomeLink]>,
2037}
2038
2039struct ClusterOutcomeAuditFlight {
2040 key: ClusterOutcomeAuditKey,
2041 gate: Weak<tokio::sync::Mutex<()>>,
2042}
2043
2044struct PruneLatchGuard(Arc<AtomicBool>);
2045
2046impl Drop for PruneLatchGuard {
2047 fn drop(&mut self) {
2048 self.0.store(false, Ordering::Release);
2049 }
2050}
2051
2052impl std::fmt::Debug for LeaderLeaseStore {
2053 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2054 formatter
2055 .debug_struct("LeaderLeaseStore")
2056 .field("ttl_ms", &self.ttl_ms)
2057 .finish_non_exhaustive()
2058 }
2059}
2060
2061impl LeaderLeaseStore {
2062 #[must_use]
2067 pub fn new(store: Arc<dyn ObjectStore>, ttl_ms: i64) -> Self {
2068 Self {
2069 store,
2070 ttl_ms,
2071 prune_running: Arc::new(AtomicBool::new(false)),
2072 outcome_audit_cache: Mutex::new(None),
2073 outcome_audit_flights: Mutex::new(Vec::new()),
2074 }
2075 }
2076
2077 pub async fn verify_store_contract(&self, timeout: Duration) -> Result<(), LeaseError> {
2086 probe_object_store_conditional_update(
2087 self.store.as_ref(),
2088 STORE_CONTRACT_PROBE_PREFIX,
2089 timeout,
2090 )
2091 .await
2092 .map_err(|error| match error {
2093 CheckpointStoreError::Invalid(message) => LeaseError::Invalid(message),
2094 error => LeaseError::Io(error.to_string()),
2095 })
2096 }
2097
2098 fn recovery_fault_inventory_from(record: &LeaderAuthorityRecord) -> RecoveryFaultInventory {
2099 RecoveryFaultInventory {
2100 revision: record.recovery_fault_revision,
2101 faults: record
2102 .recovery_fault_slots
2103 .iter()
2104 .filter(|slot| slot.active)
2105 .map(AuthorityRecoveryFaultSlot::fault)
2106 .collect(),
2107 }
2108 }
2109
2110 pub(crate) async fn recovery_fault_inventory(
2111 &self,
2112 ) -> Result<RecoveryFaultInventory, ClusterCheckpointAuthorityError> {
2113 let record = self
2114 .load_record()
2115 .await?
2116 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2117 Ok(Self::recovery_fault_inventory_from(&record))
2118 }
2119
2120 #[cfg(test)]
2121 pub(crate) async fn record_recovery_fault(
2122 &self,
2123 publisher: RecoveryFaultPublisher,
2124 request_sequence: u64,
2125 ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2126 self.record_recovery_fault_with_disposition(
2127 publisher,
2128 request_sequence,
2129 RecoveryFaultDisposition::Recoverable,
2130 )
2131 .await
2132 }
2133
2134 pub(crate) async fn record_recovery_fault_with_disposition(
2135 &self,
2136 publisher: RecoveryFaultPublisher,
2137 request_sequence: u64,
2138 disposition: RecoveryFaultDisposition,
2139 ) -> Result<RecordRecoveryFaultResult, ClusterCheckpointAuthorityError> {
2140 publisher.validate().map_err(LeaseError::Invalid)?;
2141 if request_sequence == 0 {
2142 return Err(LeaseError::Invalid(
2143 "recovery fault request sequence must be nonzero".into(),
2144 )
2145 .into());
2146 }
2147
2148 loop {
2149 let head = self
2150 .load_published_authority_head()
2151 .await?
2152 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2153 let current = &head.record;
2154 let node_id = publisher.participant.node_id;
2155 let slot_index = current
2156 .recovery_fault_slots
2157 .binary_search_by_key(&node_id, |slot| slot.publisher.participant.node_id);
2158 let (insert_at, replace) = match slot_index {
2159 Ok(index) => {
2160 let slot = ¤t.recovery_fault_slots[index];
2161 if slot.matches_request(publisher, request_sequence, disposition) {
2162 return Ok(if slot.active {
2163 RecordRecoveryFaultResult::Active
2164 } else {
2165 RecordRecoveryFaultResult::AlreadyCleared
2166 });
2167 }
2168 if slot.disposition == RecoveryFaultDisposition::Terminal {
2171 return Ok(RecordRecoveryFaultResult::TerminalFenceActive);
2172 }
2173 if publisher.process_term < slot.publisher.process_term {
2174 return Ok(RecordRecoveryFaultResult::Superseded);
2175 }
2176 if publisher.process_term == slot.publisher.process_term
2177 && publisher.participant.boot_incarnation
2178 != slot.publisher.participant.boot_incarnation
2179 {
2180 return Err(LeaseError::Invalid(format!(
2181 "stable node {node_id} has two recovery fault publishers for process term {}",
2182 publisher.process_term
2183 ))
2184 .into());
2185 }
2186 if publisher.process_term == slot.publisher.process_term
2187 && request_sequence < slot.request_sequence
2188 {
2189 return Ok(RecordRecoveryFaultResult::CoveredByNewerRequest);
2190 }
2191 (index, true)
2192 }
2193 Err(index) => (index, false),
2194 };
2195
2196 let sequence =
2197 current.lease.seq.checked_add(1).ok_or_else(|| {
2198 LeaseError::Invalid("leader authority sequence exhausted".into())
2199 })?;
2200 let mut lease = current.lease.clone();
2201 lease.seq = sequence;
2202 let mut candidate = current.preserve_with_lease(lease);
2203 let slot = AuthorityRecoveryFaultSlot {
2204 publisher,
2205 request_sequence,
2206 fault_sequence: sequence,
2210 disposition,
2211 active: true,
2212 };
2213 if replace {
2214 candidate.recovery_fault_slots[insert_at] = slot;
2215 } else {
2216 candidate.recovery_fault_slots.insert(insert_at, slot);
2217 }
2218 candidate.recovery_fault_revision = sequence;
2219 candidate.abort_topology_preparation(super::topology::TopologyAbortReason::Recovery);
2220 candidate.validate()?;
2221
2222 match self
2223 .create_authority_record(Some(&head), &candidate)
2224 .await?
2225 {
2226 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2227 return Ok(RecordRecoveryFaultResult::Active);
2228 }
2229 AuthorityCreateOutcome::Contended(_) => tokio::task::yield_now().await,
2230 }
2231 }
2232 }
2233
2234 pub(crate) async fn authorize_recovery_release(
2235 &self,
2236 clearer: RecoveryFaultPublisher,
2237 terminal: &RecoveryAnnouncement,
2238 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2239 self.authorize_recovery_release_with_topology(clearer, terminal, None)
2240 .await
2241 }
2242
2243 pub(crate) async fn authorize_recovery_release_with_topology(
2244 &self,
2245 clearer: RecoveryFaultPublisher,
2246 terminal: &RecoveryAnnouncement,
2247 context: Option<(&AssignmentSnapshotStore, &super::ProcessLeaseAuthority)>,
2248 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2249 clearer.validate().map_err(LeaseError::Invalid)?;
2250 let (_, terminal_reference) = encode_recovery_release_terminal(terminal)?;
2251 let reporter = NodeId(clearer.participant.node_id);
2252 let expected = terminal.round.fault_sequence(reporter);
2253 let current = self
2254 .load_record()
2255 .await?
2256 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2257 self.audit_topology_recovery_terminal(¤t, terminal, context)
2258 .await?;
2259 if terminal.round.topology_binding().is_some()
2260 && current
2261 .committed_topology_operation()
2262 .is_none_or(|operation| operation.phase != super::TopologyAdmissionPhase::Active)
2263 {
2264 return Ok(false);
2265 }
2266 let Some(release_head) = current.recovery_release_head.as_ref() else {
2267 return Ok(false);
2268 };
2269 if release_head.terminal != terminal_reference
2270 || current.recovery_fault_revision != release_head.sequence
2271 {
2272 return Ok(false);
2273 }
2274 if current
2275 .recovery_fault_slots
2276 .iter()
2277 .any(|slot| slot.active || slot.disposition == RecoveryFaultDisposition::Terminal)
2278 {
2279 return Ok(false);
2280 }
2281 let slot_index = current
2282 .recovery_fault_slots
2283 .binary_search_by_key(&reporter.0, |slot| slot.publisher.participant.node_id);
2284 let Some(expected_sequence) = expected else {
2285 return Ok(match slot_index {
2286 Ok(index) => !current.recovery_fault_slots[index].active,
2287 Err(_) => true,
2288 });
2289 };
2290 let Ok(index) = slot_index else {
2291 return Ok(false);
2292 };
2293 let slot = ¤t.recovery_fault_slots[index];
2294 Ok(slot.publisher == clearer && slot.fault_sequence == expected_sequence && !slot.active)
2295 }
2296
2297 pub(crate) async fn stage_recovery_release_terminal(
2298 &self,
2299 terminal: &RecoveryAnnouncement,
2300 ) -> Result<RecoveryReleaseTerminalRef, ClusterCheckpointAuthorityError> {
2301 let (encoded, reference) = encode_recovery_release_terminal(terminal)?;
2302 let path = recovery_release_terminal_path(&reference);
2303 let put_error = match self
2304 .store
2305 .put_opts(
2306 &path,
2307 PutPayload::from(encoded),
2308 PutOptions {
2309 mode: PutMode::Create,
2310 ..PutOptions::default()
2311 },
2312 )
2313 .await
2314 {
2315 Ok(_)
2316 | Err(
2317 object_store::Error::AlreadyExists { .. }
2318 | object_store::Error::Precondition { .. },
2319 ) => None,
2320 Err(error) => Some(error),
2321 };
2322 match self.load_recovery_release_terminal(&reference).await {
2323 Ok(stored) if stored == *terminal => Ok(reference),
2324 Ok(_) => Err(LeaseError::Invalid(format!(
2325 "recovery release terminal '{}' differs from the proposed content",
2326 reference.sha256
2327 ))
2328 .into()),
2329 Err(reconcile_error) => {
2330 if let Some(put_error) = put_error {
2331 Err(LeaseError::Io(format!(
2332 "recovery release terminal write failed ({put_error}); reconciliation failed ({reconcile_error})"
2333 ))
2334 .into())
2335 } else {
2336 Err(reconcile_error)
2337 }
2338 }
2339 }
2340 }
2341
2342 pub(crate) async fn load_recovery_release_terminal(
2343 &self,
2344 reference: &RecoveryReleaseTerminalRef,
2345 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2346 reference.validate()?;
2347 let result = match self
2348 .store
2349 .get(&recovery_release_terminal_path(reference))
2350 .await
2351 {
2352 Ok(result) => result,
2353 Err(object_store::Error::NotFound { .. }) => {
2354 return Err(LeaseError::Invalid(format!(
2355 "recovery release terminal '{}' is missing",
2356 reference.sha256
2357 ))
2358 .into());
2359 }
2360 Err(error) => return Err(LeaseError::Io(error.to_string()).into()),
2361 };
2362 if result.meta.size != reference.encoded_len {
2363 return Err(LeaseError::Invalid(format!(
2364 "recovery release terminal '{}' is {} bytes, expected {}",
2365 reference.sha256, result.meta.size, reference.encoded_len
2366 ))
2367 .into());
2368 }
2369 let bytes = result
2370 .bytes()
2371 .await
2372 .map_err(|error| LeaseError::Io(error.to_string()))?;
2373 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
2374 return Err(LeaseError::Invalid(format!(
2375 "recovery release terminal '{}' payload length changed while reading",
2376 reference.sha256
2377 ))
2378 .into());
2379 }
2380 let terminal: RecoveryAnnouncement = serde_json::from_slice(&bytes).map_err(|error| {
2381 LeaseError::Invalid(format!(
2382 "recovery release terminal '{}': {error}",
2383 reference.sha256
2384 ))
2385 })?;
2386 let (canonical, actual_reference) = encode_recovery_release_terminal(&terminal)?;
2387 if &actual_reference != reference || canonical.as_ref() != bytes.as_ref() {
2388 return Err(LeaseError::Invalid(format!(
2389 "recovery release terminal '{}' does not match its content-addressed reference",
2390 reference.sha256
2391 ))
2392 .into());
2393 }
2394 Ok(terminal)
2395 }
2396
2397 pub(crate) async fn record_recovery_release_commit(
2398 &self,
2399 proof: &LeaderProof,
2400 reference: RecoveryReleaseTerminalRef,
2401 ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2402 self.record_recovery_release_commit_with_topology(proof, reference, None)
2403 .await
2404 }
2405
2406 pub(crate) async fn record_recovery_release_commit_with_topology(
2407 &self,
2408 proof: &LeaderProof,
2409 reference: RecoveryReleaseTerminalRef,
2410 context: Option<(&AssignmentSnapshotStore, &super::ProcessLeaseAuthority)>,
2411 ) -> Result<RecordRecoveryReleaseCommitResult, ClusterCheckpointAuthorityError> {
2412 reference.validate()?;
2413 let terminal = self.load_recovery_release_terminal(&reference).await?;
2414 if &terminal.round.leader_proof != proof || !proof.is_canonical() {
2415 return Err(ClusterCheckpointAuthorityError::Fenced);
2416 }
2417
2418 loop {
2419 let published = self
2420 .load_published_authority_head()
2421 .await?
2422 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2423 let current = &published.record;
2424 self.audit_topology_recovery_terminal(current, &terminal, context)
2425 .await?;
2426 if let Some(winner) = current.recovery_release_head.as_ref() {
2427 if winner.terminal == reference
2428 || winner.terminal.generation() >= reference.generation()
2429 {
2430 self.load_recovery_release_terminal(&winner.terminal)
2431 .await?;
2432 return if winner.terminal == reference {
2433 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2434 } else {
2435 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2436 winner: winner.terminal.clone(),
2437 })
2438 };
2439 }
2440 }
2441 if !current.lease.matches_proof(proof) {
2442 return Err(ClusterCheckpointAuthorityError::Fenced);
2443 }
2444 let RecoverPhase::ReleaseCommitted { epoch } = terminal.phase else {
2445 return Err(DecisionError::Conflict(
2446 "recovery terminal must be a committed Release".into(),
2447 )
2448 .into());
2449 };
2450 self.audit_recovery_topology_from(current, &terminal.round, Some(epoch), context)
2451 .await?;
2452 let fault_inventory = Self::recovery_fault_inventory_from(current);
2453 if fault_inventory.revision != terminal.round.fault_revision()
2454 || fault_inventory.faults != terminal.round.faults
2455 || fault_inventory
2456 .faults
2457 .iter()
2458 .copied()
2459 .any(RecoveryFault::is_terminal)
2460 {
2461 return Ok(RecordRecoveryReleaseCommitResult::FaultsChanged);
2462 }
2463
2464 let base_sequence = current.lease.seq;
2465 let sequence = base_sequence
2466 .checked_add(1)
2467 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
2468 let mut candidate = current.preserve_with_lease(LeaderLease {
2469 seq: sequence,
2470 renewal_sequence: current.lease.renewal_sequence,
2471 token: current.lease.token,
2472 owner: current.lease.owner.clone(),
2473 expires_at_ms: current.lease.expires_at_ms,
2474 catalog_manifest: current.lease.catalog_manifest.clone(),
2475 });
2476 if terminal.round.topology_binding().is_some() {
2477 candidate.version = candidate.version.max(TOPOLOGY_RECOVERY_RECORD_VERSION);
2478 Self::release_recovered_topology(&mut candidate, &terminal.round, sequence)?;
2479 }
2480 candidate.recovery_fault_slots.retain(|slot| {
2483 slot.active
2484 && terminal
2485 .round
2486 .stopped_participant_incarnation(NodeId(slot.publisher.participant.node_id))
2487 == Some(slot.publisher.participant.boot_incarnation)
2488 });
2489 for slot in &mut candidate.recovery_fault_slots {
2490 slot.active = false;
2491 }
2492 candidate.recovery_fault_revision = sequence;
2493 candidate.recovery_release_commit = Some(AuthorityRecoveryReleaseCommit {
2494 terminal: reference.clone(),
2495 leader_proof: proof.clone(),
2496 });
2497 candidate.recovery_release_head = Some(RecoveryReleaseLink {
2498 sequence,
2499 terminal: reference.clone(),
2500 });
2501 candidate.validate()?;
2502
2503 match self
2504 .create_authority_record(Some(&published), &candidate)
2505 .await?
2506 {
2507 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
2508 return Ok(RecordRecoveryReleaseCommitResult::Created(reference));
2509 }
2510 AuthorityCreateOutcome::Contended(winner_head) => {
2511 if let Some(winner) = winner_head.recovery_release_head.as_ref() {
2512 if winner.terminal == reference
2513 || winner.terminal.generation() >= reference.generation()
2514 {
2515 self.load_recovery_release_terminal(&winner.terminal)
2516 .await?;
2517 return if winner.terminal == reference {
2518 Ok(RecordRecoveryReleaseCommitResult::Unchanged(reference))
2519 } else {
2520 Ok(RecordRecoveryReleaseCommitResult::Conflict {
2521 winner: winner.terminal.clone(),
2522 })
2523 };
2524 }
2525 }
2526 if !winner_head.lease.matches_proof(proof) {
2527 return Err(ClusterCheckpointAuthorityError::Fenced);
2528 }
2529 if winner_head.lease.seq <= base_sequence {
2530 return Err(LeaseError::Invalid(
2531 "recovery release authority contention did not advance the sequence"
2532 .into(),
2533 )
2534 .into());
2535 }
2536 tokio::task::yield_now().await;
2537 }
2538 }
2539 }
2540 }
2541
2542 async fn recovery_release_terminal_from(
2543 &self,
2544 head: &LeaderAuthorityRecord,
2545 link: &RecoveryReleaseLink,
2546 ) -> Result<RecoveryAnnouncement, ClusterCheckpointAuthorityError> {
2547 let admission = if link.sequence == head.lease.seq {
2548 head.clone()
2549 } else {
2550 read_authority_record(self.store.as_ref(), link.sequence)
2551 .await?
2552 .ok_or_else(|| {
2553 LeaseError::Invalid(
2554 "recovery release admission is missing from retained authority history"
2555 .into(),
2556 )
2557 })?
2558 };
2559 let commit = admission
2560 .recovery_release_commit
2561 .as_ref()
2562 .filter(|commit| {
2563 commit.terminal == link.terminal
2564 && admission.recovery_release_head.as_ref() == Some(link)
2565 })
2566 .ok_or_else(|| {
2567 LeaseError::Invalid(
2568 "recovery release head does not match its admitting authority record".into(),
2569 )
2570 })?;
2571 let terminal = self.load_recovery_release_terminal(&link.terminal).await?;
2572 if terminal.round.leader_proof != commit.leader_proof {
2573 return Err(LeaseError::Invalid(
2574 "recovery release terminal does not match its admitting leader proof".into(),
2575 )
2576 .into());
2577 }
2578 Ok(terminal)
2579 }
2580
2581 pub(crate) async fn recovery_admission_snapshot(
2582 &self,
2583 ) -> Result<RecoveryAdmissionSnapshot, ClusterCheckpointAuthorityError> {
2584 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2585 let head = self
2586 .load_record()
2587 .await?
2588 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2589 if let Some(operation) = head.committed_topology_operation() {
2590 self.audit_topology_operation(operation).await?;
2591 }
2592 let Some(link) = head.recovery_release_head.clone() else {
2593 return Ok(RecoveryAdmissionSnapshot {
2594 committed_release: None,
2595 fault_inventory: Self::recovery_fault_inventory_from(&head),
2596 authority_sequence: head.lease.seq,
2597 release_head: None,
2598 topology: head
2599 .committed_topology_operation()
2600 .and_then(|operation| operation.commit.clone()),
2601 });
2602 };
2603 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2604 let rechecked = self
2605 .load_record()
2606 .await?
2607 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2608 if rechecked.lease.seq >= head.lease.seq
2609 && rechecked.recovery_release_head == head.recovery_release_head
2610 && rechecked.recovery_fault_revision == head.recovery_fault_revision
2611 && rechecked.recovery_fault_slots == head.recovery_fault_slots
2612 && rechecked.committed_topology_operation() == head.committed_topology_operation()
2613 {
2614 return Ok(RecoveryAdmissionSnapshot {
2615 committed_release: Some(terminal?),
2616 fault_inventory: Self::recovery_fault_inventory_from(&rechecked),
2617 authority_sequence: rechecked.lease.seq,
2618 release_head: rechecked.recovery_release_head.clone(),
2619 topology: rechecked
2620 .committed_topology_operation()
2621 .and_then(|operation| operation.commit.clone()),
2622 });
2623 }
2624 tokio::task::yield_now().await;
2625 }
2626 Err(LeaseError::Io(format!(
2627 "recovery admission changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2628 ))
2629 .into())
2630 }
2631
2632 pub(crate) async fn recovery_admission_is_current(
2633 &self,
2634 snapshot: &RecoveryAdmissionSnapshot,
2635 leader_proof: &LeaderProof,
2636 ) -> Result<bool, ClusterCheckpointAuthorityError> {
2637 let current = self
2638 .load_record()
2639 .await?
2640 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2641 let inventory = Self::recovery_fault_inventory_from(¤t);
2642 let topology = current
2643 .committed_topology_operation()
2644 .and_then(|operation| operation.commit.as_ref());
2645 let release_matches_topology = current.live_topology_migration().is_none_or(|operation| {
2646 operation.phase == super::TopologyAdmissionPhase::Active
2647 && snapshot
2648 .committed_release
2649 .as_ref()
2650 .and_then(|release| release.round.topology_binding())
2651 .is_some_and(|binding| Some(binding.commit()) == operation.commit.as_ref())
2652 });
2653 Ok(topology == snapshot.topology.as_ref()
2654 && release_matches_topology
2655 && current.lease.matches_proof(leader_proof)
2656 && current.lease.seq >= snapshot.authority_sequence
2657 && current.recovery_release_head == snapshot.release_head
2658 && inventory == snapshot.fault_inventory
2659 && inventory.faults().is_empty())
2660 }
2661
2662 pub(crate) async fn latest_recovery_release_terminal(
2663 &self,
2664 ) -> Result<Option<RecoveryAnnouncement>, ClusterCheckpointAuthorityError> {
2665 for _ in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
2666 let Some(head) = self.load_record().await? else {
2667 return Ok(None);
2668 };
2669 let Some(link) = head.recovery_release_head.clone() else {
2670 return Ok(None);
2671 };
2672 let terminal = self.recovery_release_terminal_from(&head, &link).await;
2673 let rechecked = self
2674 .load_record()
2675 .await?
2676 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
2677 if rechecked.recovery_release_head.as_ref() == Some(&link) {
2678 return Ok(Some(terminal?));
2679 }
2680 tokio::task::yield_now().await;
2681 }
2682 Err(LeaseError::Io(format!(
2683 "recovery release head changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
2684 ))
2685 .into())
2686 }
2687
2688 fn ttl(&self) -> Result<Duration, LeaseError> {
2689 let ttl = u64::try_from(self.ttl_ms)
2690 .map_err(|_| LeaseError::Invalid("lease TTL must be positive".into()))?;
2691 if ttl == 0 {
2692 return Err(LeaseError::Invalid("lease TTL must be positive".into()));
2693 }
2694 Ok(Duration::from_millis(ttl))
2695 }
2696
2697 fn diagnostic_expiry(&self, now_ms: i64) -> Result<i64, LeaseError> {
2698 now_ms
2699 .checked_add(self.ttl_ms)
2700 .ok_or_else(|| LeaseError::Invalid("diagnostic lease expiry overflow".into()))
2701 }
2702
2703 #[cfg(test)]
2704 async fn list_seqs(&self) -> Result<Vec<u64>, LeaseError> {
2705 let prefix = OsPath::from(LEASE_PREFIX);
2706 let mut entries = self.store.list(Some(&prefix));
2707 let mut sequences = Vec::new();
2708 while let Some(entry) = entries.next().await {
2709 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
2710 if sequences.len() == MAX_TEST_LEADER_LEASE_RECORDS {
2711 return Err(LeaseError::Invalid(format!(
2712 "test leader history exceeds {MAX_TEST_LEADER_LEASE_RECORDS} records"
2713 )));
2714 }
2715 sequences.push(lease_sequence_from_path(&entry.location)?);
2716 }
2717 sequences.sort_unstable();
2718 sequences.dedup();
2719 Ok(sequences)
2720 }
2721
2722 fn schedule_history_prune(&self) {
2723 if self
2724 .prune_running
2725 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
2726 .is_err()
2727 {
2728 return;
2729 }
2730 let Ok(runtime) = tokio::runtime::Handle::try_current() else {
2731 self.prune_running.store(false, Ordering::Release);
2732 return;
2733 };
2734 let store = Arc::clone(&self.store);
2735 let prune_running = Arc::clone(&self.prune_running);
2736 let grace_ms = self.ttl_ms.saturating_mul(2).max(1);
2737 runtime.spawn(async move {
2738 let _latch = PruneLatchGuard(prune_running);
2739 match tokio::time::timeout(
2740 LEADER_LEASE_PRUNE_TIMEOUT,
2741 Self::prune_history(&store, grace_ms),
2742 )
2743 .await
2744 {
2745 Ok(Ok(())) => {}
2746 Ok(Err(error)) => {
2747 tracing::warn!(%error, "leader lease history prune failed");
2748 }
2749 Err(_) => {
2750 tracing::warn!(
2751 timeout = ?LEADER_LEASE_PRUNE_TIMEOUT,
2752 "leader lease history prune timed out"
2753 );
2754 }
2755 }
2756 });
2757 }
2758
2759 async fn prune_history(store: &Arc<dyn ObjectStore>, grace_ms: i64) -> Result<(), LeaseError> {
2760 let authority = Self::new(Arc::clone(store), 1);
2761 let Some(head) = authority.load_record().await? else {
2762 return Ok(());
2763 };
2764 let head_sequence = head.lease.seq;
2765 let mut retained = BTreeSet::from([head_sequence]);
2766 authority
2767 .retain_topology_history(&head, &mut retained)
2768 .await?;
2769 if let Some(reservation) = &head.assignment_drain_reservation {
2770 retained.insert(reservation.authority_sequence);
2771 }
2772 if let Some(previous) = head_sequence
2773 .checked_sub(1)
2774 .filter(|sequence| *sequence != 0)
2775 {
2776 retained.insert(previous);
2777 }
2778 let floor = head
2779 .outcome_floor
2780 .as_ref()
2781 .map_or(0, |floor| floor.authority_before_epoch);
2782 let artifact_floor = head
2783 .outcome_floor
2784 .as_ref()
2785 .map_or(0, |floor| floor.artifact_before_epoch);
2786 let mut link = head.outcome_head;
2787 let mut outcome_links = 0;
2788 let mut terminal_commit_links = BTreeSet::new();
2789 let mut expected_commit_head = head.commit_head;
2790 while let Some(current) = link {
2791 if current.epoch < floor {
2792 if head
2793 .outcome_floor
2794 .as_ref()
2795 .and_then(|floor| floor.terminal_anchor_link)
2796 != Some(current)
2797 {
2798 return Err(LeaseError::Invalid(
2799 "outcome floor does not anchor the retained authority chain".into(),
2800 ));
2801 }
2802 break;
2803 }
2804 if !consume_live_authority_link(&mut outcome_links) {
2805 return Err(LeaseError::Invalid(format!(
2806 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2807 )));
2808 }
2809 let outcome_record = read_authority_record(store.as_ref(), current.sequence)
2810 .await?
2811 .filter(|record| {
2812 record.outcome_head == Some(current)
2813 && record
2814 .checkpoint_outcome
2815 .as_ref()
2816 .map(|outcome| (outcome.epoch, outcome.checkpoint_id))
2817 == Some((current.epoch, current.checkpoint_id))
2818 })
2819 .ok_or_else(|| {
2820 LeaseError::Invalid("retained outcome authority chain is broken".into())
2821 })?;
2822 if outcome_record.commit_head != expected_commit_head {
2823 return Err(LeaseError::Invalid(
2824 "retained outcome chain did not preserve the exact Commit head".into(),
2825 ));
2826 }
2827 retained.insert(current.sequence);
2828 if outcome_record
2829 .checkpoint_outcome
2830 .as_ref()
2831 .is_some_and(CheckpointOutcome::is_commit)
2832 {
2833 terminal_commit_links.insert(current);
2834 expected_commit_head = outcome_record.previous_commit;
2835 } else {
2836 expected_commit_head = outcome_record.commit_head;
2837 }
2838 link = outcome_record.previous_outcome;
2839 }
2840 let mut commit_link = head.commit_head;
2841 let mut commit_links = 0;
2842 let mut retained_commit_links = BTreeSet::new();
2843 while let Some(current) = commit_link {
2844 if current.epoch < artifact_floor {
2845 break;
2846 }
2847 if !consume_live_authority_link(&mut commit_links) {
2848 return Err(LeaseError::Invalid(format!(
2849 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2850 )));
2851 }
2852 let commit_record = read_authority_record(store.as_ref(), current.sequence)
2853 .await?
2854 .filter(|record| {
2855 record.commit_head == Some(current)
2856 && record.checkpoint_outcome.as_ref().is_some_and(|outcome| {
2857 outcome.is_commit()
2858 && outcome.epoch == current.epoch
2859 && outcome.checkpoint_id == current.checkpoint_id
2860 })
2861 })
2862 .ok_or_else(|| {
2863 LeaseError::Invalid("retained Commit authority chain is broken".into())
2864 })?;
2865 retained.insert(current.sequence);
2866 retained_commit_links.insert(current);
2867 commit_link = commit_record.previous_commit;
2868 }
2869 match (
2870 commit_link,
2871 head.outcome_floor
2872 .as_ref()
2873 .and_then(|floor| floor.committed_anchor_link),
2874 head.outcome_floor
2875 .as_ref()
2876 .and_then(|floor| floor.committed_anchor.as_ref()),
2877 ) {
2878 (Some(link), Some(anchor_link), Some(anchor))
2879 if link == anchor_link
2880 && link.epoch == anchor.epoch
2881 && link.checkpoint_id == anchor.checkpoint_id => {}
2882 (None, None, None) => {}
2883 _ => {
2884 return Err(LeaseError::Invalid(
2885 "Commit chain does not meet the retained artifact-floor anchor".into(),
2886 ));
2887 }
2888 }
2889 Self::validate_retained_commit_boundary(
2890 &head,
2891 expected_commit_head,
2892 &retained_commit_links,
2893 &terminal_commit_links,
2894 artifact_floor,
2895 )?;
2896 let mut assignment_link = head.assignment_decision_head;
2897 let mut assignment_links = 0;
2898 while let Some(current) = assignment_link {
2899 if !consume_live_authority_link(&mut assignment_links) {
2900 return Err(LeaseError::Invalid(format!(
2901 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
2902 )));
2903 }
2904 if head
2905 .assignment_decision_floor
2906 .as_ref()
2907 .is_some_and(|floor| current.target_version < floor.before_target_version)
2908 {
2909 if head
2910 .assignment_decision_floor
2911 .as_ref()
2912 .and_then(|floor| floor.terminal_anchor_link)
2913 != Some(current)
2914 {
2915 return Err(LeaseError::Invalid(
2916 "assignment-decision floor does not anchor the retained authority chain"
2917 .into(),
2918 ));
2919 }
2920 break;
2921 }
2922 let decision_record = read_authority_record(store.as_ref(), current.sequence)
2923 .await?
2924 .filter(|record| {
2925 record.assignment_decision_head == Some(current)
2926 && record
2927 .assignment_decision
2928 .as_ref()
2929 .map(AuthorityAssignmentDecision::target_version)
2930 == Some(current.target_version)
2931 })
2932 .ok_or_else(|| {
2933 LeaseError::Invalid(
2934 "retained assignment-decision authority chain is broken".into(),
2935 )
2936 })?;
2937 retained.insert(current.sequence);
2938 assignment_link = decision_record.previous_assignment_decision;
2939 }
2940 let retained_release = match head.recovery_release_head.as_ref() {
2941 Some(link) => {
2942 let admission = read_authority_record(store.as_ref(), link.sequence)
2943 .await?
2944 .filter(|record| {
2945 record.recovery_release_head.as_ref() == Some(link)
2946 && record
2947 .recovery_release_commit
2948 .as_ref()
2949 .is_some_and(|commit| commit.terminal == link.terminal)
2950 })
2951 .ok_or_else(|| {
2952 LeaseError::Invalid(
2953 "retained recovery release authority link is broken".into(),
2954 )
2955 })?;
2956 if admission.lease.seq != link.sequence {
2957 return Err(LeaseError::Invalid(
2958 "recovery release admission sequence does not match its retained link"
2959 .into(),
2960 ));
2961 }
2962 let terminal = authority
2963 .load_recovery_release_terminal(&link.terminal)
2964 .await
2965 .map_err(|error| LeaseError::Invalid(error.to_string()))?;
2966 if admission
2967 .recovery_release_commit
2968 .as_ref()
2969 .is_none_or(|commit| terminal.round.leader_proof != commit.leader_proof)
2970 {
2971 return Err(LeaseError::Invalid(
2972 "retained recovery release terminal does not match its admission".into(),
2973 ));
2974 }
2975 retained.insert(link.sequence);
2976 Some(link.terminal.clone())
2977 }
2978 None => None,
2979 };
2980
2981 let mut history_exhausted = false;
2982 for _ in 0..LEADER_LEASE_MAX_PRUNE_BATCHES {
2983 let (candidates, exhausted) =
2984 Self::prune_candidates(store, &retained, head_sequence, grace_ms).await?;
2985 if candidates.is_empty() {
2986 history_exhausted = true;
2987 break;
2988 }
2989 let deletions =
2990 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
2991 let mut results = store.delete_stream(Box::pin(deletions));
2992 while let Some(result) = results.next().await {
2993 if let Err(error) = result {
2994 if !matches!(error, object_store::Error::NotFound { .. }) {
2995 return Err(LeaseError::Io(error.to_string()));
2996 }
2997 }
2998 }
2999 if exhausted {
3000 history_exhausted = true;
3001 break;
3002 }
3003 tokio::task::yield_now().await;
3004 }
3005 if !history_exhausted {
3006 return Err(LeaseError::Io(
3007 "leader lease history still exceeds the bounded prune budget".into(),
3008 ));
3009 }
3010 Self::prune_recovery_release_terminals(store, retained_release.as_ref(), grace_ms).await?;
3011 if let Some(floor) = head.assignment_decision_floor.as_ref() {
3012 let before = head.assignment_drain_reservation.as_ref().map_or(
3013 floor.before_target_version,
3014 |reservation| {
3015 floor
3016 .before_target_version
3017 .min(reservation.proposal.version)
3018 },
3019 );
3020 authority.prune_drain_proposals(before).await?;
3021 }
3022 Ok(())
3023 }
3024
3025 async fn prune_recovery_release_terminals(
3026 store: &Arc<dyn ObjectStore>,
3027 retained: Option<&RecoveryReleaseTerminalRef>,
3028 grace_ms: i64,
3029 ) -> Result<(), LeaseError> {
3030 let Some(retained) = retained else {
3031 return Ok(());
3032 };
3033 retained.validate()?;
3034 for _ in 0..RECOVERY_RELEASE_GC_MAX_BATCHES {
3035 let prefix = OsPath::from(RECOVERY_RELEASE_TERMINAL_PREFIX);
3036 let mut listed = store.list(Some(&prefix));
3037 let now = now_millis();
3038 let mut candidates = Vec::with_capacity(RECOVERY_RELEASE_GC_BATCH_RECORDS);
3039 let mut exhausted = true;
3040 while let Some(entry) = listed.next().await {
3041 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3042 let (generation, digest) = recovery_release_terminal_coordinates(&entry.location)?;
3043 let canonical = OsPath::from(format!(
3044 "{RECOVERY_RELEASE_TERMINAL_PREFIX}generation={generation:020}/sha256={digest}.json"
3045 ));
3046 if canonical != entry.location {
3047 return Err(LeaseError::Invalid(format!(
3048 "noncanonical recovery release terminal path {}",
3049 entry.location
3050 )));
3051 }
3052 if entry.location == recovery_release_terminal_path(retained)
3053 || generation > retained.generation()
3054 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
3055 {
3056 continue;
3057 }
3058 candidates.push(entry.location);
3059 if candidates.len() == RECOVERY_RELEASE_GC_BATCH_RECORDS {
3060 exhausted = false;
3061 break;
3062 }
3063 }
3064 if candidates.is_empty() {
3065 return Ok(());
3066 }
3067 let deletions =
3068 futures::stream::iter(candidates.into_iter().map(Ok::<_, object_store::Error>));
3069 let mut results = store.delete_stream(Box::pin(deletions));
3070 while let Some(result) = results.next().await {
3071 if let Err(error) = result {
3072 if !matches!(error, object_store::Error::NotFound { .. }) {
3073 return Err(LeaseError::Io(error.to_string()));
3074 }
3075 }
3076 }
3077 if exhausted {
3078 return Ok(());
3079 }
3080 tokio::task::yield_now().await;
3081 }
3082 Ok(())
3083 }
3084
3085 async fn prune_candidates(
3086 store: &Arc<dyn ObjectStore>,
3087 retained: &BTreeSet<u64>,
3088 snapshot_head_sequence: u64,
3089 grace_ms: i64,
3090 ) -> Result<(Vec<OsPath>, bool), LeaseError> {
3091 let prefix = OsPath::from(LEASE_PREFIX);
3092 let mut listed = store.list(Some(&prefix));
3093 let mut candidates = Vec::with_capacity(LEADER_LEASE_PRUNE_BATCH_RECORDS);
3094 let now = now_millis();
3095 let mut exhausted = true;
3096 while let Some(entry) = listed.next().await {
3097 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3098 let sequence = lease_sequence_from_path(&entry.location)?;
3099 if sequence >= snapshot_head_sequence
3100 || retained.contains(&sequence)
3101 || now.saturating_sub(entry.last_modified.timestamp_millis()) < grace_ms
3102 {
3103 continue;
3104 }
3105 candidates.push(entry.location);
3106 if candidates.len() == LEADER_LEASE_PRUNE_BATCH_RECORDS {
3107 exhausted = false;
3108 break;
3109 }
3110 }
3111 Ok((candidates, exhausted))
3112 }
3113
3114 pub async fn load(&self) -> Result<Option<LeaderLease>, LeaseError> {
3119 Ok(self.load_record().await?.map(|record| record.lease))
3120 }
3121
3122 async fn load_record(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3123 Ok(self
3124 .load_published_authority_head()
3125 .await?
3126 .map(|head| head.record))
3127 }
3128
3129 async fn load_published_authority_head(
3130 &self,
3131 ) -> Result<Option<PublishedAuthorityHead>, LeaseError> {
3132 let pointer = if let Some(pointer) =
3133 read_authority_head_pointer(self.store.as_ref()).await?
3134 {
3135 pointer
3136 } else {
3137 let Some(discovered) = self.discover_authority_head().await? else {
3138 let Some(pointer) = read_authority_head_pointer(self.store.as_ref()).await? else {
3139 return Ok(None);
3140 };
3141 return self.load_authority_head_target(pointer).await.map(Some);
3142 };
3143 let discovered_sequence = discovered.lease.seq;
3144 let published_sequence = self
3145 .publish_authority_head(discovered_sequence, None)
3146 .await?;
3147 let pointer = self
3148 .reload_authority_head_pointer(published_sequence)
3149 .await?;
3150 if published_sequence > discovered_sequence
3151 || pointer.pointer.sequence > published_sequence
3152 {
3153 return self.load_authority_head_target(pointer).await.map(Some);
3154 }
3155 pointer
3156 };
3157
3158 let sequence = pointer.pointer.sequence;
3159 let successor_sequence = sequence.checked_add(1);
3160 let (record, successor) = if let Some(successor_sequence) = successor_sequence {
3161 tokio::try_join!(
3162 read_authority_record(self.store.as_ref(), sequence),
3163 read_authority_record(self.store.as_ref(), successor_sequence)
3164 )?
3165 } else {
3166 (
3167 read_authority_record(self.store.as_ref(), sequence).await?,
3168 None,
3169 )
3170 };
3171 let Some(record) = record else {
3172 let rechecked = read_authority_head_pointer(self.store.as_ref())
3173 .await?
3174 .ok_or_else(|| {
3175 LeaseError::Invalid(
3176 "leader authority head disappeared while reading its target".into(),
3177 )
3178 })?;
3179 if rechecked.pointer.sequence > sequence {
3180 return self
3181 .reload_published_authority_snapshot(rechecked.pointer.sequence)
3182 .await
3183 .map(Some);
3184 }
3185 if rechecked.pointer.sequence < sequence {
3186 return Err(LeaseError::Invalid(format!(
3187 "leader authority head regressed from sequence {sequence} to {}",
3188 rechecked.pointer.sequence
3189 )));
3190 }
3191 return Err(LeaseError::Invalid(format!(
3192 "leader authority head points ahead to missing sequence {sequence}"
3193 )));
3194 };
3195 let Some(successor_sequence) = successor_sequence else {
3196 return Ok(Some(PublishedAuthorityHead { record, pointer }));
3197 };
3198 if successor.is_none() {
3199 return Ok(Some(PublishedAuthorityHead { record, pointer }));
3200 }
3201
3202 if let Some(after_successor) = successor_sequence.checked_add(1) {
3203 if read_authority_record(self.store.as_ref(), after_successor)
3204 .await?
3205 .is_some()
3206 {
3207 let rechecked = read_authority_head_pointer(self.store.as_ref())
3208 .await?
3209 .ok_or_else(|| {
3210 LeaseError::Invalid(
3211 "leader authority head disappeared while checking pointer lag".into(),
3212 )
3213 })?;
3214 if rechecked.pointer.sequence == sequence {
3215 return Err(LeaseError::Invalid(format!(
3216 "leader authority head at sequence {sequence} lags by more than one record"
3217 )));
3218 }
3219 if rechecked.pointer.sequence < sequence {
3220 return Err(LeaseError::Invalid(format!(
3221 "leader authority head regressed from sequence {sequence} to {}",
3222 rechecked.pointer.sequence
3223 )));
3224 }
3225 return self
3226 .reload_published_authority_snapshot(rechecked.pointer.sequence)
3227 .await
3228 .map(Some);
3229 }
3230 }
3231
3232 let published_sequence = self
3233 .publish_authority_head(successor_sequence, Some(&pointer))
3234 .await?;
3235 self.reload_published_authority_snapshot(published_sequence)
3236 .await
3237 .map(Some)
3238 }
3239
3240 async fn reload_published_authority_snapshot(
3241 &self,
3242 minimum_sequence: u64,
3243 ) -> Result<PublishedAuthorityHead, LeaseError> {
3244 let pointer = self.reload_authority_head_pointer(minimum_sequence).await?;
3245 self.load_authority_head_target(pointer).await
3246 }
3247
3248 async fn reload_authority_head_pointer(
3249 &self,
3250 minimum_sequence: u64,
3251 ) -> Result<VersionedAuthorityHeadPointer, LeaseError> {
3252 let pointer = read_authority_head_pointer(self.store.as_ref())
3253 .await?
3254 .ok_or_else(|| {
3255 LeaseError::Invalid(
3256 "leader authority head disappeared while reloading its published snapshot"
3257 .into(),
3258 )
3259 })?;
3260 if pointer.pointer.sequence < minimum_sequence {
3261 return Err(LeaseError::Invalid(format!(
3262 "leader authority head regressed from sequence {minimum_sequence} to {}",
3263 pointer.pointer.sequence
3264 )));
3265 }
3266 Ok(pointer)
3267 }
3268
3269 async fn load_authority_head_target(
3270 &self,
3271 mut pointer: VersionedAuthorityHeadPointer,
3272 ) -> Result<PublishedAuthorityHead, LeaseError> {
3273 for attempt in 0..MAX_LEASE_HEAD_READ_ATTEMPTS {
3274 let sequence = pointer.pointer.sequence;
3275 if let Some(record) = read_authority_record(self.store.as_ref(), sequence).await? {
3276 return Ok(PublishedAuthorityHead { record, pointer });
3277 }
3278
3279 let rechecked = read_authority_head_pointer(self.store.as_ref())
3280 .await?
3281 .ok_or_else(|| {
3282 LeaseError::Invalid(
3283 "leader authority head disappeared while reading its target".into(),
3284 )
3285 })?;
3286 if rechecked.pointer.sequence < sequence {
3287 return Err(LeaseError::Invalid(format!(
3288 "leader authority head regressed from sequence {sequence} to {}",
3289 rechecked.pointer.sequence
3290 )));
3291 }
3292 if rechecked.pointer.sequence == sequence {
3293 return Err(LeaseError::Invalid(format!(
3294 "leader authority head points ahead to missing sequence {sequence}"
3295 )));
3296 }
3297 pointer = rechecked;
3298 if attempt + 1 < MAX_LEASE_HEAD_READ_ATTEMPTS {
3299 tokio::task::yield_now().await;
3300 }
3301 }
3302 Err(LeaseError::Io(format!(
3303 "leader authority head target changed during {MAX_LEASE_HEAD_READ_ATTEMPTS} read attempts"
3304 )))
3305 }
3306
3307 async fn discover_authority_head(&self) -> Result<Option<LeaderAuthorityRecord>, LeaseError> {
3308 let prefix = OsPath::from(LEASE_PREFIX);
3309 let mut listed = self.store.list(Some(&prefix));
3310 let mut discovered = 0usize;
3311 let mut maximum = None;
3312 while let Some(entry) = listed.next().await {
3313 let entry = entry.map_err(|error| LeaseError::Io(error.to_string()))?;
3314 discovered = discovered.checked_add(1).ok_or_else(|| {
3315 LeaseError::Invalid("leader authority discovery count overflowed".into())
3316 })?;
3317 if discovered > MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS {
3318 return Err(LeaseError::Invalid(format!(
3319 "leader authority head discovery exceeds the fixed {MAX_AUTHORITY_HEAD_DISCOVERY_RECORDS}-record bound"
3320 )));
3321 }
3322 let sequence = lease_sequence_from_path(&entry.location)?;
3323 maximum = Some(maximum.map_or(sequence, |current: u64| current.max(sequence)));
3324 }
3325 let Some(sequence) = maximum else {
3326 return Ok(None);
3327 };
3328 read_authority_record(self.store.as_ref(), sequence)
3329 .await?
3330 .map(Some)
3331 .ok_or_else(|| {
3332 LeaseError::Io(format!(
3333 "discovered leader authority sequence {sequence} vanished before publication"
3334 ))
3335 })
3336 }
3337
3338 async fn publish_authority_head(
3339 &self,
3340 sequence: u64,
3341 expected: Option<&VersionedAuthorityHeadPointer>,
3342 ) -> Result<u64, LeaseError> {
3343 if let Some(expected) = expected {
3344 if expected.pointer.sequence.checked_add(1) != Some(sequence) {
3345 return Err(LeaseError::Invalid(
3346 "leader authority head update is not the exact successor".into(),
3347 ));
3348 }
3349 if expected.update_version.e_tag.is_none() && expected.update_version.version.is_none()
3350 {
3351 return Err(LeaseError::Invalid(
3352 "leader authority store did not provide a native conditional update version"
3353 .into(),
3354 ));
3355 }
3356 }
3357
3358 let options = PutOptions {
3359 mode: expected.map_or(PutMode::Create, |expected| {
3360 PutMode::Update(expected.update_version.clone())
3361 }),
3362 ..PutOptions::default()
3363 };
3364 let result = self
3365 .store
3366 .put_opts(
3367 &authority_head_path(),
3368 PutPayload::from(encode_authority_head_pointer(sequence)?),
3369 options,
3370 )
3371 .await;
3372 if result.is_ok() {
3373 return Ok(sequence);
3374 }
3375 let write_error = result
3376 .expect_err("successful authority head writes return above")
3377 .to_string();
3378 let current = read_authority_head_pointer(self.store.as_ref())
3379 .await?
3380 .ok_or_else(|| {
3381 LeaseError::Io(format!(
3382 "leader authority head write failed ({write_error}) and no pointer was durable"
3383 ))
3384 })?;
3385 if let Some(expected) = expected {
3386 if current.pointer.sequence < expected.pointer.sequence {
3387 return Err(LeaseError::Invalid(format!(
3388 "leader authority head regressed from sequence {} to {}",
3389 expected.pointer.sequence, current.pointer.sequence
3390 )));
3391 }
3392 }
3393 if current.pointer.sequence >= sequence {
3394 return Ok(current.pointer.sequence);
3395 }
3396 Err(LeaseError::Io(format!(
3397 "leader authority head write failed ({write_error}) and remained at sequence {}",
3398 current.pointer.sequence
3399 )))
3400 }
3401
3402 async fn read_catalog_manifest_blob(
3403 &self,
3404 reference: &CatalogManifestRef,
3405 ) -> Result<(CatalogManifest, Bytes), CatalogManifestError> {
3406 reference.validate()?;
3407 let path = reference.object_path();
3408 let result = match self.store.get(&path).await {
3409 Ok(result) => result,
3410 Err(object_store::Error::NotFound { .. }) => {
3411 return Err(CatalogManifestError::Invalid(format!(
3412 "catalog manifest blob '{}' is missing",
3413 reference.sha256
3414 )));
3415 }
3416 Err(error) => {
3417 return Err(CatalogManifestError::Authority(LeaseError::Io(
3418 error.to_string(),
3419 )));
3420 }
3421 };
3422 if result.meta.size != reference.encoded_len {
3423 return Err(CatalogManifestError::Invalid(format!(
3424 "catalog manifest blob '{}' is {} bytes, expected {}",
3425 reference.sha256, result.meta.size, reference.encoded_len
3426 )));
3427 }
3428 let bytes = result
3429 .bytes()
3430 .await
3431 .map_err(|error| CatalogManifestError::Authority(LeaseError::Io(error.to_string())))?;
3432 if u64::try_from(bytes.len()).ok() != Some(reference.encoded_len) {
3433 return Err(CatalogManifestError::Invalid(format!(
3434 "catalog manifest blob '{}' payload length changed while reading",
3435 reference.sha256
3436 )));
3437 }
3438 let manifest: CatalogManifest = serde_json::from_slice(&bytes)?;
3439 let (canonical, actual_reference) = manifest.encode_and_reference()?;
3440 if actual_reference != *reference || canonical.as_slice() != bytes.as_ref() {
3441 return Err(CatalogManifestError::Invalid(format!(
3442 "catalog manifest blob '{}' does not match its sealed reference",
3443 reference.sha256
3444 )));
3445 }
3446 Ok((manifest, bytes))
3447 }
3448
3449 pub(super) async fn load_catalog_manifest(
3450 &self,
3451 reference: &CatalogManifestRef,
3452 ) -> Result<CatalogManifest, CatalogManifestError> {
3453 self.read_catalog_manifest_blob(reference)
3454 .await
3455 .map(|(manifest, _)| manifest)
3456 }
3457
3458 async fn ensure_catalog_manifest_blob(
3459 &self,
3460 encoded: &[u8],
3461 reference: &CatalogManifestRef,
3462 ) -> Result<(), CatalogManifestError> {
3463 reference.validate()?;
3464 let options = PutOptions {
3465 mode: PutMode::Create,
3466 ..PutOptions::default()
3467 };
3468 let path = reference.object_path();
3469 let payload = PutPayload::from(Bytes::copy_from_slice(encoded));
3470 let put_error = match self.store.put_opts(&path, payload, options).await {
3471 Ok(_)
3472 | Err(
3473 object_store::Error::AlreadyExists { .. }
3474 | object_store::Error::Precondition { .. },
3475 ) => None,
3476 Err(error) => Some(error),
3477 };
3478
3479 match self.read_catalog_manifest_blob(reference).await {
3480 Ok((_, stored)) if stored.as_ref() == encoded => Ok(()),
3481 Ok(_) => Err(CatalogManifestError::Invalid(format!(
3482 "catalog manifest blob '{}' differs from the proposed content",
3483 reference.sha256
3484 ))),
3485 Err(error) => {
3486 if let Some(put_error) = put_error {
3487 Err(CatalogManifestError::Authority(LeaseError::Io(format!(
3488 "catalog manifest write failed ({put_error}); reconciliation failed ({error})"
3489 ))))
3490 } else {
3491 Err(error)
3492 }
3493 }
3494 }
3495 }
3496
3497 async fn create_authority_record(
3498 &self,
3499 expected: Option<&PublishedAuthorityHead>,
3500 candidate: &LeaderAuthorityRecord,
3501 ) -> Result<AuthorityCreateOutcome, LeaseError> {
3502 let encoded = encode_authority_record(candidate)?;
3503 let expected_pointer = match expected {
3504 None => {
3505 if candidate.lease.seq != 1 {
3506 return Err(LeaseError::Invalid(format!(
3507 "cannot append authority sequence {} to an empty published namespace",
3508 candidate.lease.seq
3509 )));
3510 }
3511 None
3512 }
3513 Some(head) if head.record.lease.seq == candidate.lease.seq => {
3514 return if head.record == *candidate {
3515 Ok(AuthorityCreateOutcome::ExistingIdentical)
3516 } else {
3517 Ok(AuthorityCreateOutcome::Contended(Box::new(
3518 head.record.clone(),
3519 )))
3520 };
3521 }
3522 Some(head) if head.record.lease.seq > candidate.lease.seq => {
3523 return Ok(AuthorityCreateOutcome::Contended(Box::new(
3524 head.record.clone(),
3525 )));
3526 }
3527 Some(head) if head.record.lease.seq.checked_add(1) == Some(candidate.lease.seq) => {
3528 head.record.validate_topology_successor(candidate)?;
3529 head.record
3530 .validate_topology_admission_successor(candidate)?;
3531 Some(&head.pointer)
3532 }
3533 Some(head) => {
3534 return Err(LeaseError::Invalid(format!(
3535 "cannot append authority sequence {} after published sequence {}",
3536 candidate.lease.seq, head.record.lease.seq
3537 )));
3538 }
3539 };
3540 let options = PutOptions {
3541 mode: PutMode::Create,
3542 ..PutOptions::default()
3543 };
3544 let result = self
3545 .store
3546 .put_opts(
3547 &lease_path(candidate.lease.seq),
3548 PutPayload::from(encoded),
3549 options,
3550 )
3551 .await;
3552 let created = result.is_ok();
3553 if let Err(error) = result {
3554 let Some(at_sequence) =
3555 read_authority_record(self.store.as_ref(), candidate.lease.seq).await?
3556 else {
3557 return Err(LeaseError::Io(format!(
3558 "authority record {} write failed ({error}) and no record was durable",
3559 candidate.lease.seq
3560 )));
3561 };
3562 if at_sequence != *candidate {
3563 let winner = self.load_record().await?.ok_or_else(|| {
3564 LeaseError::Io("authority contender was not published or readable".into())
3565 })?;
3566 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3567 }
3568 }
3569
3570 let published_sequence = self
3571 .publish_authority_head(candidate.lease.seq, expected_pointer)
3572 .await?;
3573 if published_sequence > candidate.lease.seq {
3574 if created {
3575 self.schedule_history_prune();
3576 }
3577 let winner = self
3578 .load_record()
3579 .await?
3580 .ok_or_else(|| LeaseError::Io("newer published authority head vanished".into()))?;
3581 return Ok(AuthorityCreateOutcome::Contended(Box::new(winner)));
3582 }
3583 self.schedule_history_prune();
3584 if created {
3585 Ok(AuthorityCreateOutcome::Created)
3586 } else {
3587 Ok(AuthorityCreateOutcome::ExistingIdentical)
3588 }
3589 }
3590
3591 pub(super) async fn seal_catalog(
3600 &self,
3601 proof: &LeaderProof,
3602 manifest: &CatalogManifest,
3603 ) -> Result<CatalogSealOutcome, CatalogManifestError> {
3604 if !proof.is_canonical() {
3605 return Err(CatalogManifestError::Fenced);
3606 }
3607 let (encoded, reference) = manifest.encode_and_reference()?;
3608
3609 let current = self
3610 .load_record()
3611 .await?
3612 .ok_or(CatalogManifestError::Fenced)?;
3613 if let Some(sealed) = ¤t.lease.catalog_manifest {
3614 let durable = self.load_catalog_manifest(sealed).await?;
3615 return if durable == *manifest {
3616 Ok(CatalogSealOutcome::ExistingIdentical)
3617 } else {
3618 Err(CatalogManifestError::Conflict)
3619 };
3620 }
3621 if !current.lease.matches_proof(proof) {
3622 return Err(CatalogManifestError::Fenced);
3623 }
3624 self.ensure_catalog_manifest_blob(&encoded, &reference)
3625 .await?;
3626
3627 loop {
3628 let published = self
3629 .load_published_authority_head()
3630 .await?
3631 .ok_or(CatalogManifestError::Fenced)?;
3632 let current = &published.record;
3633 if let Some(sealed) = ¤t.lease.catalog_manifest {
3634 let durable = self.load_catalog_manifest(sealed).await?;
3635 return if durable == *manifest {
3636 Ok(CatalogSealOutcome::ExistingIdentical)
3637 } else {
3638 Err(CatalogManifestError::Conflict)
3639 };
3640 }
3641 if !current.lease.matches_proof(proof) {
3642 return Err(CatalogManifestError::Fenced);
3643 }
3644
3645 let base_sequence = current.lease.seq;
3646 let candidate_lease = LeaderLease {
3647 seq: current.lease.seq.checked_add(1).ok_or_else(|| {
3648 CatalogManifestError::Authority(LeaseError::Invalid(
3649 "lease sequence exhausted".into(),
3650 ))
3651 })?,
3652 renewal_sequence: current.lease.renewal_sequence,
3653 token: current.lease.token,
3654 owner: current.lease.owner.clone(),
3655 expires_at_ms: current.lease.expires_at_ms,
3656 catalog_manifest: Some(reference.clone()),
3657 };
3658 let candidate = current.preserve_with_lease(candidate_lease);
3659 match self
3660 .create_authority_record(Some(&published), &candidate)
3661 .await?
3662 {
3663 AuthorityCreateOutcome::Created => return Ok(CatalogSealOutcome::Created),
3664 AuthorityCreateOutcome::ExistingIdentical => {
3665 return Ok(CatalogSealOutcome::ExistingIdentical);
3666 }
3667 AuthorityCreateOutcome::Contended(winner) => {
3668 if let Some(sealed) = &winner.lease.catalog_manifest {
3669 let durable = self.load_catalog_manifest(sealed).await?;
3670 return if durable == *manifest {
3671 Ok(CatalogSealOutcome::ExistingIdentical)
3672 } else {
3673 Err(CatalogManifestError::Conflict)
3674 };
3675 }
3676 if !winner.lease.matches_proof(proof) {
3677 return Err(CatalogManifestError::Fenced);
3678 }
3679 if winner.lease.seq <= base_sequence {
3680 return Err(CatalogManifestError::Authority(LeaseError::Invalid(
3681 "catalog seal contention did not advance the authority sequence".into(),
3682 )));
3683 }
3684 tokio::task::yield_now().await;
3685 }
3686 }
3687 }
3688 }
3689
3690 async fn audited_cluster_outcomes_from(
3691 &self,
3692 head: &LeaderAuthorityRecord,
3693 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
3694 let floor = head.outcome_floor.as_ref();
3695 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
3696 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
3697 let mut terminal_records: BTreeMap<u64, LeaderAuthorityRecord> = BTreeMap::new();
3698 let mut terminal_newest_first = Vec::new();
3699 let mut terminal_links = Vec::new();
3700 let mut link = head.outcome_head;
3701 let mut expected_commit_head = head.commit_head;
3702 let mut traversed = 0;
3703 while let Some(current) = link {
3704 if current.epoch < authority_before_epoch {
3705 break;
3706 }
3707 if !consume_live_authority_link(&mut traversed) {
3708 return Err(DecisionError::Conflict(format!(
3709 "live outcome retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3710 ))
3711 .into());
3712 }
3713 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3714 record.clone()
3715 } else {
3716 let record = read_authority_record(self.store.as_ref(), current.sequence)
3717 .await?
3718 .ok_or_else(|| {
3719 DecisionError::InventoryChanged(format!(
3720 "cluster outcome authority record {} disappeared during audit",
3721 current.sequence
3722 ))
3723 })?;
3724 terminal_records.insert(current.sequence, record.clone());
3725 record
3726 };
3727 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3728 DecisionError::Conflict(format!(
3729 "cluster outcome head epoch {} points to non-outcome authority record {}",
3730 current.epoch, current.sequence
3731 ))
3732 })?;
3733 if record.outcome_head != Some(current)
3734 || outcome.epoch != current.epoch
3735 || outcome.checkpoint_id != current.checkpoint_id
3736 {
3737 return Err(DecisionError::Conflict(format!(
3738 "cluster outcome link epoch {} sequence {} does not match its authority record",
3739 current.epoch, current.sequence
3740 ))
3741 .into());
3742 }
3743 if record.commit_head != expected_commit_head {
3744 return Err(DecisionError::Conflict(format!(
3745 "cluster outcome epoch {} did not preserve the exact Commit head",
3746 outcome.epoch
3747 ))
3748 .into());
3749 }
3750 expected_commit_head = if outcome.is_commit() {
3751 record.previous_commit
3752 } else {
3753 record.commit_head
3754 };
3755 terminal_newest_first.push(outcome);
3756 terminal_links.push(current);
3757 link = record.previous_outcome;
3758 }
3759
3760 if let Some(floor) = floor {
3761 match (
3762 link,
3763 floor.terminal_anchor_link,
3764 floor.terminal_anchor.as_ref(),
3765 ) {
3766 (Some(link), Some(anchor_link), Some(anchor))
3767 if link == anchor_link
3768 && link.epoch == anchor.epoch
3769 && link.checkpoint_id == anchor.checkpoint_id => {}
3770 (None, None, None) => {}
3771 _ => {
3772 return Err(DecisionError::Conflict(format!(
3773 "cluster outcome chain does not meet durable floor {} at its terminal anchor",
3774 floor.authority_before_epoch
3775 ))
3776 .into());
3777 }
3778 }
3779 } else if link.is_some() {
3780 return Err(DecisionError::Conflict(
3781 "cluster outcome chain stopped without a durable retention floor".into(),
3782 )
3783 .into());
3784 }
3785
3786 let mut commit_newest_first = Vec::new();
3787 let mut commit_links = Vec::new();
3788 link = head.commit_head;
3789 traversed = 0;
3790 while let Some(current) = link {
3791 if current.epoch < artifact_before_epoch {
3792 break;
3793 }
3794 if !consume_live_authority_link(&mut traversed) {
3795 return Err(DecisionError::Conflict(format!(
3796 "live Commit retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
3797 ))
3798 .into());
3799 }
3800 let record = if let Some(record) = terminal_records.get(¤t.sequence) {
3801 record.clone()
3802 } else {
3803 read_authority_record(self.store.as_ref(), current.sequence)
3804 .await?
3805 .ok_or_else(|| {
3806 DecisionError::InventoryChanged(format!(
3807 "cluster Commit authority record {} disappeared during audit",
3808 current.sequence
3809 ))
3810 })?
3811 };
3812 let outcome = record.checkpoint_outcome.clone().ok_or_else(|| {
3813 DecisionError::Conflict(format!(
3814 "cluster Commit head epoch {} points to non-outcome authority record {}",
3815 current.epoch, current.sequence
3816 ))
3817 })?;
3818 if !outcome.is_commit()
3819 || record.commit_head != Some(current)
3820 || outcome.epoch != current.epoch
3821 || outcome.checkpoint_id != current.checkpoint_id
3822 {
3823 return Err(DecisionError::Conflict(format!(
3824 "cluster Commit link epoch {} sequence {} does not match its authority record",
3825 current.epoch, current.sequence
3826 ))
3827 .into());
3828 }
3829 commit_newest_first.push(outcome);
3830 commit_links.push(current);
3831 link = record.previous_commit;
3832 }
3833
3834 if let Some(floor) = floor {
3835 match (
3836 link,
3837 floor.committed_anchor_link,
3838 floor.committed_anchor.as_ref(),
3839 ) {
3840 (Some(link), Some(anchor_link), Some(anchor))
3841 if link == anchor_link
3842 && link.epoch == anchor.epoch
3843 && link.checkpoint_id == anchor.checkpoint_id => {}
3844 (None, None, None) => {}
3845 _ => {
3846 return Err(DecisionError::Conflict(format!(
3847 "cluster Commit chain does not meet durable artifact floor {} at its committed anchor",
3848 floor.artifact_before_epoch
3849 ))
3850 .into());
3851 }
3852 }
3853 } else if link.is_some() {
3854 return Err(DecisionError::Conflict(
3855 "cluster Commit chain stopped without a durable retention floor".into(),
3856 )
3857 .into());
3858 }
3859
3860 terminal_newest_first.reverse();
3861 terminal_links.reverse();
3862 commit_newest_first.reverse();
3863 commit_links.reverse();
3864 let boundary_commit_head = floor
3865 .and_then(|floor| floor.terminal_anchor.as_ref())
3866 .and_then(|anchor| {
3867 commit_links
3868 .iter()
3869 .rev()
3870 .find(|link| link.epoch <= anchor.epoch)
3871 .copied()
3872 .or_else(|| floor.and_then(|floor| floor.committed_anchor_link))
3873 });
3874 if expected_commit_head != boundary_commit_head {
3875 return Err(DecisionError::Conflict(
3876 "cluster terminal chain does not preserve Commit continuity at its durable floor"
3877 .into(),
3878 )
3879 .into());
3880 }
3881 for (outcome, terminal_link) in terminal_newest_first.iter().zip(&terminal_links) {
3882 if !outcome.is_commit() {
3883 continue;
3884 }
3885 let commit_link = commit_links
3886 .binary_search_by_key(&outcome.epoch, |link| link.epoch)
3887 .ok()
3888 .map(|index| commit_links[index]);
3889 if commit_link != Some(*terminal_link) {
3890 return Err(DecisionError::Conflict(format!(
3891 "cluster Commit epoch {} is not linked identically from both durable outcome heads",
3892 outcome.epoch
3893 ))
3894 .into());
3895 }
3896 }
3897 if let Some((terminal_anchor, terminal_anchor_link)) = floor.and_then(|floor| {
3898 floor
3899 .terminal_anchor
3900 .as_ref()
3901 .zip(floor.terminal_anchor_link)
3902 }) {
3903 if terminal_anchor.is_commit() {
3904 let commit_link = if terminal_anchor.epoch >= artifact_before_epoch {
3905 commit_links
3906 .binary_search_by_key(&terminal_anchor.epoch, |link| link.epoch)
3907 .ok()
3908 .map(|index| commit_links[index])
3909 } else {
3910 floor.and_then(|floor| {
3911 (floor.committed_anchor.as_ref() == Some(terminal_anchor))
3912 .then_some(floor.committed_anchor_link)
3913 .flatten()
3914 })
3915 };
3916 if commit_link != Some(terminal_anchor_link) {
3917 return Err(DecisionError::Conflict(format!(
3918 "cluster terminal Commit anchor epoch {} is not linked identically from the Commit chain",
3919 terminal_anchor.epoch
3920 ))
3921 .into());
3922 }
3923 }
3924 }
3925 let mut outcomes = Vec::with_capacity(
3926 terminal_newest_first
3927 .len()
3928 .saturating_add(commit_newest_first.len())
3929 .saturating_add(2),
3930 );
3931 outcomes.extend(floor.and_then(|floor| floor.committed_anchor.clone()));
3932 outcomes.extend(floor.and_then(|floor| floor.terminal_anchor.clone()));
3933 outcomes.extend(terminal_newest_first);
3934 outcomes.extend(commit_newest_first);
3935 outcomes.sort_unstable_by_key(|outcome| outcome.epoch);
3936 let mut merged: Vec<CheckpointOutcome> = Vec::with_capacity(outcomes.len());
3937 for outcome in outcomes {
3938 if let Some(previous) = merged.last() {
3939 if previous.epoch == outcome.epoch {
3940 if previous != &outcome {
3941 return Err(DecisionError::Conflict(format!(
3942 "cluster outcome epoch {} has conflicting terminal and Commit-chain records",
3943 outcome.epoch
3944 ))
3945 .into());
3946 }
3947 continue;
3948 }
3949 }
3950 merged.push(outcome);
3951 }
3952
3953 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
3954 .load_or_create_deployment_id()
3955 .await?;
3956 for outcome in &merged {
3957 if outcome.deployment_id != expected_deployment {
3958 return Err(DecisionError::Conflict(format!(
3959 "cluster outcome epoch {} belongs to deployment {}, current deployment is {}",
3960 outcome.epoch, outcome.deployment_id, expected_deployment
3961 ))
3962 .into());
3963 }
3964 }
3965 for pair in merged.windows(2) {
3966 let previous = &pair[0];
3967 let current = &pair[1];
3968 if current.epoch <= previous.epoch || current.checkpoint_id <= previous.checkpoint_id {
3969 return Err(DecisionError::Conflict(format!(
3970 "cluster outcomes regress from epoch {} checkpoint {} to epoch {} checkpoint {}",
3971 previous.epoch, previous.checkpoint_id, current.epoch, current.checkpoint_id
3972 ))
3973 .into());
3974 }
3975 }
3976 Ok(ClusterOutcomeAuditSnapshot {
3977 outcomes: Arc::from(merged),
3978 terminal_links: Arc::from(terminal_links),
3979 commit_links: Arc::from(commit_links),
3980 })
3981 }
3982
3983 fn cluster_outcome_audit_key(head: &LeaderAuthorityRecord) -> ClusterOutcomeAuditKey {
3984 ClusterOutcomeAuditKey {
3985 terminal_head: head.outcome_head,
3986 commit_head: head.commit_head,
3987 floor: head.outcome_floor.clone(),
3988 }
3989 }
3990
3991 fn cached_cluster_outcome_audit(
3992 &self,
3993 key: &ClusterOutcomeAuditKey,
3994 authority_sequence: u64,
3995 ) -> Option<ClusterOutcomeAuditSnapshot> {
3996 let mut cache = self.outcome_audit_cache.lock();
3997 let cached = cache.as_mut().filter(|cached| cached.key == *key)?;
3998 cached.authority_sequence = cached.authority_sequence.max(authority_sequence);
3999 Some(cached.snapshot.clone())
4000 }
4001
4002 fn install_cluster_outcome_audit(
4003 &self,
4004 key: ClusterOutcomeAuditKey,
4005 authority_sequence: u64,
4006 snapshot: ClusterOutcomeAuditSnapshot,
4007 ) {
4008 let mut cache = self.outcome_audit_cache.lock();
4009 if cache.as_ref().is_none_or(|cached| {
4010 cached.authority_sequence < authority_sequence
4011 || (cached.authority_sequence == authority_sequence && cached.key == key)
4012 }) {
4013 *cache = Some(ClusterOutcomeAuditCache {
4014 key,
4015 authority_sequence,
4016 snapshot,
4017 });
4018 }
4019 }
4020
4021 fn cluster_outcome_audit_gate(
4022 &self,
4023 key: &ClusterOutcomeAuditKey,
4024 ) -> Arc<tokio::sync::Mutex<()>> {
4025 let mut flights = self.outcome_audit_flights.lock();
4026 flights.retain(|flight| flight.gate.strong_count() != 0);
4027 if let Some(gate) = flights
4028 .iter()
4029 .find(|flight| flight.key == *key)
4030 .and_then(|flight| flight.gate.upgrade())
4031 {
4032 return gate;
4033 }
4034 let gate = Arc::new(tokio::sync::Mutex::new(()));
4035 flights.push(ClusterOutcomeAuditFlight {
4036 key: key.clone(),
4037 gate: Arc::downgrade(&gate),
4038 });
4039 gate
4040 }
4041
4042 async fn cached_audited_cluster_outcomes_from(
4043 &self,
4044 head: &LeaderAuthorityRecord,
4045 ) -> Result<ClusterOutcomeAuditSnapshot, ClusterCheckpointAuthorityError> {
4046 let key = Self::cluster_outcome_audit_key(head);
4047 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
4048 return Ok(snapshot);
4049 }
4050
4051 let gate = self.cluster_outcome_audit_gate(&key);
4052 let _guard = gate.lock().await;
4053 if let Some(snapshot) = self.cached_cluster_outcome_audit(&key, head.lease.seq) {
4054 return Ok(snapshot);
4055 }
4056
4057 let snapshot = self.audited_cluster_outcomes_from(head).await?;
4058 self.install_cluster_outcome_audit(key, head.lease.seq, snapshot.clone());
4059 Ok(snapshot)
4060 }
4061
4062 fn outcomes_retained_by_floor(
4063 floor: &AuthorityOutcomeFloor,
4064 snapshot: &ClusterOutcomeAuditSnapshot,
4065 ) -> ClusterOutcomeAuditSnapshot {
4066 let outcomes = &snapshot.outcomes;
4067 let mut retained = Vec::with_capacity(outcomes.len());
4068 if let Some(anchor) = floor.committed_anchor.as_ref() {
4069 retained.push(anchor.clone());
4070 }
4071 if let Some(anchor) = floor.terminal_anchor.as_ref() {
4072 if retained.last() != Some(anchor) {
4073 retained.push(anchor.clone());
4074 }
4075 }
4076 retained.extend(
4077 outcomes
4078 .iter()
4079 .filter(|outcome| {
4080 outcome.epoch >= floor.authority_before_epoch
4081 || (outcome.is_commit() && outcome.epoch >= floor.artifact_before_epoch)
4082 })
4083 .cloned(),
4084 );
4085 retained.sort_unstable_by_key(|outcome| outcome.epoch);
4086 retained.dedup();
4087 ClusterOutcomeAuditSnapshot {
4088 outcomes: Arc::from(retained),
4089 terminal_links: Arc::from(
4090 snapshot
4091 .terminal_links
4092 .iter()
4093 .filter(|link| link.epoch >= floor.authority_before_epoch)
4094 .copied()
4095 .collect::<Vec<_>>(),
4096 ),
4097 commit_links: Arc::from(
4098 snapshot
4099 .commit_links
4100 .iter()
4101 .filter(|link| link.epoch >= floor.artifact_before_epoch)
4102 .copied()
4103 .collect::<Vec<_>>(),
4104 ),
4105 }
4106 }
4107
4108 async fn build_cluster_outcome_floor(
4109 &self,
4110 current: &LeaderAuthorityRecord,
4111 snapshot: &ClusterOutcomeAuditSnapshot,
4112 artifact_before_epoch: u64,
4113 authority_before_epoch: u64,
4114 ) -> Result<AuthorityOutcomeFloor, ClusterCheckpointAuthorityError> {
4115 let terminal_anchor = snapshot
4116 .outcomes
4117 .iter()
4118 .rev()
4119 .find(|outcome| outcome.epoch < authority_before_epoch)
4120 .cloned();
4121 let terminal_anchor_link = match terminal_anchor.as_ref() {
4122 Some(anchor)
4123 if current
4124 .outcome_floor
4125 .as_ref()
4126 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(anchor)) =>
4127 {
4128 current
4129 .outcome_floor
4130 .as_ref()
4131 .and_then(|floor| floor.terminal_anchor_link)
4132 }
4133 Some(anchor) => {
4134 let index = snapshot
4135 .terminal_links
4136 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4137 .map_err(|_| {
4138 DecisionError::Conflict(format!(
4139 "cluster outcome epoch {} has no audited authority link",
4140 anchor.epoch
4141 ))
4142 })?;
4143 let link = snapshot.terminal_links[index];
4144 if link.checkpoint_id != anchor.checkpoint_id {
4145 return Err(DecisionError::Conflict(format!(
4146 "cluster outcome epoch {} checkpoint {} does not match audited authority link checkpoint {}",
4147 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4148 ))
4149 .into());
4150 }
4151 Some(link)
4152 }
4153 None => None,
4154 };
4155 let committed_anchor = snapshot
4156 .outcomes
4157 .iter()
4158 .rev()
4159 .find(|outcome| outcome.epoch < artifact_before_epoch && outcome.is_commit())
4160 .cloned();
4161 let committed_anchor_link = match committed_anchor.as_ref() {
4162 Some(anchor)
4163 if current
4164 .outcome_floor
4165 .as_ref()
4166 .is_some_and(|floor| floor.committed_anchor.as_ref() == Some(anchor)) =>
4167 {
4168 current
4169 .outcome_floor
4170 .as_ref()
4171 .and_then(|floor| floor.committed_anchor_link)
4172 }
4173 Some(anchor) => {
4174 let index = snapshot
4175 .commit_links
4176 .binary_search_by_key(&anchor.epoch, |link| link.epoch)
4177 .map_err(|_| {
4178 DecisionError::Conflict(format!(
4179 "cluster Commit epoch {} has no audited Commit link",
4180 anchor.epoch
4181 ))
4182 })?;
4183 let link = snapshot.commit_links[index];
4184 if link.checkpoint_id != anchor.checkpoint_id {
4185 return Err(DecisionError::Conflict(format!(
4186 "cluster Commit epoch {} checkpoint {} does not match audited Commit link checkpoint {}",
4187 anchor.epoch, anchor.checkpoint_id, link.checkpoint_id
4188 ))
4189 .into());
4190 }
4191 Some(link)
4192 }
4193 None => None,
4194 };
4195 let floor = AuthorityOutcomeFloor {
4196 deployment_id: CheckpointDecisionStore::new(Arc::clone(&self.store))
4197 .load_or_create_deployment_id()
4198 .await?,
4199 artifact_before_epoch,
4200 authority_before_epoch,
4201 terminal_anchor,
4202 terminal_anchor_link,
4203 committed_anchor,
4204 committed_anchor_link,
4205 };
4206 floor.validate()?;
4207 Ok(floor)
4208 }
4209
4210 async fn compact_cluster_outcome_history_before_append(
4211 &self,
4212 proof: &LeaderProof,
4213 current: &LeaderAuthorityRecord,
4214 snapshot: &ClusterOutcomeAuditSnapshot,
4215 ) -> Result<bool, ClusterCheckpointAuthorityError> {
4216 if snapshot.terminal_links.len() < OUTCOME_HISTORY_COMPACTION_TRIGGER {
4217 return Ok(false);
4218 }
4219 let authority_before_epoch = snapshot.terminal_links
4220 [snapshot.terminal_links.len() - OUTCOME_HISTORY_RETAINED_LINKS]
4221 .epoch;
4222 let artifact_before_epoch = current
4223 .outcome_floor
4224 .as_ref()
4225 .map_or(0, |floor| floor.artifact_before_epoch);
4226 let floor = self
4227 .build_cluster_outcome_floor(
4228 current,
4229 snapshot,
4230 artifact_before_epoch,
4231 authority_before_epoch,
4232 )
4233 .await?;
4234 let published = self
4237 .load_published_authority_head()
4238 .await?
4239 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4240 let rechecked = &published.record;
4241 if !rechecked.lease.matches_proof(proof) {
4242 return Err(ClusterCheckpointAuthorityError::Fenced);
4243 }
4244 if rechecked.outcome_head != current.outcome_head
4245 || rechecked.commit_head != current.commit_head
4246 || rechecked.outcome_floor != current.outcome_floor
4247 {
4248 return Ok(true);
4249 }
4250
4251 let sequence = rechecked
4252 .lease
4253 .seq
4254 .checked_add(1)
4255 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
4256 let mut lease = rechecked.lease.clone();
4257 lease.seq = sequence;
4258 let mut next = rechecked.preserve_with_lease(lease);
4259 next.outcome_floor = Some(floor.clone());
4260 next.validate()?;
4261 match self
4262 .create_authority_record(Some(&published), &next)
4263 .await?
4264 {
4265 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4266 self.install_cluster_outcome_audit(
4267 Self::cluster_outcome_audit_key(&next),
4268 next.lease.seq,
4269 Self::outcomes_retained_by_floor(&floor, snapshot),
4270 );
4271 Ok(true)
4272 }
4273 AuthorityCreateOutcome::Contended(winner) => {
4274 if !winner.lease.matches_proof(proof) {
4275 return Err(ClusterCheckpointAuthorityError::Fenced);
4276 }
4277 if winner.lease.seq <= current.lease.seq {
4278 return Err(LeaseError::Invalid(
4279 "outcome history compaction contention did not advance the authority sequence"
4280 .into(),
4281 )
4282 .into());
4283 }
4284 Ok(true)
4285 }
4286 }
4287 }
4288
4289 async fn audited_cluster_outcomes(
4290 &self,
4291 ) -> Result<
4292 (Option<LeaderAuthorityRecord>, Arc<[CheckpointOutcome]>),
4293 ClusterCheckpointAuthorityError,
4294 > {
4295 const AUDIT_RETRIES: usize = 3;
4296 for attempt in 0..AUDIT_RETRIES {
4297 let Some(head) = self.load_record().await? else {
4298 *self.outcome_audit_cache.lock() = None;
4299 return Ok((None, Arc::from([])));
4300 };
4301 match self.cached_audited_cluster_outcomes_from(&head).await {
4302 Err(ClusterCheckpointAuthorityError::Decision(
4303 DecisionError::InventoryChanged(_),
4304 )) if attempt + 1 < AUDIT_RETRIES => {
4305 tokio::task::yield_now().await;
4306 }
4307 Ok(snapshot) => return Ok((Some(head), snapshot.outcomes)),
4308 Err(error) => return Err(error),
4309 }
4310 }
4311 Err(DecisionError::InventoryChanged(
4312 "cluster outcome audit exhausted stability retries".into(),
4313 )
4314 .into())
4315 }
4316
4317 async fn audited_assignment_decisions_from(
4318 &self,
4319 head: &LeaderAuthorityRecord,
4320 ) -> Result<Vec<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
4321 let mut newest_first = Vec::new();
4322 let floor = head.assignment_decision_floor.as_ref();
4323 let before_target_version = floor.map_or(0, |floor| floor.before_target_version);
4324 let mut stopped_at_anchor = false;
4325 let mut link = head.assignment_decision_head;
4326 let mut traversed = 0;
4327 while let Some(current) = link {
4328 if !consume_live_authority_link(&mut traversed) {
4329 return Err(DecisionError::Conflict(format!(
4330 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound"
4331 ))
4332 .into());
4333 }
4334 if current.target_version < before_target_version {
4335 if floor.and_then(|floor| floor.terminal_anchor_link) != Some(current) {
4336 return Err(DecisionError::Conflict(format!(
4337 "assignment decision chain does not meet durable floor {before_target_version} at its terminal anchor"
4338 ))
4339 .into());
4340 }
4341 stopped_at_anchor = true;
4342 break;
4343 }
4344 let record = read_authority_record(self.store.as_ref(), current.sequence)
4345 .await?
4346 .ok_or_else(|| {
4347 DecisionError::InventoryChanged(format!(
4348 "assignment decision authority record {} disappeared during audit",
4349 current.sequence
4350 ))
4351 })?;
4352 let decision = record.assignment_decision.clone().ok_or_else(|| {
4353 DecisionError::Conflict(format!(
4354 "assignment decision head version {} points to non-decision authority record {}",
4355 current.target_version, current.sequence
4356 ))
4357 })?;
4358 if record.assignment_decision_head != Some(current)
4359 || decision.target_version() != current.target_version
4360 {
4361 return Err(DecisionError::Conflict(format!(
4362 "assignment decision link version {} sequence {} does not match its authority record",
4363 current.target_version, current.sequence
4364 ))
4365 .into());
4366 }
4367 newest_first.push(decision);
4368 link = record.previous_assignment_decision;
4369 }
4370 if floor.and_then(|floor| floor.terminal_anchor_link).is_some() && !stopped_at_anchor {
4371 return Err(DecisionError::Conflict(
4372 "assignment decision chain stopped without its durable retention anchor".into(),
4373 )
4374 .into());
4375 }
4376 newest_first.reverse();
4377 if let Some(pair) = newest_first
4378 .windows(2)
4379 .find(|pair| pair[0].target_version() >= pair[1].target_version())
4380 {
4381 return Err(DecisionError::Conflict(format!(
4382 "assignment decisions regress from version {} to {}",
4383 pair[0].target_version(),
4384 pair[1].target_version()
4385 ))
4386 .into());
4387 }
4388 Ok(newest_first)
4389 }
4390
4391 async fn exact_assignment_decision_link(
4392 &self,
4393 head: &LeaderAuthorityRecord,
4394 decision: &AuthorityAssignmentDecision,
4395 ) -> Result<AssignmentDecisionLink, ClusterCheckpointAuthorityError> {
4396 if let Some(floor) = head.assignment_decision_floor.as_ref() {
4397 if floor.terminal_anchor.as_ref() == Some(decision) {
4398 return floor.terminal_anchor_link.ok_or_else(|| {
4399 DecisionError::Conflict(
4400 "assignment decision floor lost its terminal authority link".into(),
4401 )
4402 .into()
4403 });
4404 }
4405 }
4406 let before_target_version = head
4407 .assignment_decision_floor
4408 .as_ref()
4409 .map_or(0, |floor| floor.before_target_version);
4410 let mut link = head.assignment_decision_head;
4411 let mut traversed = 0;
4412 while let Some(current) = link {
4413 if !consume_live_authority_link(&mut traversed) {
4414 return Err(DecisionError::Conflict(format!(
4415 "live assignment-decision retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup"
4416 ))
4417 .into());
4418 }
4419 if current.target_version < before_target_version {
4420 break;
4421 }
4422 let record = read_authority_record(self.store.as_ref(), current.sequence)
4423 .await?
4424 .ok_or_else(|| {
4425 DecisionError::InventoryChanged(format!(
4426 "assignment decision authority record {} disappeared during exact lookup",
4427 current.sequence
4428 ))
4429 })?;
4430 if record.assignment_decision.as_ref() == Some(decision) {
4431 return Ok(current);
4432 }
4433 link = record.previous_assignment_decision;
4434 }
4435 Err(DecisionError::Conflict(format!(
4436 "assignment decision version {} is not linked from the durable authority head",
4437 decision.target_version()
4438 ))
4439 .into())
4440 }
4441
4442 async fn reject_consumed_checkpoint_assignment(
4443 &self,
4444 current: &LeaderAuthorityRecord,
4445 assignment_fence: &CheckpointAssignmentFence,
4446 ) -> Result<(), ClusterCheckpointAuthorityError> {
4447 let decisions = self.audited_assignment_decisions_from(current).await?;
4448 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4449 if assignment_fence.assignment_version < floor.before_target_version {
4453 return Err(DecisionError::Conflict(format!(
4454 "checkpoint assignment version {} is below durable assignment-decision floor {}",
4455 assignment_fence.assignment_version, floor.before_target_version
4456 ))
4457 .into());
4458 }
4459 }
4460 let Some(newest) = decisions.last() else {
4461 return Ok(());
4462 };
4463 if newest.target_version() > assignment_fence.assignment_version {
4464 return Err(DecisionError::Conflict(format!(
4465 "checkpoint assignment version {} was consumed by assignment decision version {}",
4466 assignment_fence.assignment_version,
4467 newest.target_version()
4468 ))
4469 .into());
4470 }
4471 if newest.target_version() == assignment_fence.assignment_version
4472 && newest.materialized_target() != *assignment_fence
4473 {
4474 return Err(DecisionError::Conflict(format!(
4475 "checkpoint assignment version {} does not match its materialized assignment decision",
4476 assignment_fence.assignment_version
4477 ))
4478 .into());
4479 }
4480 Ok(())
4481 }
4482
4483 pub async fn begin_cluster_checkpoint_artifacts(
4492 &self,
4493 proof: &LeaderProof,
4494 inventory: CheckpointArtifactInventory,
4495 ) -> Result<CheckpointArtifactInventory, ClusterCheckpointAuthorityError> {
4496 if !proof.is_canonical() {
4497 return Err(ClusterCheckpointAuthorityError::Fenced);
4498 }
4499 inventory.validate().map_err(DecisionError::Conflict)?;
4500 let assignment_fence = inventory.assignment_fence.as_ref().ok_or_else(|| {
4501 DecisionError::Conflict(
4502 "cluster checkpoint artifact inventory requires an assignment fence".into(),
4503 )
4504 })?;
4505 if assignment_fence.participant_incarnation(proof.owner.node_id)
4506 != Some(proof.owner.boot_id)
4507 {
4508 return Err(DecisionError::Conflict(
4509 "checkpoint artifact admitting leader is outside the assignment fence".into(),
4510 )
4511 .into());
4512 }
4513 let initial = self
4514 .load_record()
4515 .await?
4516 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4517 if !initial.lease.matches_proof(proof) {
4518 return Err(ClusterCheckpointAuthorityError::Fenced);
4519 }
4520 let deployment_id = CheckpointDecisionStore::new(Arc::clone(&self.store))
4521 .load_or_create_deployment_id()
4522 .await?;
4523 if inventory.deployment_id != deployment_id {
4524 return Err(DecisionError::Conflict(format!(
4525 "checkpoint artifact inventory deployment {} does not match authority deployment {deployment_id}",
4526 inventory.deployment_id
4527 ))
4528 .into());
4529 }
4530
4531 loop {
4532 let published = self
4533 .load_published_authority_head()
4534 .await?
4535 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4536 let current = &published.record;
4537 if !current.lease.matches_proof(proof) {
4538 return Err(ClusterCheckpointAuthorityError::Fenced);
4539 }
4540 self.reject_consumed_checkpoint_assignment(current, assignment_fence)
4541 .await?;
4542 current.validate_topology_checkpoint_inventory(&inventory)?;
4543 self.validate_committed_topology_checkpoint_inventory(current, &inventory)
4544 .await?;
4545 if let Some(active) = current.active_checkpoint_artifacts.as_ref() {
4546 if active == &inventory
4547 && current.active_checkpoint_artifact_leader_proof.as_ref() == Some(proof)
4548 && current.outcome_head.is_none_or(|head| {
4549 head.epoch != inventory.attempt.epoch
4550 || head.checkpoint_id != inventory.attempt.checkpoint_id
4551 })
4552 {
4553 return Ok(active.clone());
4554 }
4555 return Err(DecisionError::Conflict(format!(
4556 "checkpoint {} still has unresolved or inherited cluster artifacts",
4557 active.attempt.checkpoint_id
4558 ))
4559 .into());
4560 }
4561 if let Some(head) = current.outcome_head {
4562 let terminal =
4563 crate::checkpoint::CheckpointAttempt::new(head.epoch, head.checkpoint_id);
4564 if inventory.attempt.relation_to(terminal)
4565 != crate::checkpoint::CheckpointAttemptRelation::Newer
4566 {
4567 return Err(DecisionError::Conflict(format!(
4568 "checkpoint {} does not advance terminal checkpoint {}",
4569 inventory.attempt.checkpoint_id, head.checkpoint_id
4570 ))
4571 .into());
4572 }
4573 }
4574
4575 let sequence =
4576 current.lease.seq.checked_add(1).ok_or_else(|| {
4577 LeaseError::Invalid("leader authority sequence exhausted".into())
4578 })?;
4579 let mut lease = current.lease.clone();
4580 lease.seq = sequence;
4581 let mut next = current.preserve_with_lease(lease);
4582 next.active_checkpoint_artifacts = Some(inventory.clone());
4583 next.active_checkpoint_artifact_leader_proof = Some(proof.clone());
4584 next.validate()?;
4585 match self
4586 .create_authority_record(Some(&published), &next)
4587 .await?
4588 {
4589 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4590 return Ok(inventory);
4591 }
4592 AuthorityCreateOutcome::Contended(winner) => {
4593 if !winner.lease.matches_proof(proof) {
4594 return Err(ClusterCheckpointAuthorityError::Fenced);
4595 }
4596 tokio::task::yield_now().await;
4597 }
4598 }
4599 }
4600 }
4601
4602 pub async fn finish_cluster_checkpoint_artifact_cleanup(
4608 &self,
4609 proof: &LeaderProof,
4610 expected: &CheckpointArtifactInventory,
4611 ) -> Result<(), ClusterCheckpointAuthorityError> {
4612 if !proof.is_canonical() {
4613 return Err(ClusterCheckpointAuthorityError::Fenced);
4614 }
4615 expected.validate().map_err(DecisionError::Conflict)?;
4616 if expected.assignment_fence.is_none() {
4617 return Err(DecisionError::Conflict(
4618 "cluster checkpoint artifact inventory requires an assignment fence".into(),
4619 )
4620 .into());
4621 }
4622
4623 loop {
4624 let published = self
4625 .load_published_authority_head()
4626 .await?
4627 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4628 let current = &published.record;
4629 if !current.lease.matches_proof(proof) {
4630 return Err(ClusterCheckpointAuthorityError::Fenced);
4631 }
4632 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
4633 let matching_abort = snapshot.outcomes.iter().any(|outcome| {
4634 matches!(outcome.verdict, CheckpointVerdict::Abort)
4635 && outcome.deployment_id == expected.deployment_id
4636 && outcome.epoch == expected.attempt.epoch
4637 && outcome.checkpoint_id == expected.attempt.checkpoint_id
4638 && outcome.assignment_fence.as_ref() == expected.assignment_fence.as_ref()
4639 });
4640 match current.active_checkpoint_artifacts.as_ref() {
4641 None if matching_abort => return Ok(()),
4642 None => {
4643 return Err(DecisionError::Conflict(format!(
4644 "checkpoint {} has no retained cluster artifact inventory",
4645 expected.attempt.checkpoint_id
4646 ))
4647 .into());
4648 }
4649 Some(active) if active != expected => {
4650 return Err(DecisionError::Conflict(format!(
4651 "checkpoint {} still has unresolved cluster artifacts",
4652 active.attempt.checkpoint_id
4653 ))
4654 .into());
4655 }
4656 Some(_) if !matching_abort => {
4657 return Err(DecisionError::Conflict(format!(
4658 "checkpoint {} has no matching durable Abort",
4659 expected.attempt.checkpoint_id
4660 ))
4661 .into());
4662 }
4663 Some(_) => {}
4664 }
4665
4666 let sequence =
4667 current.lease.seq.checked_add(1).ok_or_else(|| {
4668 LeaseError::Invalid("leader authority sequence exhausted".into())
4669 })?;
4670 let mut lease = current.lease.clone();
4671 lease.seq = sequence;
4672 let mut next = current.preserve_with_lease(lease);
4673 next.active_checkpoint_artifacts = None;
4674 next.active_checkpoint_artifact_leader_proof = None;
4675 next.validate()?;
4676 match self
4677 .create_authority_record(Some(&published), &next)
4678 .await?
4679 {
4680 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
4681 return Ok(());
4682 }
4683 AuthorityCreateOutcome::Contended(winner) => {
4684 if !winner.lease.matches_proof(proof) {
4685 return Err(ClusterCheckpointAuthorityError::Fenced);
4686 }
4687 tokio::task::yield_now().await;
4688 }
4689 }
4690 }
4691 }
4692
4693 pub async fn record_cluster_outcome(
4702 &self,
4703 proof: &LeaderProof,
4704 epoch: u64,
4705 checkpoint_id: u64,
4706 assignment_fence: CheckpointAssignmentFence,
4707 verdict: CheckpointVerdict,
4708 committed_checkpoint: Option<CommittedCheckpointRef>,
4709 ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4710 self.record_cluster_outcome_inner(
4711 proof,
4712 epoch,
4713 checkpoint_id,
4714 assignment_fence,
4715 verdict,
4716 committed_checkpoint,
4717 false,
4718 )
4719 .await
4720 }
4721
4722 pub async fn abort_unadmitted_cluster_checkpoint(
4729 &self,
4730 proof: &LeaderProof,
4731 attempt: crate::checkpoint::CheckpointAttempt,
4732 assignment_fence: CheckpointAssignmentFence,
4733 ) -> Result<RecordOutcomeResult, ClusterCheckpointAuthorityError> {
4734 self.record_cluster_outcome_inner(
4735 proof,
4736 attempt.epoch,
4737 attempt.checkpoint_id,
4738 assignment_fence,
4739 CheckpointVerdict::Abort,
4740 None,
4741 true,
4742 )
4743 .await
4744 }
4745
4746 async fn validate_assignment_recovery_snapshot(
4747 &self,
4748 decision: &AssignmentRecoveryDecision,
4749 ) -> Result<(), ClusterCheckpointAuthorityError> {
4750 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
4751 let predecessor = snapshots
4752 .load()
4753 .await
4754 .map_err(|error| {
4755 assignment_snapshot_error("load assignment recovery predecessor", error)
4756 })?
4757 .ok_or_else(|| {
4758 LeaseError::Invalid(
4759 "assignment recovery requires a durable predecessor snapshot".into(),
4760 )
4761 })?;
4762 let predecessor_fence = predecessor.assignment_fence().map_err(|error| {
4763 LeaseError::Invalid(format!(
4764 "assignment recovery predecessor has no valid fence: {error}"
4765 ))
4766 })?;
4767 if predecessor.draining || predecessor_fence != decision.predecessor {
4768 return Err(LeaseError::Invalid(
4769 "assignment recovery predecessor is not the exact committed assignment head".into(),
4770 )
4771 .into());
4772 }
4773 let proposal = snapshots
4774 .load_recovery_proposal(&decision.proposal)
4775 .await
4776 .map_err(|error| {
4777 assignment_snapshot_error("load assignment recovery proposal", error)
4778 })?;
4779 let proposal_fence = proposal.assignment_fence().map_err(|error| {
4780 LeaseError::Invalid(format!(
4781 "assignment recovery proposal has no valid fence: {error}"
4782 ))
4783 })?;
4784 if proposal_fence != decision.target {
4785 return Err(LeaseError::Invalid(
4786 "assignment recovery proposal does not match its authorized target fence".into(),
4787 )
4788 .into());
4789 }
4790 Ok(())
4791 }
4792
4793 async fn current_assignment_checkpoint(
4794 &self,
4795 current: &LeaderAuthorityRecord,
4796 predecessor: &CheckpointAssignmentFence,
4797 purpose: &str,
4798 ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
4799 let Some(commit_head) = current.commit_head else {
4800 return Ok(None);
4801 };
4802 let outcome = self
4803 .cluster_outcome_from_snapshot(current, commit_head.epoch)
4804 .await?
4805 .ok_or_else(|| {
4806 DecisionError::Conflict(format!(
4807 "{purpose} checkpoint {} is not live",
4808 commit_head.checkpoint_id
4809 ))
4810 })?;
4811 if !outcome.is_commit()
4812 || outcome.checkpoint_id != commit_head.checkpoint_id
4813 || outcome.assignment_fence.as_ref() != Some(predecessor)
4814 {
4815 return Err(DecisionError::Conflict(format!(
4816 "{purpose} checkpoint {} does not bind its predecessor fence",
4817 commit_head.checkpoint_id
4818 ))
4819 .into());
4820 }
4821 let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
4822 DecisionError::Conflict(format!(
4823 "{purpose} checkpoint {} has no committed index reference",
4824 commit_head.checkpoint_id
4825 ))
4826 })?;
4827 let index = CheckpointDecisionStore::new(Arc::clone(&self.store))
4828 .validate_committed_checkpoint_for_outcome(&outcome)
4829 .await?;
4830 if index.assignment_fence.as_ref() != Some(predecessor) {
4831 return Err(DecisionError::Conflict(format!(
4832 "{purpose} checkpoint index {} does not bind its predecessor fence",
4833 commit_head.checkpoint_id
4834 ))
4835 .into());
4836 }
4837 Ok(Some(reference.clone()))
4838 }
4839
4840 async fn validate_current_assignment_checkpoint(
4841 &self,
4842 current: &LeaderAuthorityRecord,
4843 reference: &CommittedCheckpointRef,
4844 predecessor: &CheckpointAssignmentFence,
4845 purpose: &str,
4846 ) -> Result<(), ClusterCheckpointAuthorityError> {
4847 let current_reference = self
4848 .current_assignment_checkpoint(current, predecessor, purpose)
4849 .await?
4850 .ok_or_else(|| {
4851 DecisionError::Conflict(format!("{purpose} has no authoritative checkpoint Commit"))
4852 })?;
4853 if current_reference != *reference {
4854 return Err(DecisionError::Conflict(format!(
4855 "{purpose} checkpoint {} is not the current Commit head {}",
4856 reference.checkpoint_id, current_reference.checkpoint_id
4857 ))
4858 .into());
4859 }
4860 Ok(())
4861 }
4862
4863 fn aborted_handoff_target(
4864 transition: &AssignmentDrainTransition,
4865 ) -> Result<CheckpointAssignmentFence, ClusterCheckpointAuthorityError> {
4866 let mut target = transition.predecessor.clone();
4867 target.assignment_version = transition.target.assignment_version;
4868 if !target.is_canonical() {
4869 return Err(LeaseError::Invalid(
4870 "assignment drain abort produced an invalid handoff target".into(),
4871 )
4872 .into());
4873 }
4874 Ok(target)
4875 }
4876
4877 async fn next_assignment_handoff_pin(
4878 &self,
4879 current: &LeaderAuthorityRecord,
4880 decision: &AuthorityAssignmentDecision,
4881 ) -> Result<Option<AssignmentHandoffPin>, ClusterCheckpointAuthorityError> {
4882 match decision {
4883 AuthorityAssignmentDecision::Drain(drain) => match drain.verdict {
4884 AssignmentDrainVerdict::Commit => {
4885 if current.assignment_handoff_pin.is_some() {
4886 return Err(DecisionError::Conflict(
4887 "assignment decision cannot overtake an unresolved state handoff"
4888 .into(),
4889 )
4890 .into());
4891 }
4892 let reference = drain.handoff_checkpoint.as_ref().ok_or_else(|| {
4893 LeaseError::Invalid(
4894 "committed assignment drain has no handoff checkpoint".into(),
4895 )
4896 })?;
4897 self.validate_current_assignment_checkpoint(
4898 current,
4899 reference,
4900 &drain.transition.predecessor,
4901 "assignment drain handoff",
4902 )
4903 .await?;
4904 Ok(Some(AssignmentHandoffPin {
4905 target: drain.transition.target.clone(),
4906 checkpoint: reference.clone(),
4907 }))
4908 }
4909 AssignmentDrainVerdict::Abort => match current.assignment_handoff_pin.as_ref() {
4910 None => {
4911 let checkpoint = self
4912 .current_assignment_checkpoint(
4913 current,
4914 &drain.transition.predecessor,
4915 "assignment drain abort",
4916 )
4917 .await?;
4918 match checkpoint {
4919 Some(checkpoint) => Ok(Some(AssignmentHandoffPin {
4920 target: Self::aborted_handoff_target(&drain.transition)?,
4921 checkpoint,
4922 })),
4923 None => Ok(None),
4924 }
4925 }
4926 Some(pin) if pin.target == drain.transition.predecessor => {
4927 Ok(Some(AssignmentHandoffPin {
4928 target: Self::aborted_handoff_target(&drain.transition)?,
4929 checkpoint: pin.checkpoint.clone(),
4930 }))
4931 }
4932 Some(_) => Err(DecisionError::Conflict(
4933 "assignment decision cannot overtake an unresolved state handoff".into(),
4934 )
4935 .into()),
4936 },
4937 },
4938 AuthorityAssignmentDecision::Recovery(recovery) => {
4939 if let Some(pin) = current.assignment_handoff_pin.as_ref() {
4940 if pin.target != recovery.predecessor
4941 || pin.checkpoint != recovery.recovery_checkpoint
4942 {
4943 return Err(DecisionError::Conflict(
4944 "assignment recovery does not continue the exact unresolved state handoff"
4945 .into(),
4946 )
4947 .into());
4948 }
4949 } else {
4950 self.validate_current_assignment_checkpoint(
4951 current,
4952 &recovery.recovery_checkpoint,
4953 &recovery.predecessor,
4954 "assignment recovery",
4955 )
4956 .await?;
4957 }
4958 Ok(Some(AssignmentHandoffPin {
4959 target: recovery.target.clone(),
4960 checkpoint: recovery.recovery_checkpoint.clone(),
4961 }))
4962 }
4963 }
4964 }
4965
4966 async fn record_assignment_decision(
4967 &self,
4968 proof: &LeaderProof,
4969 decision: AuthorityAssignmentDecision,
4970 assignments: Option<&AssignmentSnapshotStore>,
4971 ) -> Result<RecordAuthorityAssignmentDecisionResult, ClusterCheckpointAuthorityError> {
4972 decision.validate()?;
4973 if decision.leader_proof() != proof || !proof.is_canonical() {
4974 return Err(ClusterCheckpointAuthorityError::Fenced);
4975 }
4976
4977 loop {
4978 let published = self
4979 .load_published_authority_head()
4980 .await?
4981 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
4982 let current = &published.record;
4983 if !current.lease.matches_proof(proof) {
4984 return Err(ClusterCheckpointAuthorityError::Fenced);
4985 }
4986 if let Some(floor) = current.assignment_decision_floor.as_ref() {
4987 if decision.target_version() < floor.before_target_version {
4988 return Err(DecisionError::Conflict(format!(
4989 "assignment decision version {} is below durable retention floor {}",
4990 decision.target_version(),
4991 floor.before_target_version
4992 ))
4993 .into());
4994 }
4995 }
4996 let decisions = self.audited_assignment_decisions_from(current).await?;
4997 if let Some(winner) = decisions
4998 .iter()
4999 .find(|winner| winner.target_version() == decision.target_version())
5000 {
5001 return if winner == &decision {
5002 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5003 winner.clone(),
5004 ))
5005 } else {
5006 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5007 winner: winner.clone(),
5008 })
5009 };
5010 }
5011 self.validate_assignment_topology_settlement(current, &decision, assignments)
5012 .await?;
5013 current.reject_assignment_checkpoint_overtake(&decision)?;
5014 if let Some(last) = decisions.last() {
5015 if decision.target_version() <= last.target_version() {
5016 return Err(DecisionError::Conflict(format!(
5017 "assignment decision version {} does not advance durable version {}",
5018 decision.target_version(),
5019 last.target_version()
5020 ))
5021 .into());
5022 }
5023 }
5024 if let AuthorityAssignmentDecision::Recovery(recovery) = &decision {
5025 self.validate_assignment_recovery_snapshot(recovery).await?;
5026 }
5027 let assignment_handoff_pin =
5028 self.next_assignment_handoff_pin(current, &decision).await?;
5029
5030 let base_sequence = current.lease.seq;
5031 let sequence = base_sequence
5032 .checked_add(1)
5033 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5034 let mut candidate = current.preserve_with_lease(LeaderLease {
5035 seq: sequence,
5036 renewal_sequence: current.lease.renewal_sequence,
5037 token: current.lease.token,
5038 owner: current.lease.owner.clone(),
5039 expires_at_ms: current.lease.expires_at_ms,
5040 catalog_manifest: current.lease.catalog_manifest.clone(),
5041 });
5042 candidate.assignment_decision = Some(decision.clone());
5043 candidate.previous_assignment_decision = current.assignment_decision_head;
5044 candidate.assignment_decision_head = Some(AssignmentDecisionLink {
5045 sequence,
5046 target_version: decision.target_version(),
5047 });
5048 candidate.assignment_handoff_pin = assignment_handoff_pin;
5049 candidate.assignment_drain_reservation = None;
5050 candidate.validate()?;
5051
5052 match self
5053 .create_authority_record(Some(&published), &candidate)
5054 .await?
5055 {
5056 AuthorityCreateOutcome::Created => {
5057 return Ok(RecordAuthorityAssignmentDecisionResult::Created(decision));
5058 }
5059 AuthorityCreateOutcome::ExistingIdentical => {
5060 return Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(decision));
5061 }
5062 AuthorityCreateOutcome::Contended(winner_head) => {
5063 let winners = self.audited_assignment_decisions_from(&winner_head).await?;
5064 if let Some(winner) = winners
5065 .iter()
5066 .find(|winner| winner.target_version() == decision.target_version())
5067 {
5068 return if winner == &decision {
5069 Ok(RecordAuthorityAssignmentDecisionResult::Unchanged(
5070 winner.clone(),
5071 ))
5072 } else {
5073 Ok(RecordAuthorityAssignmentDecisionResult::Conflict {
5074 winner: winner.clone(),
5075 })
5076 };
5077 }
5078 if !winner_head.lease.matches_proof(proof) {
5079 return Err(ClusterCheckpointAuthorityError::Fenced);
5080 }
5081 if winner_head.lease.seq > base_sequence {
5082 tokio::task::yield_now().await;
5083 continue;
5084 }
5085 return Err(LeaseError::Invalid(
5086 "assignment authority contention did not advance the sequence".into(),
5087 )
5088 .into());
5089 }
5090 }
5091 }
5092 }
5093
5094 pub async fn record_assignment_drain_decision(
5102 &self,
5103 proof: &LeaderProof,
5104 decision: AssignmentDrainDecision,
5105 ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5106 let assignments = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5107 self.record_assignment_drain_decision_with_assignments(proof, decision, &assignments)
5108 .await
5109 }
5110
5111 pub async fn record_assignment_drain_decision_with_assignments(
5117 &self,
5118 proof: &LeaderProof,
5119 decision: AssignmentDrainDecision,
5120 assignments: &AssignmentSnapshotStore,
5121 ) -> Result<RecordAssignmentDrainDecisionResult, ClusterCheckpointAuthorityError> {
5122 match Box::pin(self.record_assignment_decision(
5123 proof,
5124 AuthorityAssignmentDecision::Drain(decision),
5125 Some(assignments),
5126 ))
5127 .await?
5128 {
5129 RecordAuthorityAssignmentDecisionResult::Created(
5130 AuthorityAssignmentDecision::Drain(decision),
5131 ) => Ok(RecordAssignmentDrainDecisionResult::Created(decision)),
5132 RecordAuthorityAssignmentDecisionResult::Unchanged(
5133 AuthorityAssignmentDecision::Drain(decision),
5134 ) => Ok(RecordAssignmentDrainDecisionResult::Unchanged(decision)),
5135 RecordAuthorityAssignmentDecisionResult::Conflict {
5136 winner: AuthorityAssignmentDecision::Drain(winner),
5137 } => Ok(RecordAssignmentDrainDecisionResult::Conflict { winner }),
5138 RecordAuthorityAssignmentDecisionResult::Conflict {
5139 winner: AuthorityAssignmentDecision::Recovery(winner),
5140 } => Err(DecisionError::Conflict(format!(
5141 "assignment recovery decision already settled target version {}",
5142 winner.target_version()
5143 ))
5144 .into()),
5145 RecordAuthorityAssignmentDecisionResult::Created(_)
5146 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5147 "assignment decision recorder returned the wrong decision kind".into(),
5148 )
5149 .into()),
5150 }
5151 }
5152
5153 pub(crate) async fn record_assignment_recovery_decision(
5158 &self,
5159 proof: &LeaderProof,
5160 decision: AssignmentRecoveryDecision,
5161 ) -> Result<RecordAssignmentRecoveryDecisionResult, ClusterCheckpointAuthorityError> {
5162 match Box::pin(self.record_assignment_decision(
5163 proof,
5164 AuthorityAssignmentDecision::Recovery(decision),
5165 None,
5166 ))
5167 .await?
5168 {
5169 RecordAuthorityAssignmentDecisionResult::Created(
5170 AuthorityAssignmentDecision::Recovery(decision),
5171 ) => Ok(RecordAssignmentRecoveryDecisionResult::Created(decision)),
5172 RecordAuthorityAssignmentDecisionResult::Unchanged(
5173 AuthorityAssignmentDecision::Recovery(decision),
5174 ) => Ok(RecordAssignmentRecoveryDecisionResult::Unchanged(decision)),
5175 RecordAuthorityAssignmentDecisionResult::Conflict {
5176 winner: AuthorityAssignmentDecision::Recovery(winner),
5177 } => Ok(RecordAssignmentRecoveryDecisionResult::Conflict { winner }),
5178 RecordAuthorityAssignmentDecisionResult::Conflict {
5179 winner: AuthorityAssignmentDecision::Drain(winner),
5180 } => Err(DecisionError::Conflict(format!(
5181 "assignment drain decision already settled target version {}",
5182 winner.target_version()
5183 ))
5184 .into()),
5185 RecordAuthorityAssignmentDecisionResult::Created(_)
5186 | RecordAuthorityAssignmentDecisionResult::Unchanged(_) => Err(LeaseError::Invalid(
5187 "assignment decision recorder returned the wrong decision kind".into(),
5188 )
5189 .into()),
5190 }
5191 }
5192
5193 async fn assignment_decision(
5194 &self,
5195 target_version: u64,
5196 ) -> Result<Option<AuthorityAssignmentDecision>, ClusterCheckpointAuthorityError> {
5197 if target_version == 0 {
5198 return Err(LeaseError::Invalid(
5199 "assignment decision target version must be nonzero".into(),
5200 )
5201 .into());
5202 }
5203 let Some(head) = self.load_record().await? else {
5204 return Ok(None);
5205 };
5206 if let Some(floor) = head.assignment_decision_floor.as_ref() {
5207 if target_version < floor.before_target_version {
5208 return Err(DecisionError::Conflict(format!(
5209 "assignment decision version {target_version} is below durable retention floor {}",
5210 floor.before_target_version
5211 ))
5212 .into());
5213 }
5214 }
5215 Ok(self
5216 .audited_assignment_decisions_from(&head)
5217 .await?
5218 .into_iter()
5219 .find(|decision| decision.target_version() == target_version))
5220 }
5221
5222 pub async fn assignment_drain_decision(
5227 &self,
5228 target_version: u64,
5229 ) -> Result<Option<AssignmentDrainDecision>, ClusterCheckpointAuthorityError> {
5230 match self.assignment_decision(target_version).await? {
5231 Some(AuthorityAssignmentDecision::Drain(decision)) => Ok(Some(decision)),
5232 Some(AuthorityAssignmentDecision::Recovery(_)) => Err(DecisionError::Conflict(
5233 format!("target version {target_version} was settled by assignment recovery"),
5234 )
5235 .into()),
5236 None => Ok(None),
5237 }
5238 }
5239
5240 pub async fn assignment_recovery_decision(
5245 &self,
5246 target_version: u64,
5247 ) -> Result<Option<AssignmentRecoveryDecision>, ClusterCheckpointAuthorityError> {
5248 match self.assignment_decision(target_version).await? {
5249 Some(AuthorityAssignmentDecision::Recovery(decision)) => Ok(Some(decision)),
5250 Some(AuthorityAssignmentDecision::Drain(_)) => Err(DecisionError::Conflict(format!(
5251 "target version {target_version} was settled by assignment drain"
5252 ))
5253 .into()),
5254 None => Ok(None),
5255 }
5256 }
5257
5258 pub async fn assignment_handoff_checkpoint(
5263 &self,
5264 target: &CheckpointAssignmentFence,
5265 ) -> Result<Option<CommittedCheckpointRef>, ClusterCheckpointAuthorityError> {
5266 if !target.is_canonical() {
5267 return Err(LeaseError::Invalid(
5268 "assignment handoff checkpoint lookup requires a canonical target fence".into(),
5269 )
5270 .into());
5271 }
5272 Ok(self.load_record().await?.and_then(|head| {
5273 head.assignment_handoff_pin
5274 .filter(|pin| pin.target == *target)
5275 .map(|pin| pin.checkpoint)
5276 }))
5277 }
5278
5279 pub async fn materialize_assignment_recovery(
5287 &self,
5288 target_version: u64,
5289 ) -> Result<RotateOutcome, ClusterCheckpointAuthorityError> {
5290 let decision = self
5291 .assignment_recovery_decision(target_version)
5292 .await?
5293 .ok_or_else(|| {
5294 DecisionError::Conflict(format!(
5295 "assignment recovery target {target_version} has no authority decision"
5296 ))
5297 })?;
5298 let snapshots = AssignmentSnapshotStore::new(Arc::clone(&self.store));
5299 let proposal = snapshots
5300 .load_recovery_proposal(&decision.proposal)
5301 .await
5302 .map_err(|error| {
5303 assignment_snapshot_error("load authorized assignment recovery proposal", error)
5304 })?;
5305 if proposal.assignment_fence().map_err(|error| {
5306 LeaseError::Invalid(format!(
5307 "authorized assignment recovery proposal has no valid fence: {error}"
5308 ))
5309 })? != decision.target
5310 {
5311 return Err(LeaseError::Invalid(
5312 "authorized assignment recovery proposal does not match its target".into(),
5313 )
5314 .into());
5315 }
5316 snapshots
5317 .materialize_recovery(&decision.proposal)
5318 .await
5319 .map_err(|error| {
5320 assignment_snapshot_error("materialize authorized assignment recovery", error)
5321 .into()
5322 })
5323 }
5324
5325 pub async fn prune_assignment_drain_decisions_before(
5337 &self,
5338 proof: &LeaderProof,
5339 before_target_version: u64,
5340 ) -> Result<u64, ClusterCheckpointAuthorityError> {
5341 if before_target_version == 0 {
5342 return Ok(self
5343 .load_record()
5344 .await?
5345 .and_then(|head| head.assignment_decision_floor)
5346 .map_or(0, |floor| floor.before_target_version));
5347 }
5348 if !proof.is_canonical() {
5349 return Err(ClusterCheckpointAuthorityError::Fenced);
5350 }
5351 loop {
5352 let published = self
5353 .load_published_authority_head()
5354 .await?
5355 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5356 let current = &published.record;
5357 if !current.lease.matches_proof(proof) {
5358 return Err(ClusterCheckpointAuthorityError::Fenced);
5359 }
5360 if let Some(floor) = current.assignment_decision_floor.as_ref() {
5361 if floor.before_target_version >= before_target_version {
5362 self.schedule_history_prune();
5363 return Ok(floor.before_target_version);
5364 }
5365 }
5366
5367 let decisions = self.audited_assignment_decisions_from(current).await?;
5368 let terminal_anchor = decisions
5369 .iter()
5370 .rev()
5371 .find(|decision| decision.target_version() < before_target_version)
5372 .cloned()
5373 .or_else(|| {
5374 current
5375 .assignment_decision_floor
5376 .as_ref()
5377 .and_then(|floor| floor.terminal_anchor.clone())
5378 });
5379 let terminal_anchor_link = match terminal_anchor.as_ref() {
5380 Some(anchor) => Some(self.exact_assignment_decision_link(current, anchor).await?),
5381 None => None,
5382 };
5383 let floor = AuthorityAssignmentDecisionFloor {
5384 before_target_version,
5385 terminal_anchor,
5386 terminal_anchor_link,
5387 };
5388 floor.validate()?;
5389
5390 let base_sequence = current.lease.seq;
5391 let sequence = base_sequence
5392 .checked_add(1)
5393 .ok_or_else(|| LeaseError::Invalid("leader authority sequence exhausted".into()))?;
5394 let mut next = current.preserve_with_lease(LeaderLease {
5395 seq: sequence,
5396 renewal_sequence: current.lease.renewal_sequence,
5397 token: current.lease.token,
5398 owner: current.lease.owner.clone(),
5399 expires_at_ms: current.lease.expires_at_ms,
5400 catalog_manifest: current.lease.catalog_manifest.clone(),
5401 });
5402 next.assignment_decision_floor = Some(floor);
5403 next.validate()?;
5404
5405 match self
5406 .create_authority_record(Some(&published), &next)
5407 .await?
5408 {
5409 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5410 return Ok(before_target_version);
5411 }
5412 AuthorityCreateOutcome::Contended(winner) => {
5413 if let Some(winner_floor) = winner.assignment_decision_floor.as_ref() {
5414 if winner_floor.before_target_version >= before_target_version {
5415 self.schedule_history_prune();
5416 return Ok(winner_floor.before_target_version);
5417 }
5418 }
5419 if !winner.lease.matches_proof(proof) {
5420 return Err(ClusterCheckpointAuthorityError::Fenced);
5421 }
5422 if winner.lease.seq > base_sequence {
5423 tokio::task::yield_now().await;
5424 continue;
5425 }
5426 return Err(LeaseError::Invalid(
5427 "assignment decision floor contention did not advance the sequence".into(),
5428 )
5429 .into());
5430 }
5431 }
5432 }
5433 }
5434
5435 async fn cluster_outcome_from_snapshot(
5436 &self,
5437 head: &LeaderAuthorityRecord,
5438 epoch: u64,
5439 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5440 let floor = head.outcome_floor.as_ref();
5441 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5442 if epoch < artifact_before_epoch {
5443 return Ok(None);
5444 }
5445 let authority_before_epoch = floor.map_or(0, |floor| floor.authority_before_epoch);
5446 let cache_key = Self::cluster_outcome_audit_key(head);
5447 if let Some(snapshot) = self.cached_cluster_outcome_audit(&cache_key, head.lease.seq) {
5448 return Ok(snapshot
5449 .outcomes
5450 .binary_search_by_key(&epoch, |outcome| outcome.epoch)
5451 .ok()
5452 .map(|index| snapshot.outcomes[index].clone())
5453 .filter(|outcome| {
5454 outcome.epoch >= authority_before_epoch
5455 || outcome.is_commit()
5456 || floor
5457 .is_some_and(|floor| floor.terminal_anchor.as_ref() == Some(outcome))
5458 }));
5459 }
5460 if let Some(anchor) = floor
5461 .and_then(|floor| floor.terminal_anchor.as_ref())
5462 .filter(|anchor| anchor.epoch == epoch)
5463 {
5464 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5465 .load_or_create_deployment_id()
5466 .await?;
5467 if anchor.deployment_id != expected_deployment {
5468 return Err(DecisionError::Conflict(format!(
5469 "cluster terminal anchor epoch {} does not belong to current deployment {}",
5470 anchor.epoch, expected_deployment
5471 ))
5472 .into());
5473 }
5474 return Ok(Some(anchor.clone()));
5475 }
5476
5477 let commit_only = epoch < authority_before_epoch;
5478 let before_epoch = if commit_only {
5479 artifact_before_epoch
5480 } else {
5481 authority_before_epoch
5482 };
5483 let mut current = if commit_only {
5484 head.commit_head
5485 } else {
5486 head.outcome_head
5487 };
5488 let expected_anchor = if commit_only {
5489 floor.and_then(|floor| floor.committed_anchor_link)
5490 } else {
5491 floor.and_then(|floor| floor.terminal_anchor_link)
5492 };
5493 let expected_anchor_outcome = if commit_only {
5494 floor.and_then(|floor| floor.committed_anchor.as_ref())
5495 } else {
5496 floor.and_then(|floor| floor.terminal_anchor.as_ref())
5497 };
5498 let mut traversed = 0;
5499 while let Some(link) = current {
5500 if link.epoch < before_epoch {
5501 match (expected_anchor, expected_anchor_outcome) {
5502 (Some(anchor_link), Some(anchor))
5503 if link == anchor_link
5504 && link.epoch == anchor.epoch
5505 && link.checkpoint_id == anchor.checkpoint_id =>
5506 {
5507 return Ok(None)
5508 }
5509 (None, None) => return Ok(None),
5510 _ => {
5511 return Err(DecisionError::Conflict(format!(
5512 "cluster {} chain does not meet durable floor {before_epoch} at its exact anchor",
5513 if commit_only { "Commit" } else { "terminal outcome" }
5514 ))
5515 .into());
5516 }
5517 }
5518 }
5519 if link.epoch < epoch {
5520 return Ok(None);
5521 }
5522 if !consume_live_authority_link(&mut traversed) {
5523 return Err(DecisionError::Conflict(format!(
5524 "live {} retention exceeds the fixed {MAX_LIVE_AUTHORITY_LINKS}-link authority bound during exact lookup",
5525 if commit_only { "Commit" } else { "outcome" }
5526 ))
5527 .into());
5528 }
5529
5530 let admission;
5531 let record = if link.sequence == head.lease.seq {
5532 head
5533 } else {
5534 admission = read_authority_record(self.store.as_ref(), link.sequence)
5535 .await?
5536 .ok_or_else(|| {
5537 DecisionError::InventoryChanged(format!(
5538 "cluster {} authority record {} disappeared during exact lookup",
5539 if commit_only { "Commit" } else { "outcome" },
5540 link.sequence
5541 ))
5542 })?;
5543 &admission
5544 };
5545 let outcome = record.checkpoint_outcome.as_ref().ok_or_else(|| {
5546 DecisionError::Conflict(format!(
5547 "cluster {} head epoch {} points to non-outcome authority record {}",
5548 if commit_only { "Commit" } else { "outcome" },
5549 link.epoch,
5550 link.sequence
5551 ))
5552 })?;
5553 let record_head = if commit_only {
5554 record.commit_head
5555 } else {
5556 record.outcome_head
5557 };
5558 if record_head != Some(link)
5559 || (commit_only && !outcome.is_commit())
5560 || outcome.epoch != link.epoch
5561 || outcome.checkpoint_id != link.checkpoint_id
5562 {
5563 return Err(DecisionError::Conflict(format!(
5564 "cluster {} link epoch {} sequence {} does not match its authority record",
5565 if commit_only { "Commit" } else { "outcome" },
5566 link.epoch,
5567 link.sequence
5568 ))
5569 .into());
5570 }
5571 if outcome.epoch == epoch {
5572 let expected_deployment = CheckpointDecisionStore::new(Arc::clone(&self.store))
5573 .load_or_create_deployment_id()
5574 .await?;
5575 if floor.is_some_and(|floor| floor.deployment_id != expected_deployment)
5576 || outcome.deployment_id != expected_deployment
5577 {
5578 return Err(DecisionError::Conflict(format!(
5579 "cluster outcome epoch {} does not belong to current deployment {}",
5580 outcome.epoch, expected_deployment
5581 ))
5582 .into());
5583 }
5584 return Ok(Some(outcome.clone()));
5585 }
5586 current = if commit_only {
5587 record.previous_commit
5588 } else {
5589 record.previous_outcome
5590 };
5591 }
5592 if expected_anchor.is_some() {
5593 return Err(DecisionError::Conflict(format!(
5594 "cluster {} chain stopped without its durable floor anchor",
5595 if commit_only { "Commit" } else { "outcome" }
5596 ))
5597 .into());
5598 }
5599 Ok(None)
5600 }
5601
5602 async fn stable_cluster_outcome(
5603 &self,
5604 epoch: u64,
5605 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5606 const LOOKUP_RETRIES: usize = 3;
5607 for attempt in 0..LOOKUP_RETRIES {
5608 let Some(head) = self.load_record().await? else {
5609 return Ok(None);
5610 };
5611 match self.cluster_outcome_from_snapshot(&head, epoch).await {
5612 Err(ClusterCheckpointAuthorityError::Decision(
5613 DecisionError::InventoryChanged(_),
5614 )) if attempt + 1 < LOOKUP_RETRIES => {
5615 tokio::task::yield_now().await;
5616 }
5617 result => return result,
5618 }
5619 }
5620 Err(DecisionError::InventoryChanged(
5621 "cluster outcome exact lookup exhausted stability retries".into(),
5622 )
5623 .into())
5624 }
5625
5626 pub async fn cluster_outcome(
5632 &self,
5633 epoch: u64,
5634 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5635 self.stable_cluster_outcome(epoch).await
5636 }
5637
5638 pub async fn create_committed_checkpoint(
5644 &self,
5645 index: &CommittedCheckpointIndex,
5646 ) -> Result<CommittedCheckpointRef, ClusterCheckpointAuthorityError> {
5647 CheckpointDecisionStore::new(Arc::clone(&self.store))
5648 .create_committed_checkpoint(index)
5649 .await
5650 .map_err(Into::into)
5651 }
5652
5653 pub async fn load_committed_checkpoint(
5659 &self,
5660 reference: &CommittedCheckpointRef,
5661 ) -> Result<CommittedCheckpointIndex, ClusterCheckpointAuthorityError> {
5662 CheckpointDecisionStore::new(Arc::clone(&self.store))
5663 .load_committed_checkpoint(reference)
5664 .await
5665 .map_err(Into::into)
5666 }
5667
5668 pub async fn cluster_outcome_with_committed_checkpoint(
5675 &self,
5676 epoch: u64,
5677 ) -> Result<
5678 Option<(CheckpointOutcome, Option<CommittedCheckpointIndex>)>,
5679 ClusterCheckpointAuthorityError,
5680 > {
5681 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5682 let Some(outcome) = self.stable_cluster_outcome(epoch).await? else {
5683 return Ok(None);
5684 };
5685 let index = if outcome.is_commit() {
5686 let reference = outcome.committed_checkpoint.as_ref().ok_or_else(|| {
5687 DecisionError::Conflict(format!(
5688 "cluster Commit epoch {} checkpoint {} has no committed checkpoint index",
5689 outcome.epoch, outcome.checkpoint_id
5690 ))
5691 })?;
5692 let index = decisions.load_committed_checkpoint(reference).await?;
5693 if index.epoch != outcome.epoch
5694 || index.checkpoint_id != outcome.checkpoint_id
5695 || index.scope != outcome.scope
5696 || index.assignment_fence.as_ref() != outcome.assignment_fence.as_ref()
5697 || index.deployment_id != outcome.deployment_id
5698 {
5699 return Err(DecisionError::Conflict(format!(
5700 "cluster Commit epoch {} checkpoint {} does not match committed checkpoint '{}'",
5701 outcome.epoch, outcome.checkpoint_id, reference.sha256
5702 ))
5703 .into());
5704 }
5705 Some(index)
5706 } else {
5707 None
5708 };
5709 Ok(Some((outcome, index)))
5710 }
5711
5712 pub async fn cluster_outcome_inventory(
5718 &self,
5719 ) -> Result<ClusterOutcomeInventory, ClusterCheckpointAuthorityError> {
5720 let (head, outcomes) = self.audited_cluster_outcomes().await?;
5721 Ok(Self::cluster_outcome_inventory_from_audit(
5722 head.as_ref(),
5723 &outcomes,
5724 ))
5725 }
5726
5727 fn cluster_outcome_inventory_from_audit(
5728 head: Option<&LeaderAuthorityRecord>,
5729 outcomes: &[CheckpointOutcome],
5730 ) -> ClusterOutcomeInventory {
5731 let floor = head.and_then(|head| head.outcome_floor.as_ref());
5732 let artifact_before_epoch = floor.map_or(0, |floor| floor.artifact_before_epoch);
5733 ClusterOutcomeInventory {
5734 outcomes: outcomes
5735 .iter()
5736 .filter(|outcome| outcome.epoch >= artifact_before_epoch)
5737 .cloned()
5738 .collect(),
5739 retention_boundary: ClusterOutcomeRetentionBoundary::from_floor(floor),
5740 }
5741 }
5742
5743 pub async fn cluster_outcomes(
5749 &self,
5750 ) -> Result<Vec<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5751 Ok(self.cluster_outcome_inventory().await?.outcomes)
5752 }
5753
5754 pub async fn highest_cluster_committed_outcome(
5760 &self,
5761 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5762 Ok(self
5763 .cluster_outcomes()
5764 .await?
5765 .into_iter()
5766 .rev()
5767 .find(CheckpointOutcome::is_commit))
5768 }
5769
5770 pub async fn highest_cluster_terminal_outcome(
5776 &self,
5777 ) -> Result<Option<CheckpointOutcome>, ClusterCheckpointAuthorityError> {
5778 Ok(self.audited_cluster_outcomes().await?.1.last().cloned())
5779 }
5780
5781 fn cleanup_participant_ids(
5782 index: &CommittedCheckpointIndex,
5783 ) -> Result<Vec<u64>, ClusterCheckpointAuthorityError> {
5784 index.validate().map_err(DecisionError::Conflict)?;
5785 let participant_ids = index
5786 .participants
5787 .iter()
5788 .map(|participant| participant.participant_id)
5789 .collect::<Vec<_>>();
5790 if participant_ids.is_empty()
5791 || participant_ids.len() > MAX_CHECKPOINT_PARTICIPANTS
5792 || participant_ids[0] == 0
5793 || !participant_ids.windows(2).all(|pair| pair[0] < pair[1])
5794 {
5795 return Err(DecisionError::Conflict(
5796 "committed checkpoint participants are not canonical".into(),
5797 )
5798 .into());
5799 }
5800 Ok(participant_ids)
5801 }
5802
5803 fn cleanup_stop_before(head: &LeaderAuthorityRecord) -> Option<CommittedCheckpointRef> {
5804 head.outcome_floor
5805 .as_ref()
5806 .filter(|floor| floor.artifact_before_epoch != 0)
5807 .and_then(|floor| floor.committed_anchor.as_ref())
5808 .and_then(|outcome| outcome.committed_checkpoint.clone())
5809 .into_iter()
5810 .chain(head.topology_operations.iter().filter_map(|operation| {
5811 operation.has_target_commit().then_some(
5812 operation
5813 .cut
5814 .as_ref()?
5815 .committed
5816 .as_ref()?
5817 .checkpoint
5818 .clone(),
5819 )
5820 }))
5821 .max_by_key(|reference| (reference.epoch, reference.checkpoint_id))
5822 }
5823
5824 pub async fn cluster_artifact_cleanup(
5832 &self,
5833 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
5834 Ok(self
5835 .load_record()
5836 .await?
5837 .and_then(|head| head.artifact_cleanup))
5838 }
5839
5840 pub async fn begin_cluster_artifact_cleanup<V, Fut>(
5850 &self,
5851 proof: &LeaderProof,
5852 protected: CommittedCheckpointRef,
5853 validate_artifacts: V,
5854 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError>
5855 where
5856 V: Fn(CheckpointOutcome) -> Fut + Send + Sync + 'static,
5857 Fut: std::future::Future<Output = Result<(), String>> + Send + 'static,
5858 {
5859 if !proof.is_canonical() {
5860 return Err(ClusterCheckpointAuthorityError::Fenced);
5861 }
5862 protected.validate().map_err(DecisionError::Conflict)?;
5863 loop {
5864 let published = self
5865 .load_published_authority_head()
5866 .await?
5867 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5868 let current = &published.record;
5869 if !current.lease.matches_proof(proof) {
5870 return Err(ClusterCheckpointAuthorityError::Fenced);
5871 }
5872 if let Some(active) = current.artifact_cleanup.as_ref() {
5873 if active.protected == protected {
5874 return Ok(Some(active.clone()));
5875 }
5876 return Err(DecisionError::Conflict(format!(
5877 "cluster artifact cleanup for protected checkpoint {} is still active",
5878 active.protected.checkpoint_id
5879 ))
5880 .into());
5881 }
5882 if current.cleanup_is_pinned(&protected) {
5883 return Ok(None);
5884 }
5885
5886 let snapshot = self.cached_audited_cluster_outcomes_from(current).await?;
5887 let protected_outcome = snapshot
5888 .outcomes
5889 .iter()
5890 .find(|outcome| {
5891 outcome.is_commit() && outcome.committed_checkpoint.as_ref() == Some(&protected)
5892 })
5893 .cloned()
5894 .ok_or_else(|| {
5895 DecisionError::Conflict(format!(
5896 "protected checkpoint {} is not a live cluster Commit",
5897 protected.checkpoint_id
5898 ))
5899 })?;
5900 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
5901 let protected_index = decisions
5902 .validate_committed_checkpoint_for_outcome(&protected_outcome)
5903 .await?;
5904 validate_artifacts(protected_outcome.clone())
5905 .await
5906 .map_err(|error| {
5907 DecisionError::Conflict(format!(
5908 "protected cluster checkpoint {} failed durable artifact preflight: {error}",
5909 protected.checkpoint_id
5910 ))
5911 })?;
5912 let Some(expired) = protected_index.predecessor.clone() else {
5913 return Ok(None);
5914 };
5915 let stop_before = Self::cleanup_stop_before(current);
5916 if stop_before
5917 .as_ref()
5918 .is_some_and(|stop| expired.epoch <= stop.epoch)
5919 {
5920 return Ok(None);
5921 }
5922 self.audit_topology_cleanup_roots().await?;
5924 let expired_index = decisions.load_committed_checkpoint(&expired).await?;
5925 protected_index
5926 .validate_predecessor_index(&expired_index)
5927 .map_err(DecisionError::Conflict)?;
5928
5929 let authority_before_epoch = current
5930 .outcome_floor
5931 .as_ref()
5932 .map_or(protected.epoch, |floor| {
5933 floor.authority_before_epoch.max(protected.epoch)
5934 });
5935 let floor = self
5936 .build_cluster_outcome_floor(
5937 current,
5938 &snapshot,
5939 protected.epoch,
5940 authority_before_epoch,
5941 )
5942 .await?;
5943 if floor
5944 .committed_anchor
5945 .as_ref()
5946 .and_then(|outcome| outcome.committed_checkpoint.as_ref())
5947 != Some(&expired)
5948 {
5949 return Err(DecisionError::Conflict(
5950 "protected checkpoint predecessor is not the artifact-floor Commit anchor"
5951 .into(),
5952 )
5953 .into());
5954 }
5955 let cursor = ClusterArtifactCleanupCursor {
5956 protected: protected.clone(),
5957 current: expired,
5958 next: expired_index.predecessor.clone(),
5959 stop_before,
5960 participant_ids: Self::cleanup_participant_ids(&expired_index)?,
5961 phase: ClusterArtifactCleanupPhase::DeleteData,
5962 };
5963 cursor.validate()?;
5964
5965 let rechecked = self
5968 .load_published_authority_head()
5969 .await?
5970 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
5971 if !rechecked.record.lease.matches_proof(proof) {
5972 return Err(ClusterCheckpointAuthorityError::Fenced);
5973 }
5974 if rechecked.record.outcome_head != current.outcome_head
5975 || rechecked.record.commit_head != current.commit_head
5976 || rechecked.record.outcome_floor != current.outcome_floor
5977 || rechecked.record.artifact_cleanup != current.artifact_cleanup
5978 || rechecked.record.assignment_handoff_pin != current.assignment_handoff_pin
5979 || rechecked.record.topology_operations != current.topology_operations
5980 {
5981 tokio::task::yield_now().await;
5982 continue;
5983 }
5984 let sequence =
5985 rechecked.record.lease.seq.checked_add(1).ok_or_else(|| {
5986 LeaseError::Invalid("leader authority sequence exhausted".into())
5987 })?;
5988 let mut lease = rechecked.record.lease.clone();
5989 lease.seq = sequence;
5990 let mut next = rechecked.record.preserve_with_lease(lease);
5991 next.outcome_floor = Some(floor.clone());
5992 next.artifact_cleanup = Some(cursor.clone());
5993 next.validate()?;
5994 match self
5995 .create_authority_record(Some(&rechecked), &next)
5996 .await?
5997 {
5998 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
5999 self.install_cluster_outcome_audit(
6000 Self::cluster_outcome_audit_key(&next),
6001 next.lease.seq,
6002 Self::outcomes_retained_by_floor(&floor, &snapshot),
6003 );
6004 return Ok(Some(cursor));
6005 }
6006 AuthorityCreateOutcome::Contended(winner) => {
6007 if !winner.lease.matches_proof(proof) {
6008 return Err(ClusterCheckpointAuthorityError::Fenced);
6009 }
6010 tokio::task::yield_now().await;
6011 }
6012 }
6013 }
6014 }
6015
6016 pub async fn mark_cluster_artifact_data_deleted(
6021 &self,
6022 proof: &LeaderProof,
6023 expected: &ClusterArtifactCleanupCursor,
6024 ) -> Result<ClusterArtifactCleanupCursor, ClusterCheckpointAuthorityError> {
6025 if !proof.is_canonical() {
6026 return Err(ClusterCheckpointAuthorityError::Fenced);
6027 }
6028 expected.validate()?;
6029 if expected.phase != ClusterArtifactCleanupPhase::DeleteData {
6030 return Err(DecisionError::Conflict(
6031 "cluster artifact data transition requires a DeleteData cursor".into(),
6032 )
6033 .into());
6034 }
6035 let mut replacement = expected.clone();
6036 replacement.phase = ClusterArtifactCleanupPhase::DeleteMetadata;
6037 loop {
6038 let published = self
6039 .load_published_authority_head()
6040 .await?
6041 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6042 let current = &published.record;
6043 if !current.lease.matches_proof(proof) {
6044 return Err(ClusterCheckpointAuthorityError::Fenced);
6045 }
6046 if current.artifact_cleanup.as_ref() == Some(&replacement) {
6047 return Ok(replacement);
6048 }
6049 if current.artifact_cleanup.as_ref() != Some(expected) {
6050 return Err(DecisionError::Conflict(
6051 "cluster artifact cleanup cursor changed before data completion".into(),
6052 )
6053 .into());
6054 }
6055 let sequence =
6056 current.lease.seq.checked_add(1).ok_or_else(|| {
6057 LeaseError::Invalid("leader authority sequence exhausted".into())
6058 })?;
6059 let mut lease = current.lease.clone();
6060 lease.seq = sequence;
6061 let mut next = current.preserve_with_lease(lease);
6062 next.artifact_cleanup = Some(replacement.clone());
6063 next.validate()?;
6064 match self
6065 .create_authority_record(Some(&published), &next)
6066 .await?
6067 {
6068 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6069 return Ok(replacement);
6070 }
6071 AuthorityCreateOutcome::Contended(winner) => {
6072 if !winner.lease.matches_proof(proof) {
6073 return Err(ClusterCheckpointAuthorityError::Fenced);
6074 }
6075 tokio::task::yield_now().await;
6076 }
6077 }
6078 }
6079 }
6080
6081 pub async fn mark_cluster_artifact_metadata_deleted(
6091 &self,
6092 proof: &LeaderProof,
6093 expected: &ClusterArtifactCleanupCursor,
6094 ) -> Result<Option<ClusterArtifactCleanupCursor>, ClusterCheckpointAuthorityError> {
6095 if !proof.is_canonical() {
6096 return Err(ClusterCheckpointAuthorityError::Fenced);
6097 }
6098 expected.validate()?;
6099 if expected.phase != ClusterArtifactCleanupPhase::DeleteMetadata {
6100 return Err(DecisionError::Conflict(
6101 "cluster artifact metadata transition requires a DeleteMetadata cursor".into(),
6102 )
6103 .into());
6104 }
6105 loop {
6106 let published = self
6107 .load_published_authority_head()
6108 .await?
6109 .ok_or(ClusterCheckpointAuthorityError::Fenced)?;
6110 let current = &published.record;
6111 if !current.lease.matches_proof(proof) {
6112 return Err(ClusterCheckpointAuthorityError::Fenced);
6113 }
6114 if current.artifact_cleanup.as_ref() != Some(expected) {
6115 let completed = expected.next.is_none()
6116 || expected.next.as_ref() == expected.stop_before.as_ref();
6117 if completed && current.artifact_cleanup.is_none() {
6118 return Ok(None);
6119 }
6120 if let (Some(next_reference), Some(actual)) =
6121 (expected.next.as_ref(), current.artifact_cleanup.as_ref())
6122 {
6123 if actual.phase == ClusterArtifactCleanupPhase::DeleteData
6124 && actual.protected == expected.protected
6125 && &actual.current == next_reference
6126 && actual.stop_before == expected.stop_before
6127 {
6128 return Ok(Some(actual.clone()));
6129 }
6130 }
6131 return Err(DecisionError::Conflict(
6132 "cluster artifact cleanup cursor changed before metadata completion".into(),
6133 )
6134 .into());
6135 }
6136
6137 let replacement = match expected.next.as_ref() {
6138 None => None,
6139 Some(next_reference) if Some(next_reference) == expected.stop_before.as_ref() => {
6140 None
6141 }
6142 Some(next_reference) => {
6143 let decisions = CheckpointDecisionStore::new(Arc::clone(&self.store));
6144 let index = decisions.load_committed_checkpoint(next_reference).await?;
6145 if index.scope != CheckpointScope::Cluster {
6146 return Err(DecisionError::Conflict(
6147 "cluster artifact cleanup next checkpoint is not cluster-scoped".into(),
6148 )
6149 .into());
6150 }
6151 let replacement = ClusterArtifactCleanupCursor {
6152 protected: expected.protected.clone(),
6153 current: next_reference.clone(),
6154 next: index.predecessor.clone(),
6155 stop_before: expected.stop_before.clone(),
6156 participant_ids: Self::cleanup_participant_ids(&index)?,
6157 phase: ClusterArtifactCleanupPhase::DeleteData,
6158 };
6159 replacement.validate()?;
6160 Some(replacement)
6161 }
6162 };
6163
6164 let sequence =
6165 current.lease.seq.checked_add(1).ok_or_else(|| {
6166 LeaseError::Invalid("leader authority sequence exhausted".into())
6167 })?;
6168 let mut lease = current.lease.clone();
6169 lease.seq = sequence;
6170 let mut next = current.preserve_with_lease(lease);
6171 next.artifact_cleanup = replacement.clone();
6172 next.validate()?;
6173 match self
6174 .create_authority_record(Some(&published), &next)
6175 .await?
6176 {
6177 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6178 return Ok(replacement);
6179 }
6180 AuthorityCreateOutcome::Contended(winner) => {
6181 if !winner.lease.matches_proof(proof) {
6182 return Err(ClusterCheckpointAuthorityError::Fenced);
6183 }
6184 tokio::task::yield_now().await;
6185 }
6186 }
6187 }
6188 }
6189
6190 pub async fn cluster_outcome_retention_boundary(
6196 &self,
6197 ) -> Result<ClusterOutcomeRetentionBoundary, ClusterCheckpointAuthorityError> {
6198 let head = self.load_record().await?;
6199 Ok(ClusterOutcomeRetentionBoundary::from_floor(
6200 head.as_ref().and_then(|head| head.outcome_floor.as_ref()),
6201 ))
6202 }
6203
6204 pub async fn begin_new_term(
6211 &self,
6212 owner: &LeaderLeaseOwner,
6213 now_ms: i64,
6214 ) -> Result<LeaseOutcome, LeaseError> {
6215 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6216 owner,
6217 now_ms,
6218 SameOwnerToken::Rotate,
6219 ))
6220 .await
6221 }
6222
6223 pub async fn renew_exact(
6229 &self,
6230 owner: &LeaderLeaseOwner,
6231 token: u64,
6232 now_ms: i64,
6233 ) -> Result<LeaseOutcome, LeaseError> {
6234 Box::pin(self.acquire_or_renew_current_term_for_test_inner(
6235 owner,
6236 now_ms,
6237 SameOwnerToken::Exact(token),
6238 ))
6239 .await
6240 }
6241
6242 #[cfg(test)]
6243 pub(crate) async fn acquire_or_renew_current_term_for_test(
6244 &self,
6245 owner: &LeaderLeaseOwner,
6246 now_ms: i64,
6247 ) -> Result<LeaseOutcome, LeaseError> {
6248 self.acquire_or_renew_current_term_for_test_inner(owner, now_ms, SameOwnerToken::Preserve)
6249 .await
6250 }
6251
6252 async fn acquire_or_renew_current_term_for_test_inner(
6253 &self,
6254 owner: &LeaderLeaseOwner,
6255 now_ms: i64,
6256 same_owner_token: SameOwnerToken,
6257 ) -> Result<LeaseOutcome, LeaseError> {
6258 owner.validate()?;
6259 self.ttl()?;
6260 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6261 loop {
6262 let published = self.load_published_authority_head().await?;
6263 let candidate = match published.as_ref() {
6264 None => {
6265 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6266 return Err(LeaseError::Fenced(
6267 "exact leader renewal lost the durable authority head".into(),
6268 ));
6269 }
6270 LeaderAuthorityRecord::initial(LeaderLease {
6271 seq: 1,
6272 renewal_sequence: 1,
6273 token: 1,
6274 owner: owner.clone(),
6275 expires_at_ms,
6276 catalog_manifest: None,
6277 })
6278 }
6279 Some(head) if head.record.lease.owner == *owner => {
6280 let record = &head.record;
6281 let token = match same_owner_token {
6282 #[cfg(test)]
6283 SameOwnerToken::Preserve => record.lease.token,
6284 SameOwnerToken::Rotate => {
6285 record.lease.token.checked_add(1).ok_or_else(|| {
6286 LeaseError::Invalid("leader fencing token exhausted".into())
6287 })?
6288 }
6289 SameOwnerToken::Exact(expected) if expected == record.lease.token => {
6290 expected
6291 }
6292 SameOwnerToken::Exact(_) => {
6293 return Err(LeaseError::Fenced(
6294 "exact leader renewal was superseded by a newer local term".into(),
6295 ));
6296 }
6297 };
6298 let lease = LeaderLease {
6299 seq: record.lease.seq.checked_add(1).ok_or_else(|| {
6300 LeaseError::Invalid("lease sequence exhausted".into())
6301 })?,
6302 renewal_sequence: record.lease.renewal_sequence.checked_add(1).ok_or_else(
6303 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6304 )?,
6305 token,
6306 owner: owner.clone(),
6307 expires_at_ms,
6308 catalog_manifest: record.lease.catalog_manifest.clone(),
6309 };
6310 record.preserve_with_lease(lease)
6311 }
6312 Some(head) => {
6313 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6314 return Err(LeaseError::Fenced(
6315 "exact leader renewal was superseded by a rival owner".into(),
6316 ));
6317 }
6318 return Ok(LeaseOutcome::Held(head.record.lease.clone()));
6319 }
6320 };
6321 match self
6322 .create_authority_record(published.as_ref(), &candidate)
6323 .await?
6324 {
6325 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6326 return Ok(LeaseOutcome::Acquired(candidate.lease));
6327 }
6328 AuthorityCreateOutcome::Contended(winner) if winner.lease.owner == *owner => {
6329 if matches!(
6330 same_owner_token,
6331 SameOwnerToken::Exact(expected) if winner.lease.token != expected
6332 ) {
6333 return Err(LeaseError::Fenced(
6334 "exact leader renewal lost a same-owner CAS race".into(),
6335 ));
6336 }
6337 tokio::task::yield_now().await;
6338 }
6339 AuthorityCreateOutcome::Contended(winner) => {
6340 if matches!(same_owner_token, SameOwnerToken::Exact(_)) {
6341 return Err(LeaseError::Fenced(
6342 "exact leader renewal lost a rival-owner CAS race".into(),
6343 ));
6344 }
6345 return Ok(LeaseOutcome::Held(winner.lease));
6346 }
6347 }
6348 }
6349 }
6350
6351 pub fn observe_rival(
6356 &self,
6357 owner: &LeaderLeaseOwner,
6358 lease: &LeaderLease,
6359 ) -> Result<LeaderLeaseObservation, LeaseError> {
6360 owner.validate()?;
6361 lease.validate()?;
6362 self.ttl()?;
6363 if lease.owner == *owner {
6364 return Err(LeaseError::Invalid(
6365 "leader takeover observation must belong to a rival".into(),
6366 ));
6367 }
6368 Ok(LeaderLeaseObservation {
6369 lease: lease.clone(),
6370 started: Instant::now(),
6371 })
6372 }
6373
6374 pub async fn try_takeover(
6381 &self,
6382 owner: &LeaderLeaseOwner,
6383 observation: &LeaderLeaseObservation,
6384 now_ms: i64,
6385 ) -> Result<LeaseOutcome, LeaseError> {
6386 owner.validate()?;
6387 observation.lease.validate()?;
6388 let ttl = self.ttl()?;
6389 if observation.lease.owner == *owner {
6390 return Err(LeaseError::Invalid(
6391 "leader takeover observation must belong to a rival".into(),
6392 ));
6393 }
6394 let observed_for_ttl = observation.started.elapsed() >= ttl;
6395 let expires_at_ms = self.diagnostic_expiry(now_ms)?;
6396 loop {
6397 let published = self
6398 .load_published_authority_head()
6399 .await?
6400 .ok_or_else(|| LeaseError::Invalid("observed leader lease disappeared".into()))?;
6401 let current = &published.record;
6402 let unrenewed =
6403 observed_for_ttl && current.lease.has_same_liveness_identity(&observation.lease);
6404 if !unrenewed && !self.term_released(current).await? {
6407 return Ok(LeaseOutcome::Held(current.lease.clone()));
6408 }
6409 let candidate_lease = LeaderLease {
6410 seq: current
6411 .lease
6412 .seq
6413 .checked_add(1)
6414 .ok_or_else(|| LeaseError::Invalid("lease sequence exhausted".into()))?,
6415 renewal_sequence: current.lease.renewal_sequence.checked_add(1).ok_or_else(
6416 || LeaseError::Invalid("lease renewal sequence exhausted".into()),
6417 )?,
6418 token: current
6419 .lease
6420 .token
6421 .checked_add(1)
6422 .ok_or_else(|| LeaseError::Invalid("fencing token exhausted".into()))?,
6423 owner: owner.clone(),
6424 expires_at_ms,
6425 catalog_manifest: current.lease.catalog_manifest.clone(),
6426 };
6427 let candidate = current.preserve_with_lease(candidate_lease);
6428 match self
6429 .create_authority_record(Some(&published), &candidate)
6430 .await?
6431 {
6432 AuthorityCreateOutcome::Created | AuthorityCreateOutcome::ExistingIdentical => {
6433 return Ok(LeaseOutcome::Acquired(candidate.lease));
6434 }
6435 AuthorityCreateOutcome::Contended(winner)
6436 if winner.lease.has_same_liveness_identity(&observation.lease) =>
6437 {
6438 tokio::task::yield_now().await;
6439 }
6440 AuthorityCreateOutcome::Contended(winner) => {
6441 return Ok(LeaseOutcome::Held(winner.lease));
6442 }
6443 }
6444 }
6445 }
6446}
6447
6448#[derive(Debug, Clone, Copy)]
6450pub struct LeaderLeaseConfig {
6451 pub ttl: Duration,
6453 pub renew_interval: Duration,
6455}
6456
6457impl Default for LeaderLeaseConfig {
6458 fn default() -> Self {
6459 Self {
6460 ttl: Duration::from_secs(10),
6465 renew_interval: Duration::from_secs(2),
6466 }
6467 }
6468}
6469
6470#[must_use]
6472pub fn lease_grants_leadership(
6473 lease: &Option<LeaderLease>,
6474 owner: &LeaderLeaseOwner,
6475 deadline: &LeaseDeadline,
6476) -> bool {
6477 deadline.is_live() && matches!(lease, Some(lease) if lease.owner == *owner)
6478}
6479
6480#[must_use]
6482pub fn lease_grants_proof(
6483 lease: &Option<LeaderLease>,
6484 owner: &LeaderLeaseOwner,
6485 deadline: &LeaseDeadline,
6486 proof: &LeaderProof,
6487) -> bool {
6488 deadline.is_live()
6489 && proof.owner == owner.proof_owner()
6490 && matches!(lease, Some(lease) if lease.owner == *owner && lease.matches_proof(proof))
6491}
6492
6493#[cfg(test)]
6494mod tests;