1use super::{ClusterController, LeaderProof};
4use crate::checkpoint::CheckpointAttempt;
5use crate::checkpoint_decision::CheckpointArtifactInventory;
6use crate::cluster::control::topology::{
7 TopologyAdmissionStatus, TopologyError, TopologyOperationId, TopologyPlanRef,
8};
9
10#[derive(Debug)]
12#[must_use = "periodic checkpoints resume as soon as the hold is dropped"]
13pub struct PeriodicCheckpointHold<'a>(&'a std::sync::atomic::AtomicUsize);
14
15impl Drop for PeriodicCheckpointHold<'_> {
16 fn drop(&mut self) {
17 self.0.fetch_sub(1, std::sync::atomic::Ordering::AcqRel);
18 }
19}
20
21impl ClusterController {
22 pub fn hold_periodic_checkpoints(&self) -> PeriodicCheckpointHold<'_> {
26 self.checkpoint_gap_waiters
27 .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
28 PeriodicCheckpointHold(&self.checkpoint_gap_waiters)
29 }
30
31 #[must_use]
33 pub fn periodic_checkpoints_held(&self) -> bool {
34 self.checkpoint_gap_waiters
35 .load(std::sync::atomic::Ordering::Acquire)
36 > 0
37 }
38}
39
40impl ClusterController {
41 pub async fn authorize_topology_release(
46 &self,
47 input: &crate::cluster::control::TopologyRestoreInput,
48 runtime_id: uuid::Uuid,
49 ) -> Result<bool, TopologyError> {
50 if self.is_recovering()
51 || self.is_draining()
52 || self.try_live_local_process_authority_identity().ok() != Some(input.process())
53 {
54 return Ok(false);
55 }
56 let allowed = self
57 .checkpoint_authority()
58 .map_err(|error| TopologyError::Protocol(error.to_string()))?
59 .authorize_topology_release(
60 self.snapshot.as_ref().ok_or(TopologyError::Fenced)?,
61 self.process_lease_authority
62 .get()
63 .ok_or(TopologyError::Fenced)?,
64 input,
65 runtime_id,
66 )
67 .await?;
68 Ok(allowed
69 && !self.is_recovering()
70 && !self.is_draining()
71 && self.try_live_local_process_authority_identity().ok() == Some(input.process())
72 && self
73 .checkpoint_assignment_fence(input.assignment().assignment_version)
74 .as_ref()
75 == Some(input.assignment()))
76 }
77
78 pub async fn certify_topology_installation(
83 &self,
84 input: &crate::cluster::control::TopologyRestoreInput,
85 runtime_id: uuid::Uuid,
86 ) -> Result<TopologyAdmissionStatus, TopologyError> {
87 let before = self
88 .committed_topology_restore_input(input.operation().operation_id)
89 .await?;
90 if !before.same_restore_requirements(input) || self.is_recovering() || self.is_draining() {
91 return Err(TopologyError::Fenced);
92 }
93 self.checkpoint_authority()
94 .map_err(|error| TopologyError::Protocol(error.to_string()))?
95 .certify_topology_installation(
96 self.snapshot.as_ref().ok_or(TopologyError::Fenced)?,
97 self.process_lease_authority
98 .get()
99 .ok_or(TopologyError::Fenced)?,
100 input,
101 runtime_id,
102 crate::cluster::control::TOPOLOGY_INSTALLATION_PROTOCOL_VERSION,
103 )
104 .await?;
105 let after = self
106 .committed_topology_restore_input(input.operation().operation_id)
107 .await?;
108 if !after.same_restore_requirements(input) || self.is_recovering() || self.is_draining() {
109 return Err(TopologyError::Fenced);
110 }
111 Ok(after.operation().clone())
112 }
113
114 pub async fn release_topology_target(
119 &self,
120 input: &crate::cluster::control::TopologyRestoreInput,
121 ) -> Result<TopologyAdmissionStatus, TopologyError> {
122 let proof = self.capture_leader_proof().ok_or(TopologyError::Fenced)?;
123 if self.is_recovering() || self.is_draining() {
124 return Err(TopologyError::Fenced);
125 }
126 let status = self
127 .checkpoint_authority()
128 .map_err(|error| TopologyError::Protocol(error.to_string()))?
129 .release_topology_target(
130 &proof,
131 self.snapshot.as_ref().ok_or(TopologyError::Fenced)?,
132 self.process_lease_authority
133 .get()
134 .ok_or(TopologyError::Fenced)?,
135 input,
136 )
137 .await?;
138 if self.capture_leader_proof().as_ref() != Some(&proof)
139 || self.is_recovering()
140 || self.is_draining()
141 || self.try_live_local_process_authority_identity().ok() != Some(input.process())
142 {
143 return Err(TopologyError::Fenced);
144 }
145 Ok(status)
146 }
147
148 pub async fn certify_topology_target_preparation(
156 &self,
157 input: &crate::cluster::control::TopologyRestoreInput,
158 ) -> Result<TopologyAdmissionStatus, TopologyError> {
159 let before = self
160 .topology_restore_input(input.operation().operation_id)
161 .await?;
162 if !before.same_restore_requirements(input) {
163 return Err(TopologyError::Fenced);
164 }
165 let authority = self
166 .checkpoint_authority()
167 .map_err(|e| TopologyError::Protocol(e.to_string()))?;
168 let status = authority
169 .certify_topology_target_preparation(
170 self.snapshot.as_ref().ok_or_else(|| {
171 TopologyError::Protocol("target preparation has no assignment authority".into())
172 })?,
173 self.process_lease_authority.get().ok_or_else(|| {
174 TopologyError::Protocol("target preparation has no process authority".into())
175 })?,
176 input,
177 crate::cluster::control::TOPOLOGY_COMMIT_PROTOCOL_VERSION,
178 )
179 .await?;
180 let after = self
181 .topology_restore_input(input.operation().operation_id)
182 .await?;
183 if !after.same_restore_requirements(input)
184 || status
185 .target_preparations
186 .iter()
187 .any(|receipt| !after.operation().target_preparations.contains(receipt))
188 {
189 return Err(TopologyError::Fenced);
190 }
191 Ok(after.operation().clone())
192 }
193
194 pub async fn topology_restore_input(
200 &self,
201 operation_id: TopologyOperationId,
202 ) -> Result<crate::cluster::control::TopologyRestoreInput, TopologyError> {
203 self.read_topology_restore_input(operation_id, false).await
204 }
205
206 pub async fn committed_topology_restore_input(
212 &self,
213 operation_id: TopologyOperationId,
214 ) -> Result<crate::cluster::control::TopologyRestoreInput, TopologyError> {
215 self.read_topology_restore_input(operation_id, true).await
216 }
217
218 pub async fn committed_topology_recovery_input(
225 &self,
226 operation_id: TopologyOperationId,
227 ) -> Result<crate::cluster::control::TopologyRecoveryInput, TopologyError> {
228 let before = self
229 .try_live_local_process_authority_identity()
230 .map_err(|_| TopologyError::Fenced)?;
231 if self.is_draining() {
232 return Err(TopologyError::Fenced);
233 }
234 let authority = self
235 .checkpoint_authority()
236 .map_err(|error| TopologyError::Protocol(error.to_string()))?;
237 let assignments = self.snapshot.as_ref().ok_or_else(|| {
238 TopologyError::Protocol("recovery has no assignment authority".into())
239 })?;
240 let processes = self
241 .process_lease_authority
242 .get()
243 .ok_or_else(|| TopologyError::Protocol("recovery has no process authority".into()))?;
244 let input = authority
245 .committed_topology_recovery_input(assignments, processes, operation_id, before)
246 .await?;
247 self.require_topology_restore_adoption(input.migration(), before, true)
248 .await?;
249 Ok(input)
250 }
251
252 pub async fn topology_recovery_input(
259 &self,
260 round: &super::RecoveryRound,
261 epoch: u64,
262 ) -> Result<crate::cluster::control::TopologyRecoveryInput, TopologyError> {
263 tokio::time::timeout(std::time::Duration::from_secs(15), async {
264 let binding = round.topology_binding().ok_or(TopologyError::Fenced)?;
265 if !self.is_recovering()
266 || self.is_draining()
267 || !self.recovery_round_contains_current_process(round)
268 {
269 return Err(TopologyError::Fenced);
270 }
271 self.audit_recovery_topology(round, Some(epoch))
272 .await
273 .map_err(|error| TopologyError::Conflict(error.to_string()))?;
274 let before = self
275 .try_live_local_process_authority_identity()
276 .map_err(|_| TopologyError::Fenced)?;
277 if !binding.processes().contains(&before) {
278 return Err(TopologyError::Fenced);
279 }
280 let authority = self
281 .checkpoint_authority()
282 .map_err(|error| TopologyError::Protocol(error.to_string()))?;
283 let input = authority
284 .committed_topology_recovery_input(
285 self.snapshot.as_deref().ok_or(TopologyError::Fenced)?,
286 self.process_lease_authority
287 .get()
288 .ok_or(TopologyError::Fenced)?,
289 binding.commit().operation_id,
290 before,
291 )
292 .await?;
293 let evidence = self
294 .read_local_process_authority_evidence()
295 .await
296 .map_err(|error| TopologyError::Conflict(error.to_string()))?;
297 if input.outcome().epoch != epoch
298 || input.migration().operation().commit.as_ref() != Some(binding.commit())
299 || input.migration().assignment() != &round.assignment_fence
300 || input.migration().processes() != binding.processes()
301 || input.migration().current_leader().as_ref() != Some(&round.leader_proof)
302 || evidence.participant != before.participant
303 || evidence.process_term != before.process_term
304 || !evidence
305 .adopted_assignment
306 .matches_fence(&round.assignment_fence)
307 || self
308 .checkpoint_assignment_fence(round.assignment_fence.assignment_version)
309 .is_some_and(|fence| fence != round.assignment_fence)
310 || self.try_live_local_process_authority_identity().ok() != Some(before)
311 || !self.is_recovering()
312 || self.is_draining()
313 {
314 return Err(TopologyError::Fenced);
315 }
316 Ok(input)
317 })
318 .await
319 .map_err(|_| TopologyError::ReadTimedOut)?
320 }
321
322 pub async fn certify_topology_recovery_installation(
328 &self,
329 start: &super::RecoveryAnnouncement,
330 input: &crate::cluster::control::TopologyRecoveryInput,
331 runtime_id: uuid::Uuid,
332 ) -> Result<TopologyAdmissionStatus, TopologyError> {
333 let super::RecoverPhase::Start { epoch } = start.phase else {
334 return Err(TopologyError::Fenced);
335 };
336 if self
337 .observe_recover_control()
338 .await
339 .map_err(|error| TopologyError::Conflict(error.to_string()))?
340 .as_ref()
341 != Some(start)
342 {
343 return Err(TopologyError::Fenced);
344 }
345 let fresh = self.topology_recovery_input(&start.round, epoch).await?;
346 if !fresh.same_restore_requirements(input) {
347 return Err(TopologyError::Fenced);
348 }
349 self.checkpoint_authority()
350 .map_err(|error| TopologyError::Protocol(error.to_string()))?
351 .certify_topology_recovery_installation(
352 self.snapshot.as_deref().ok_or(TopologyError::Fenced)?,
353 self.process_lease_authority
354 .get()
355 .ok_or(TopologyError::Fenced)?,
356 fresh.migration(),
357 runtime_id,
358 &start.round,
359 epoch,
360 )
361 .await?;
362 let after = self.topology_recovery_input(&start.round, epoch).await?;
363 if !after.same_restore_requirements(input)
364 || self
365 .observe_recover_control()
366 .await
367 .map_err(|error| TopologyError::Conflict(error.to_string()))?
368 .as_ref()
369 != Some(start)
370 {
371 return Err(TopologyError::Fenced);
372 }
373 Ok(after.migration().operation().clone())
374 }
375
376 async fn read_topology_restore_input(
377 &self,
378 operation_id: TopologyOperationId,
379 committed: bool,
380 ) -> Result<crate::cluster::control::TopologyRestoreInput, TopologyError> {
381 let before = self
382 .try_live_local_process_authority_identity()
383 .map_err(|_| TopologyError::Fenced)?;
384 if (!committed && self.is_recovering()) || self.is_draining() {
385 return Err(TopologyError::Fenced);
386 }
387 let authority = self
388 .checkpoint_authority()
389 .map_err(|e| TopologyError::Protocol(e.to_string()))?;
390 let assignments = self
391 .snapshot
392 .as_ref()
393 .ok_or_else(|| TopologyError::Protocol("restore has no assignment authority".into()))?;
394 let processes = self
395 .process_lease_authority
396 .get()
397 .ok_or_else(|| TopologyError::Protocol("restore has no process authority".into()))?;
398 let input = if committed {
399 authority
400 .committed_topology_restore_input(assignments, processes, operation_id, before)
401 .await?
402 } else {
403 authority
404 .topology_restore_input(assignments, processes, operation_id, before)
405 .await?
406 };
407 self.require_topology_restore_adoption(&input, before, committed)
408 .await?;
409 Ok(input)
410 }
411
412 async fn require_topology_restore_adoption(
413 &self,
414 input: &crate::cluster::control::TopologyRestoreInput,
415 before: super::LocalProcessAuthorityIdentity,
416 committed: bool,
417 ) -> Result<(), TopologyError> {
418 let evidence = self
419 .read_local_process_authority_evidence()
420 .await
421 .map_err(|e| TopologyError::Conflict(e.to_string()))?;
422 if (!committed && self.is_recovering())
423 || self.is_draining()
424 || self.try_live_local_process_authority_identity().ok() != Some(before)
425 || evidence.participant != before.participant
426 || evidence.process_term != before.process_term
427 || !evidence
428 .adopted_assignment
429 .matches_fence(input.assignment())
430 || self
431 .checkpoint_assignment_fence(input.assignment().assignment_version)
432 .as_ref()
433 != Some(input.assignment())
434 {
435 return Err(TopologyError::Fenced);
436 }
437 Ok(())
438 }
439
440 pub async fn commit_topology_target(
447 &self,
448 input: &crate::cluster::control::TopologyRestoreInput,
449 ) -> Result<TopologyAdmissionStatus, TopologyError> {
450 let proof = self.capture_leader_proof().ok_or(TopologyError::Fenced)?;
451 if self.is_recovering()
452 || self.is_draining()
453 || !self.proof_is_live(&proof)
454 || self.try_live_local_process_authority_identity().ok() != Some(input.process())
455 {
456 return Err(TopologyError::Fenced);
457 }
458 let evidence = self
459 .read_local_process_authority_evidence()
460 .await
461 .map_err(|_| TopologyError::Fenced)?;
462 if evidence.participant != input.process().participant
463 || evidence.process_term != input.process().process_term
464 || !evidence
465 .adopted_assignment
466 .matches_fence(input.assignment())
467 {
468 return Err(TopologyError::Fenced);
469 }
470 let status = self
471 .checkpoint_authority()
472 .map_err(|error| TopologyError::Protocol(error.to_string()))?
473 .commit_topology_target(
474 &proof,
475 self.snapshot.as_ref().ok_or_else(|| {
476 TopologyError::Protocol("Commit has no assignment authority".into())
477 })?,
478 self.process_lease_authority.get().ok_or_else(|| {
479 TopologyError::Protocol("Commit has no process authority".into())
480 })?,
481 input,
482 )
483 .await?;
484 if !self.proof_is_live(&proof)
485 || self.try_live_local_process_authority_identity().ok() != Some(input.process())
486 {
487 return Err(TopologyError::Fenced);
488 }
489 Ok(status)
490 }
491
492 pub async fn stage_topology_migration_root(
498 &self,
499 checkpoint_store: &dyn crate::checkpoint::CheckpointStore,
500 operation_id: TopologyOperationId,
501 expected_plan: &TopologyPlanRef,
502 ) -> Result<TopologyAdmissionStatus, TopologyError> {
503 self.stage_topology_migration_root_with_initialization(
504 checkpoint_store,
505 operation_id,
506 expected_plan,
507 |_, _| async {
508 Err(TopologyError::Unsupported(
509 "new sources require connector initialization".into(),
510 ))
511 },
512 )
513 .await
514 }
515
516 pub async fn stage_topology_migration_root_with_initialization<F, Fut>(
523 &self,
524 checkpoint_store: &dyn crate::checkpoint::CheckpointStore,
525 operation_id: TopologyOperationId,
526 expected_plan: &TopologyPlanRef,
527 initialize: F,
528 ) -> Result<TopologyAdmissionStatus, TopologyError>
529 where
530 F: FnOnce(
531 crate::cluster::control::CatalogManifest,
532 crate::cluster::control::ClusterTopologyValidation,
533 ) -> Fut,
534 Fut: std::future::Future<
535 Output = Result<
536 Vec<crate::cluster::control::TopologySourceInitialization>,
537 TopologyError,
538 >,
539 >,
540 {
541 let before = self
542 .try_live_local_process_authority_identity()
543 .map_err(|_| TopologyError::Fenced)?;
544 let proof = self.capture_leader_proof().ok_or(TopologyError::Fenced)?;
545 if self.is_recovering() || self.is_draining() || !self.proof_is_live(&proof) {
546 return Err(TopologyError::Fenced);
547 }
548 let authority = self
549 .checkpoint_authority()
550 .map_err(|e| TopologyError::Protocol(e.to_string()))?;
551 let (_, plan, _, _) = authority.topology_preparation_input(operation_id).await?;
552 let evidence = self
553 .read_local_process_authority_evidence()
554 .await
555 .map_err(|e| TopologyError::Conflict(e.to_string()))?;
556 if evidence.participant != before.participant
557 || evidence.process_term != before.process_term
558 || !evidence.adopted_assignment.matches_fence(&plan.assignment)
559 {
560 return Err(TopologyError::Fenced);
561 }
562 let status = authority
563 .stage_topology_migration_root_with_initialization(
564 &proof,
565 self.snapshot.as_ref().ok_or_else(|| {
566 TopologyError::Protocol("root has no assignment authority".into())
567 })?,
568 self.process_lease_authority.get().ok_or_else(|| {
569 TopologyError::Protocol("root has no process authority".into())
570 })?,
571 checkpoint_store,
572 operation_id,
573 expected_plan,
574 initialize,
575 )
576 .await?;
577 if self.is_recovering()
578 || self.is_draining()
579 || !self.proof_is_live(&proof)
580 || self.try_live_local_process_authority_identity().ok() != Some(before)
581 || self
582 .checkpoint_assignment_fence(plan.assignment.assignment_version)
583 .as_ref()
584 != Some(&plan.assignment)
585 {
586 return Err(TopologyError::Fenced);
587 }
588 Ok(status)
589 }
590
591 pub async fn certify_topology_candidate(
597 &self,
598 operation_id: TopologyOperationId,
599 expected_plan: &TopologyPlanRef,
600 before: super::LocalProcessAuthorityIdentity,
601 compiled: &crate::cluster::control::topology::ClusterTopologyValidation,
602 ) -> Result<TopologyAdmissionStatus, TopologyError> {
603 if self.is_recovering()
604 || self.is_draining()
605 || self.try_live_local_process_authority_identity().ok() != Some(before)
606 {
607 return Err(TopologyError::Fenced);
608 }
609 let authority = self
610 .checkpoint_authority()
611 .map_err(|error| TopologyError::Protocol(error.to_string()))?;
612 let assignments = self.snapshot.as_ref().ok_or_else(|| {
613 TopologyError::Protocol("preparation has no configured assignment authority".into())
614 })?;
615 let (_, plan, _, _) = authority.topology_preparation_input(operation_id).await?;
616 let evidence = self
617 .read_local_process_authority_evidence()
618 .await
619 .map_err(|error| TopologyError::Conflict(error.to_string()))?;
620 if evidence.participant != before.participant
621 || evidence.process_term != before.process_term
622 || !evidence.adopted_assignment.matches_fence(&plan.assignment)
623 {
624 return Err(TopologyError::Fenced);
625 }
626 let status = authority
627 .certify_topology_participant(
628 assignments,
629 self.process_lease_authority.get().ok_or_else(|| {
630 TopologyError::Protocol("process lease authority is not installed".into())
631 })?,
632 operation_id,
633 expected_plan,
634 before,
635 plan.protocol_version,
636 compiled,
637 )
638 .await?;
639 if self.is_recovering()
640 || self.is_draining()
641 || self.try_live_local_process_authority_identity().ok() != Some(before)
642 || self
643 .checkpoint_assignment_fence(plan.assignment.assignment_version)
644 .as_ref()
645 != Some(&plan.assignment)
646 {
647 return Err(TopologyError::Fenced);
648 }
649 Ok(status)
650 }
651
652 pub async fn begin_topology_checkpoint_cut(
658 &self,
659 proof: &LeaderProof,
660 operation_id: TopologyOperationId,
661 expected_plan: &TopologyPlanRef,
662 inventory: CheckpointArtifactInventory,
663 ) -> Result<TopologyAdmissionStatus, TopologyError> {
664 if !self.process_lease_is_live() || !self.proof_is_live(proof) {
665 return Err(TopologyError::Fenced);
666 }
667 let authority = self
668 .checkpoint_authority()
669 .map_err(|error| TopologyError::Protocol(error.to_string()))?;
670 let assignments = self.snapshot.as_ref().ok_or_else(|| {
671 TopologyError::Protocol("topology cut has no configured assignment store".into())
672 })?;
673 let status = authority
674 .begin_topology_checkpoint_cut(
675 proof,
676 assignments,
677 self.process_lease_authority.get().ok_or_else(|| {
678 TopologyError::Protocol("process lease authority is not installed".into())
679 })?,
680 operation_id,
681 expected_plan,
682 inventory,
683 )
684 .await?;
685 if !self.process_lease_is_live() || !self.proof_is_live(proof) {
686 return Err(TopologyError::Fenced);
687 }
688 Ok(status)
689 }
690
691 pub async fn complete_topology_checkpoint_cut(
698 &self,
699 proof: &LeaderProof,
700 attempt: CheckpointAttempt,
701 ) -> Result<TopologyAdmissionStatus, TopologyError> {
702 let before = self
703 .try_live_local_process_authority_identity()
704 .map_err(|_| TopologyError::Fenced)?;
705 let authority = self
706 .checkpoint_authority()
707 .map_err(|error| TopologyError::Protocol(error.to_string()))?;
708 let participant = before.participant;
709 let status = authority
710 .complete_topology_checkpoint_cut(proof, attempt, participant)
711 .await?;
712 if self
713 .try_live_local_process_authority_identity()
714 .ok()
715 .as_ref()
716 != Some(&before)
717 {
718 return Err(TopologyError::Fenced);
719 }
720 Ok(status)
721 }
722}