laminar_storage/
lib.rs

1//! # `LaminarDB` Storage
2//!
3//! Durability layer for `LaminarDB` - WAL, checkpointing, and recovery.
4//!
5//! ## Module Overview
6//!
7//! - [`wal`]: Write-ahead log for durability and exactly-once semantics
8//! - [`checkpoint`]: Basic checkpointing for fast recovery
9//! - [`incremental`]: Incremental checkpointing
10//! - [`per_core_wal`]: Per-core WAL segments for thread-per-core architecture
11//!
12//! **Note:** Lakehouse sinks (Delta Lake, Iceberg) are in `laminar-connectors` crate,
13//! not here. This crate handles `LaminarDB`'s internal durability, not external storage formats.
14
15#![deny(missing_docs)]
16#![warn(clippy::all, clippy::pedantic)]
17#![allow(clippy::disallowed_types)] // cold path: all storage modules (WAL, checkpoint, recovery) are off hot path
18
19/// Write-ahead log implementation - WAL for durability and exactly-once semantics
20pub mod wal;
21
22/// Checkpointing for fast recovery
23pub mod checkpoint;
24
25/// Unified checkpoint manifest types
26pub mod checkpoint_manifest;
27
28/// Checkpoint persistence trait and filesystem store
29pub mod checkpoint_store;
30
31/// Checkpoint batching for S3 cost optimization
32pub mod checkpoint_batcher;
33
34/// S3 storage class tiering for cost optimization
35pub mod tiering;
36
37/// Ring 1 changelog drainer
38pub mod changelog_drainer;
39
40/// Incremental checkpointing - Directory-based checkpoint architecture
41pub mod incremental;
42
43/// Per-core WAL segments - Thread-per-core WAL for lock-free writes
44pub mod per_core_wal;
45
46/// Object store factory — builds S3, GCS, Azure, or local backends from URL schemes.
47pub mod object_store_factory;
48
49// Re-export key types
50pub use changelog_drainer::ChangelogDrainer;
51pub use checkpoint::checkpointer::{verify_integrity, Checkpointer, CheckpointerError};
52pub use checkpoint::layout::{
53    CheckpointId, CheckpointManifestV2, CheckpointPaths, OperatorSnapshotEntry,
54    PartitionSnapshotEntry, SourceOffsetEntry,
55};
56pub use checkpoint::source_offsets::{
57    DeterminismValidator, DeterminismWarning, FilePosition, GenericPosition, KafkaPartitionOffset,
58    KafkaPosition, MysqlCdcPosition, OperatorDescriptor, OperatorDeterminismWarning,
59    PostgresCdcPosition, RecoveryPlan, SourceId, SourceOffset, SourcePosition, WarningSeverity,
60};
61pub use checkpoint::{Checkpoint, CheckpointMetadata};
62pub use checkpoint_batcher::{decode_batch, BatchMetrics, BatchMetricsSnapshot, CheckpointBatcher};
63pub use checkpoint_manifest::{CheckpointManifest, ConnectorCheckpoint, OperatorCheckpoint};
64pub use checkpoint_store::{
65    CheckpointStore, CheckpointStoreError, FileSystemCheckpointStore, ObjectStoreCheckpointStore,
66    RecoveryReport, ValidationResult,
67};
68pub use tiering::{StorageClass, StorageTier, TieringPolicy};
69pub use wal::{WalEntry, WalError, WalPosition, WriteAheadLog};
70
71// Re-export incremental checkpoint types
72pub use incremental::{
73    validate_checkpoint, wal_size, CheckpointConfig, IncrementalCheckpointError,
74    IncrementalCheckpointManager, IncrementalCheckpointMetadata, RecoveredState, RecoveryConfig,
75    RecoveryManager, StateChangelogBuffer, StateChangelogEntry, StateOp,
76};
77
78// Re-export per-core WAL types
79pub use per_core_wal::{
80    recover_per_core, CheckpointCoordinator, CoreWalWriter, PerCoreRecoveredState,
81    PerCoreRecoveryManager, PerCoreWalConfig, PerCoreWalEntry, PerCoreWalError, PerCoreWalManager,
82    PerCoreWalReader, SegmentStats, WalOperation,
83};
laminar_storage/lib.rs

laminar_storage/
lib.rs
