laminar_connectors/
lib.rs

1//! # `LaminarDB` Connectors
2//!
3//! External system connectors and the Connector SDK for streaming data
4//! in and out of `LaminarDB`.
5//!
6//! ## Connector SDK
7//!
8//! The SDK provides traits and utilities for building connectors:
9//!
10//! - [`connector`] - Core traits (`SourceConnector`, `SinkConnector`)
11//! - [`serde`] - Serialization framework (JSON, CSV, Debezium)
12//! - [`registry`] - Factory pattern for connector instantiation
13//! - `testing` - Mock connectors and test utilities (feature-gated)
14//!
15//! ## Architecture
16//!
17//! ```text
18//! Ring 0: Hot Path
19//!   Source<T>::push_arrow() <-- deserialized RecordBatch
20//!   Subscription::poll()   --> RecordBatch for serialization
21//!
22//! Ring 1: Connectors
23//!   SourceConnector(poll) -> Serde(deser) -> push_arrow
24//!   SinkConnector(write)  <- Serde(ser)   <- subscription(poll)
25//! ```
26
27#![deny(missing_docs)]
28#![warn(clippy::all, clippy::pedantic)]
29#![allow(clippy::module_name_repetitions)]
30// Connectors are Ring 1 (cold path): std HashMap/HashSet are acceptable
31// throughout config, registry, schema, checkpoint, and CDC modules.
32#![allow(clippy::disallowed_types)]
33// Common test patterns that are acceptable
34#![cfg_attr(
35    test,
36    allow(
37        clippy::field_reassign_with_default,
38        clippy::float_cmp,
39        clippy::manual_let_else,
40        clippy::needless_return,
41        clippy::unreadable_literal,
42        clippy::approx_constant,
43        clippy::cast_possible_truncation,
44        clippy::cast_possible_wrap,
45        clippy::cast_sign_loss,
46        clippy::cast_precision_loss,
47        clippy::no_effect_underscore_binding,
48        unused_mut
49    )
50)]
51
52// ── Connector SDK ──
53
54/// Connector error types.
55pub mod error;
56
57#[macro_use]
58mod macros;
59
60/// Connector configuration types.
61pub mod config;
62
63/// Core connector traits (`SourceConnector`, `SinkConnector`).
64pub mod connector;
65
66/// Connector checkpoint types.
67pub mod checkpoint;
68
69/// Connector health status types.
70pub mod health;
71
72/// Connector metrics types.
73pub mod metrics;
74
75/// Record serialization and deserialization framework.
76pub mod serde;
77
78/// Schema inference, resolution, and evolution framework.
79pub mod schema;
80
81/// Connector registry with factory pattern.
82pub mod registry;
83
84/// Testing utilities (mock connectors, helpers).
85#[cfg(any(test, feature = "testing"))]
86pub mod testing;
87
88// ── Existing Modules ──
89
90/// Kafka source and sink connectors.
91#[cfg(feature = "kafka")]
92pub mod kafka;
93
94/// Change Data Capture connectors for databases.
95pub mod cdc;
96
97/// PostgreSQL sink connector.
98#[cfg(feature = "postgres-sink")]
99pub mod postgres;
100
101/// Lookup table support for enrichment joins.
102pub mod lookup;
103
104/// Lakehouse connectors (Delta Lake, Iceberg).
105pub mod lakehouse;
106
107/// Cloud storage infrastructure (credential resolution, validation, secret masking).
108pub mod storage;
109
110/// Reference table source trait and refresh modes.
111pub mod reference;
112
113/// WebSocket source and sink connectors.
114#[cfg(feature = "websocket")]
115pub mod websocket;
116
117/// AutoLoader-style file source and sink connectors.
118#[cfg(feature = "files")]
119#[allow(
120    clippy::similar_names,
121    clippy::cast_possible_truncation,
122    clippy::must_use_candidate,
123    clippy::items_after_statements,
124    clippy::manual_let_else,
125    clippy::missing_fields_in_debug,
126    clippy::unnecessary_wraps,
127    clippy::case_sensitive_file_extension_comparisons,
128    clippy::map_unwrap_or,
129    clippy::unnecessary_literal_bound,
130    clippy::too_many_lines
131)]
132pub mod files;
laminar_connectors/lib.rs

laminar_connectors/
lib.rs
