laminar_core/io_uring/
ring.rs

1//! `io_uring` ring creation and configuration.
2//!
3//! Provides functions to create optimized `io_uring` instances with various modes:
4//! - SQPOLL: Kernel polling thread eliminates syscalls
5//! - IOPOLL: Poll completions from `NVMe` device queue
6//! - Combined: Maximum performance for `NVMe` storage
7
8use io_uring::squeue::Entry;
9use io_uring::IoUring;
10
11use super::config::{IoUringConfig, RingMode};
12use super::error::IoUringError;
13
14/// Type alias for standard `IoUring` with default entry types.
15pub type StandardIoUring = IoUring<Entry, io_uring::cqueue::Entry>;
16
17/// Wrapper around `io_uring` with mode information.
18pub struct IoUringRing {
19    ring: StandardIoUring,
20    mode: RingMode,
21    entries: u32,
22}
23
24impl std::fmt::Debug for IoUringRing {
25    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
26        f.debug_struct("IoUringRing")
27            .field("mode", &self.mode)
28            .field("entries", &self.entries)
29            .finish_non_exhaustive()
30    }
31}
32
33impl IoUringRing {
34    /// Create a new ring from the configuration.
35    ///
36    /// # Errors
37    ///
38    /// Returns an error if the ring cannot be created.
39    pub fn new(config: &IoUringConfig) -> Result<Self, IoUringError> {
40        config.validate()?;
41
42        let ring = match config.mode {
43            RingMode::Standard => create_standard_ring(config)?,
44            RingMode::SqPoll => create_sqpoll_ring(config)?,
45            RingMode::IoPoll => create_iopoll_ring_internal(config)?,
46            RingMode::SqPollIoPoll => create_sqpoll_iopoll_ring(config)?,
47        };
48
49        // Check for FEAT_NODROP: without this feature, the kernel may silently
50        // drop CQEs when the completion queue overflows, causing completed I/O
51        // operations to be lost without notification.
52        if !ring.params().is_feature_nodrop() {
53            tracing::warn!(
54                "io_uring: FEAT_NODROP not supported — CQE overflow may silently \
55                 drop completions. Consider upgrading to Linux 5.5+."
56            );
57        }
58
59        Ok(Self {
60            ring,
61            mode: config.mode,
62            entries: config.ring_entries,
63        })
64    }
65
66    /// Get a reference to the underlying `io_uring`.
67    #[must_use]
68    pub fn ring(&self) -> &StandardIoUring {
69        &self.ring
70    }
71
72    /// Get a mutable reference to the underlying `io_uring`.
73    #[must_use]
74    pub fn ring_mut(&mut self) -> &mut StandardIoUring {
75        &mut self.ring
76    }
77
78    /// Get the ring mode.
79    #[must_use]
80    pub const fn mode(&self) -> RingMode {
81        self.mode
82    }
83
84    /// Get the number of entries.
85    #[must_use]
86    pub const fn entries(&self) -> u32 {
87        self.entries
88    }
89
90    /// Check if SQPOLL is enabled.
91    #[must_use]
92    pub const fn uses_sqpoll(&self) -> bool {
93        self.mode.uses_sqpoll()
94    }
95
96    /// Check if IOPOLL is enabled.
97    #[must_use]
98    pub const fn uses_iopoll(&self) -> bool {
99        self.mode.uses_iopoll()
100    }
101}
102
103/// Create a standard `io_uring` ring.
104fn create_standard_ring(config: &IoUringConfig) -> Result<StandardIoUring, IoUringError> {
105    let mut builder = IoUring::builder();
106
107    if config.coop_taskrun {
108        builder.setup_coop_taskrun();
109    }
110
111    if config.single_issuer {
112        builder.setup_single_issuer();
113    }
114
115    builder
116        .build(config.ring_entries)
117        .map_err(IoUringError::RingCreation)
118}
119
120/// Create a SQPOLL-enabled ring.
121fn create_sqpoll_ring(config: &IoUringConfig) -> Result<StandardIoUring, IoUringError> {
122    let mut builder = IoUring::builder();
123
124    // Enable SQPOLL - kernel thread polls submission queue
125    builder.setup_sqpoll(config.sqpoll_idle_ms);
126
127    // Pin SQPOLL thread to specific CPU if requested
128    if let Some(cpu) = config.sqpoll_cpu {
129        builder.setup_sqpoll_cpu(cpu);
130    }
131
132    if config.coop_taskrun {
133        builder.setup_coop_taskrun();
134    }
135
136    if config.single_issuer {
137        builder.setup_single_issuer();
138    }
139
140    builder.build(config.ring_entries).map_err(|e| {
141        if e.raw_os_error() == Some(libc::EPERM) {
142            IoUringError::FeatureNotSupported {
143                feature: "SQPOLL".to_string(),
144                required_version: "5.11+ with CAP_SYS_NICE (or kernel.io_uring_group sysctl)"
145                    .to_string(),
146            }
147        } else {
148            IoUringError::RingCreation(e)
149        }
150    })
151}
152
153/// Create an IOPOLL-enabled ring.
154fn create_iopoll_ring_internal(config: &IoUringConfig) -> Result<StandardIoUring, IoUringError> {
155    let mut builder = IoUring::builder();
156
157    // Enable IOPOLL - poll completions from device
158    builder.setup_iopoll();
159
160    if config.coop_taskrun {
161        builder.setup_coop_taskrun();
162    }
163
164    if config.single_issuer {
165        builder.setup_single_issuer();
166    }
167
168    builder
169        .build(config.ring_entries)
170        .map_err(IoUringError::RingCreation)
171}
172
173/// Create a ring with both SQPOLL and IOPOLL.
174fn create_sqpoll_iopoll_ring(config: &IoUringConfig) -> Result<StandardIoUring, IoUringError> {
175    let mut builder = IoUring::builder();
176
177    // Enable both SQPOLL and IOPOLL
178    builder.setup_sqpoll(config.sqpoll_idle_ms);
179    builder.setup_iopoll();
180
181    if let Some(cpu) = config.sqpoll_cpu {
182        builder.setup_sqpoll_cpu(cpu);
183    }
184
185    if config.coop_taskrun {
186        builder.setup_coop_taskrun();
187    }
188
189    if config.single_issuer {
190        builder.setup_single_issuer();
191    }
192
193    builder.build(config.ring_entries).map_err(|e| {
194        if e.raw_os_error() == Some(libc::EPERM) {
195            IoUringError::FeatureNotSupported {
196                feature: "SQPOLL+IOPOLL".to_string(),
197                required_version: "5.11+ with CAP_SYS_NICE (or kernel.io_uring_group sysctl)"
198                    .to_string(),
199            }
200        } else {
201            IoUringError::RingCreation(e)
202        }
203    })
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209
210    #[test]
211    fn test_create_standard_ring() {
212        let config = IoUringConfig {
213            ring_entries: 32,
214            mode: RingMode::Standard,
215            ..Default::default()
216        };
217        let ring = IoUringRing::new(&config);
218        // May fail in CI or containers without io_uring support
219        if let Ok(r) = ring {
220            assert_eq!(r.mode(), RingMode::Standard);
221            assert_eq!(r.entries(), 32);
222            assert!(!r.uses_sqpoll());
223            assert!(!r.uses_iopoll());
224        }
225    }
226
227    #[test]
228    fn test_ring_wrapper() {
229        let config = IoUringConfig::default();
230        if let Ok(ring) = IoUringRing::new(&config) {
231            assert_eq!(ring.mode(), RingMode::Standard);
232            assert_eq!(ring.entries(), 256);
233        }
234    }
235}
laminar_core/io_uring/ring.rs

laminar_core/io_uring/
ring.rs
