summaryrefslogtreecommitdiff
path: root/rust/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2025-09-30 19:12:49 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2025-09-30 19:12:49 -0700
commitf4e0ff7e45c30f4665cfbbe2f0538e9c5789bebc (patch)
tree6703541ef1fec9d20dafc751d2e971879c429884 /rust/kernel
parentae28ed4578e6d5a481e39c5a9827f27048661fdd (diff)
parentf3f6b3664302e16ef1c6b91034a72df5564d6b8a (diff)
Merge tag 'rust-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/ojeda/linux
Pull rust updates from Miguel Ojeda: "Toolchain and infrastructure: - Derive 'Zeroable' for all structs and unions generated by 'bindgen' where possible and corresponding cleanups. To do so, add the 'pin-init' crate as a dependency to 'bindings' and 'uapi'. It also includes its first use in the 'cpufreq' module, with more to come in the next cycle. - Add warning to the 'rustdoc' target to detect broken 'srctree/' links and fix existing cases. - Remove support for unused (since v6.16) host '#[test]'s, simplifying the 'rusttest' target. Tests should generally run within KUnit. 'kernel' crate: - Add 'ptr' module with a new 'Alignment' type, which is always a power of two and is used to validate that a given value is a valid alignment and to perform masking and alignment operations: // Checked at build time. assert_eq!(Alignment::new::<16>().as_usize(), 16); // Checked at runtime. assert_eq!(Alignment::new_checked(15), None); assert_eq!(Alignment::of::<u8>().log2(), 0); assert_eq!(0x25u8.align_down(Alignment::new::<0x10>()), 0x20); assert_eq!(0x5u8.align_up(Alignment::new::<0x10>()), Some(0x10)); assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None); It also includes its first use in Nova. - Add 'core::mem::{align,size}_of{,_val}' to the prelude, matching Rust 1.80.0. - Keep going with the steps on our migration to the standard library 'core::ffi::CStr' type (use 'kernel::{fmt, prelude::fmt!}' and use upstream method names). - 'error' module: improve 'Error::from_errno' and 'to_result' documentation, including examples/tests. - 'sync' module: extend 'aref' submodule documentation now that it exists, and more updates to complete the ongoing move of 'ARef' and 'AlwaysRefCounted' to 'sync::aref'. - 'list' module: add an example/test for 'ListLinksSelfPtr' usage. - 'alloc' module: - Implement 'Box::pin_slice()', which constructs a pinned slice of elements. - Provide information about the minimum alignment guarantees of 'Kmalloc', 'Vmalloc' and 'KVmalloc'. - Take minimum alignment guarantees of allocators for 'ForeignOwnable' into account. - Remove the 'allocator_test' (including 'Cmalloc'). - Add doctest for 'Vec::as_slice()'. - Constify various methods. - 'time' module: - Add methods on 'HrTimer' that can only be called with exclusive access to an unarmed timer, or from timer callback context. - Add arithmetic operations to 'Instant' and 'Delta'. - Add a few convenience and access methods to 'HrTimer' and 'Instant'. 'macros' crate: - Reduce collections in 'quote!' macro. And a few other cleanups and improvements" * tag 'rust-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/ojeda/linux: (58 commits) gpu: nova-core: use Alignment for alignment-related operations rust: add `Alignment` type rust: macros: reduce collections in `quote!` macro rust: acpi: use `core::ffi::CStr` method names rust: of: use `core::ffi::CStr` method names rust: net: use `core::ffi::CStr` method names rust: miscdevice: use `core::ffi::CStr` method names rust: kunit: use `core::ffi::CStr` method names rust: firmware: use `core::ffi::CStr` method names rust: drm: use `core::ffi::CStr` method names rust: cpufreq: use `core::ffi::CStr` method names rust: configfs: use `core::ffi::CStr` method names rust: auxiliary: use `core::ffi::CStr` method names drm/panic: use `core::ffi::CStr` method names rust: device: use `kernel::{fmt,prelude::fmt!}` rust: sync: use `kernel::{fmt,prelude::fmt!}` rust: seq_file: use `kernel::{fmt,prelude::fmt!}` rust: kunit: use `kernel::{fmt,prelude::fmt!}` rust: file: use `kernel::{fmt,prelude::fmt!}` rust: device: use `kernel::{fmt,prelude::fmt!}` ...
Diffstat (limited to 'rust/kernel')
-rw-r--r--rust/kernel/acpi.rs7
-rw-r--r--rust/kernel/alloc.rs15
-rw-r--r--rust/kernel/alloc/allocator.rs8
-rw-r--r--rust/kernel/alloc/allocator_test.rs124
-rw-r--r--rust/kernel/alloc/kbox.rs92
-rw-r--r--rust/kernel/alloc/kvec.rs24
-rw-r--r--rust/kernel/alloc/kvec/errors.rs2
-rw-r--r--rust/kernel/alloc/layout.rs2
-rw-r--r--rust/kernel/auxiliary.rs4
-rw-r--r--rust/kernel/block/mq.rs2
-rw-r--r--rust/kernel/block/mq/gen_disk.rs4
-rw-r--r--rust/kernel/block/mq/raw_writer.rs3
-rw-r--r--rust/kernel/configfs.rs4
-rw-r--r--rust/kernel/cpu.rs1
-rw-r--r--rust/kernel/cpufreq.rs6
-rw-r--r--rust/kernel/device.rs6
-rw-r--r--rust/kernel/device/property.rs23
-rw-r--r--rust/kernel/dma.rs2
-rw-r--r--rust/kernel/drm/device.rs6
-rw-r--r--rust/kernel/drm/driver.rs2
-rw-r--r--rust/kernel/drm/file.rs2
-rw-r--r--rust/kernel/drm/gem/mod.rs2
-rw-r--r--rust/kernel/drm/ioctl.rs2
-rw-r--r--rust/kernel/error.rs66
-rw-r--r--rust/kernel/firmware.rs2
-rw-r--r--rust/kernel/fs/file.rs5
-rw-r--r--rust/kernel/kunit.rs14
-rw-r--r--rust/kernel/lib.rs5
-rw-r--r--rust/kernel/list.rs120
-rw-r--r--rust/kernel/miscdevice.rs2
-rw-r--r--rust/kernel/net/phy.rs2
-rw-r--r--rust/kernel/of.rs2
-rw-r--r--rust/kernel/prelude.rs5
-rw-r--r--rust/kernel/ptr.rs228
-rw-r--r--rust/kernel/seq_file.rs6
-rw-r--r--rust/kernel/sync/arc.rs8
-rw-r--r--rust/kernel/sync/aref.rs17
-rw-r--r--rust/kernel/task.rs7
-rw-r--r--rust/kernel/time.rs163
-rw-r--r--rust/kernel/time/hrtimer.rs152
-rw-r--r--rust/kernel/time/hrtimer/arc.rs9
-rw-r--r--rust/kernel/time/hrtimer/pin.rs9
-rw-r--r--rust/kernel/time/hrtimer/pin_mut.rs12
-rw-r--r--rust/kernel/time/hrtimer/tbox.rs9
44 files changed, 958 insertions, 228 deletions
diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs
index 7ae317368b00..37e1161c1298 100644
--- a/rust/kernel/acpi.rs
+++ b/rust/kernel/acpi.rs
@@ -37,11 +37,8 @@ impl DeviceId {
/// Create a new device id from an ACPI 'id' string.
#[inline(always)]
pub const fn new(id: &'static CStr) -> Self {
- build_assert!(
- id.len_with_nul() <= Self::ACPI_ID_LEN,
- "ID exceeds 16 bytes"
- );
- let src = id.as_bytes_with_nul();
+ let src = id.to_bytes_with_nul();
+ build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes");
// Replace with `bindings::acpi_device_id::default()` once stabilized for `const`.
// SAFETY: FFI type is valid to be zero-initialized.
let mut acpi: bindings::acpi_device_id = unsafe { core::mem::zeroed() };
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
index a2c49e5494d3..9c154209423c 100644
--- a/rust/kernel/alloc.rs
+++ b/rust/kernel/alloc.rs
@@ -2,18 +2,11 @@
//! Implementation of the kernel's memory allocation infrastructure.
-#[cfg(not(any(test, testlib)))]
pub mod allocator;
pub mod kbox;
pub mod kvec;
pub mod layout;
-#[cfg(any(test, testlib))]
-pub mod allocator_test;
-
-#[cfg(any(test, testlib))]
-pub use self::allocator_test as allocator;
-
pub use self::kbox::Box;
pub use self::kbox::KBox;
pub use self::kbox::KVBox;
@@ -137,6 +130,14 @@ pub mod flags {
/// - Implementers must ensure that all trait functions abide by the guarantees documented in the
/// `# Guarantees` sections.
pub unsafe trait Allocator {
+ /// The minimum alignment satisfied by all allocations from this allocator.
+ ///
+ /// # Guarantees
+ ///
+ /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if
+ /// the requested layout has a smaller alignment.
+ const MIN_ALIGN: usize;
+
/// Allocate memory based on `layout` and `flags`.
///
/// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout
diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
index 2692cf90c948..869d9fd69527 100644
--- a/rust/kernel/alloc/allocator.rs
+++ b/rust/kernel/alloc/allocator.rs
@@ -17,6 +17,8 @@ use crate::alloc::{AllocError, Allocator};
use crate::bindings;
use crate::pr_warn;
+const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN;
+
/// The contiguous kernel allocator.
///
/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also
@@ -128,6 +130,8 @@ impl Kmalloc {
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for Kmalloc {
+ const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
+
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
@@ -147,6 +151,8 @@ unsafe impl Allocator for Kmalloc {
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for Vmalloc {
+ const MIN_ALIGN: usize = kernel::page::PAGE_SIZE;
+
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
@@ -171,6 +177,8 @@ unsafe impl Allocator for Vmalloc {
// - passing a pointer to a valid memory allocation is OK,
// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
unsafe impl Allocator for KVmalloc {
+ const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
+
#[inline]
unsafe fn realloc(
ptr: Option<NonNull<u8>>,
diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs
deleted file mode 100644
index 90dd987d40e4..000000000000
--- a/rust/kernel/alloc/allocator_test.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users
-//! of those types (e.g. `CString`) use kernel allocators for instantiation.
-//!
-//! In order to allow userspace test cases to make use of such types as well, implement the
-//! `Cmalloc` allocator within the `allocator_test` module and type alias all kernel allocators to
-//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend.
-
-#![allow(missing_docs)]
-
-use super::{flags::*, AllocError, Allocator, Flags};
-use core::alloc::Layout;
-use core::cmp;
-use core::ptr;
-use core::ptr::NonNull;
-
-/// The userspace allocator based on libc.
-pub struct Cmalloc;
-
-pub type Kmalloc = Cmalloc;
-pub type Vmalloc = Kmalloc;
-pub type KVmalloc = Kmalloc;
-
-impl Cmalloc {
- /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of
- /// `layout`.
- pub fn aligned_layout(layout: Layout) -> Layout {
- // Note that `layout.size()` (after padding) is guaranteed to be a multiple of
- // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return
- // a properly aligned object (see comments in `kmalloc()` for more information).
- layout.pad_to_align()
- }
-}
-
-extern "C" {
- #[link_name = "aligned_alloc"]
- fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void;
-
- #[link_name = "free"]
- fn libc_free(ptr: *mut crate::ffi::c_void);
-}
-
-// SAFETY:
-// - memory remains valid until it is explicitly freed,
-// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK,
-// - `realloc` provides the guarantees as provided in the `# Guarantees` section.
-unsafe impl Allocator for Cmalloc {
- unsafe fn realloc(
- ptr: Option<NonNull<u8>>,
- layout: Layout,
- old_layout: Layout,
- flags: Flags,
- ) -> Result<NonNull<[u8]>, AllocError> {
- let src = match ptr {
- Some(src) => {
- if old_layout.size() == 0 {
- ptr::null_mut()
- } else {
- src.as_ptr()
- }
- }
- None => ptr::null_mut(),
- };
-
- if layout.size() == 0 {
- // SAFETY: `src` is either NULL or was previously allocated with this `Allocator`
- unsafe { libc_free(src.cast()) };
-
- return Ok(NonNull::slice_from_raw_parts(
- crate::alloc::dangling_from_layout(layout),
- 0,
- ));
- }
-
- // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`:
- //
- // > The value of alignment shall be a valid alignment supported by the implementation
- // [...].
- //
- // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE
- // 1003.1-2001) defines `posix_memalign`:
- //
- // > The value of alignment shall be a power of two multiple of sizeof (void *).
- //
- // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time
- // of writing, this is known to be the case on macOS (but not in glibc).
- //
- // Satisfy the stricter requirement to avoid spurious test failures on some platforms.
- let min_align = core::mem::size_of::<*const crate::ffi::c_void>();
- let layout = layout.align_to(min_align).map_err(|_| AllocError)?;
- let layout = layout.pad_to_align();
-
- // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or
- // exceeds the given size and alignment requirements.
- let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) }.cast::<u8>();
- let dst = NonNull::new(dst).ok_or(AllocError)?;
-
- if flags.contains(__GFP_ZERO) {
- // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new`
- // guarantee that `dst` points to memory of at least `layout.size()` bytes.
- unsafe { dst.as_ptr().write_bytes(0, layout.size()) };
- }
-
- if !src.is_null() {
- // SAFETY:
- // - `src` has previously been allocated with this `Allocator`; `dst` has just been
- // newly allocated, hence the memory regions do not overlap.
- // - both` src` and `dst` are properly aligned and valid for reads and writes
- unsafe {
- ptr::copy_nonoverlapping(
- src,
- dst.as_ptr(),
- cmp::min(layout.size(), old_layout.size()),
- )
- };
- }
-
- // SAFETY: `src` is either NULL or was previously allocated with this `Allocator`
- unsafe { libc_free(src.cast()) };
-
- Ok(NonNull::slice_from_raw_parts(dst, layout.size()))
- }
-}
diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs
index 856d05aa60f1..27c4b5a9b61d 100644
--- a/rust/kernel/alloc/kbox.rs
+++ b/rust/kernel/alloc/kbox.rs
@@ -7,7 +7,6 @@ use super::allocator::{KVmalloc, Kmalloc, Vmalloc};
use super::{AllocError, Allocator, Flags};
use core::alloc::Layout;
use core::borrow::{Borrow, BorrowMut};
-use core::fmt;
use core::marker::PhantomData;
use core::mem::ManuallyDrop;
use core::mem::MaybeUninit;
@@ -17,6 +16,7 @@ use core::ptr::NonNull;
use core::result::Result;
use crate::ffi::c_void;
+use crate::fmt;
use crate::init::InPlaceInit;
use crate::types::ForeignOwnable;
use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption};
@@ -290,6 +290,83 @@ where
Ok(Self::new(x, flags)?.into())
}
+ /// Construct a pinned slice of elements `Pin<Box<[T], A>>`.
+ ///
+ /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks
+ /// or mutexes.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::sync::{new_spinlock, SpinLock};
+ ///
+ /// struct Inner {
+ /// a: u32,
+ /// b: u32,
+ /// }
+ ///
+ /// #[pin_data]
+ /// struct Example {
+ /// c: u32,
+ /// #[pin]
+ /// d: SpinLock<Inner>,
+ /// }
+ ///
+ /// impl Example {
+ /// fn new() -> impl PinInit<Self, Error> {
+ /// try_pin_init!(Self {
+ /// c: 10,
+ /// d <- new_spinlock!(Inner { a: 20, b: 30 }),
+ /// })
+ /// }
+ /// }
+ ///
+ /// // Allocate a boxed slice of 10 `Example`s.
+ /// let s = KBox::pin_slice(
+ /// | _i | Example::new(),
+ /// 10,
+ /// GFP_KERNEL
+ /// )?;
+ ///
+ /// assert_eq!(s[5].c, 10);
+ /// assert_eq!(s[3].d.lock().a, 20);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn pin_slice<Func, Item, E>(
+ mut init: Func,
+ len: usize,
+ flags: Flags,
+ ) -> Result<Pin<Box<[T], A>>, E>
+ where
+ Func: FnMut(usize) -> Item,
+ Item: PinInit<T, E>,
+ E: From<AllocError>,
+ {
+ let mut buffer = super::Vec::<T, A>::with_capacity(len, flags)?;
+ for i in 0..len {
+ let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast();
+ // SAFETY:
+ // - `ptr` is a valid pointer to uninitialized memory.
+ // - `ptr` is not used if an error is returned.
+ // - `ptr` won't be moved until it is dropped, i.e. it is pinned.
+ unsafe { init(i).__pinned_init(ptr)? };
+
+ // SAFETY:
+ // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to
+ // `with_capacity()` above.
+ // - The new value at index buffer.len() + 1 is the only element being added here, and
+ // it has been initialized above by `init(i).__pinned_init(ptr)`.
+ unsafe { buffer.inc_len(1) };
+ }
+
+ let (ptr, _, _) = buffer.into_raw_parts();
+ let slice = core::ptr::slice_from_raw_parts_mut(ptr, len);
+
+ // SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid
+ // `[T]`.
+ Ok(Pin::from(unsafe { Box::from_raw(slice) }))
+ }
+
/// Convert a [`Box<T,A>`] to a [`Pin<Box<T,A>>`]. If `T` does not implement
/// [`Unpin`], then `x` will be pinned in memory and can't be moved.
pub fn into_pin(this: Self) -> Pin<Self> {
@@ -401,12 +478,17 @@ where
}
// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
-// pointer to `T`.
+// pointer to `T` allocated by `A`.
unsafe impl<T: 'static, A> ForeignOwnable for Box<T, A>
where
A: Allocator,
{
- const FOREIGN_ALIGN: usize = core::mem::align_of::<T>();
+ const FOREIGN_ALIGN: usize = if core::mem::align_of::<T>() < A::MIN_ALIGN {
+ A::MIN_ALIGN
+ } else {
+ core::mem::align_of::<T>()
+ };
+
type Borrowed<'a> = &'a T;
type BorrowedMut<'a> = &'a mut T;
@@ -435,12 +517,12 @@ where
}
// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
-// pointer to `T`.
+// pointer to `T` allocated by `A`.
unsafe impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>>
where
A: Allocator,
{
- const FOREIGN_ALIGN: usize = core::mem::align_of::<T>();
+ const FOREIGN_ALIGN: usize = <Box<T, A> as ForeignOwnable>::FOREIGN_ALIGN;
type Borrowed<'a> = Pin<&'a T>;
type BorrowedMut<'a> = Pin<&'a mut T>;
diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs
index 3c72e0bdddb8..dfc101e03f35 100644
--- a/rust/kernel/alloc/kvec.rs
+++ b/rust/kernel/alloc/kvec.rs
@@ -7,9 +7,9 @@ use super::{
layout::ArrayLayout,
AllocError, Allocator, Box, Flags,
};
+use crate::fmt;
use core::{
borrow::{Borrow, BorrowMut},
- fmt,
marker::PhantomData,
mem::{ManuallyDrop, MaybeUninit},
ops::Deref,
@@ -175,7 +175,7 @@ where
/// Returns the number of elements that can be stored within the vector without allocating
/// additional memory.
- pub fn capacity(&self) -> usize {
+ pub const fn capacity(&self) -> usize {
if const { Self::is_zst() } {
usize::MAX
} else {
@@ -185,7 +185,7 @@ where
/// Returns the number of elements stored within the vector.
#[inline]
- pub fn len(&self) -> usize {
+ pub const fn len(&self) -> usize {
self.len
}
@@ -196,7 +196,7 @@ where
/// - `additional` must be less than or equal to `self.capacity - self.len`.
/// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized.
#[inline]
- pub unsafe fn inc_len(&mut self, additional: usize) {
+ pub const unsafe fn inc_len(&mut self, additional: usize) {
// Guaranteed by the type invariant to never underflow.
debug_assert!(additional <= self.capacity() - self.len());
// INVARIANT: By the safety requirements of this method this represents the exact number of
@@ -224,6 +224,16 @@ where
}
/// Returns a slice of the entire vector.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = KVec::new();
+ /// v.push(1, GFP_KERNEL)?;
+ /// v.push(2, GFP_KERNEL)?;
+ /// assert_eq!(v.as_slice(), &[1, 2]);
+ /// # Ok::<(), Error>(())
+ /// ```
#[inline]
pub fn as_slice(&self) -> &[T] {
self
@@ -245,7 +255,7 @@ where
/// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw
/// pointer.
#[inline]
- pub fn as_ptr(&self) -> *const T {
+ pub const fn as_ptr(&self) -> *const T {
self.ptr.as_ptr()
}
@@ -261,7 +271,7 @@ where
/// assert!(!v.is_empty());
/// ```
#[inline]
- pub fn is_empty(&self) -> bool {
+ pub const fn is_empty(&self) -> bool {
self.len() == 0
}
@@ -1294,7 +1304,7 @@ impl<'vec, T> Drop for DrainAll<'vec, T> {
}
}
-#[macros::kunit_tests(rust_kvec_kunit)]
+#[macros::kunit_tests(rust_kvec)]
mod tests {
use super::*;
use crate::prelude::*;
diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs
index 348b8d27e102..21a920a4b09b 100644
--- a/rust/kernel/alloc/kvec/errors.rs
+++ b/rust/kernel/alloc/kvec/errors.rs
@@ -2,7 +2,7 @@
//! Errors for the [`Vec`] type.
-use core::fmt::{self, Debug, Formatter};
+use kernel::fmt::{self, Debug, Formatter};
use kernel::prelude::*;
/// Error type for [`Vec::push_within_capacity`].
diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs
index 93ed514f7cc7..52cbf61c4539 100644
--- a/rust/kernel/alloc/layout.rs
+++ b/rust/kernel/alloc/layout.rs
@@ -80,7 +80,7 @@ impl<T> ArrayLayout<T> {
/// # Safety
///
/// `len` must be a value, for which `len * size_of::<T>() <= isize::MAX` is true.
- pub unsafe fn new_unchecked(len: usize) -> Self {
+ pub const unsafe fn new_unchecked(len: usize) -> Self {
// INVARIANT: By the safety requirements of this function
// `len * size_of::<T>() <= isize::MAX`.
Self {
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
index 4749fb6bffef..58be09871397 100644
--- a/rust/kernel/auxiliary.rs
+++ b/rust/kernel/auxiliary.rs
@@ -105,8 +105,8 @@ pub struct DeviceId(bindings::auxiliary_device_id);
impl DeviceId {
/// Create a new [`DeviceId`] from name.
pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
- let name = name.as_bytes_with_nul();
- let modname = modname.as_bytes_with_nul();
+ let name = name.to_bytes_with_nul();
+ let modname = modname.to_bytes_with_nul();
// TODO: Replace with `bindings::auxiliary_device_id::default()` once stabilized for
// `const`.
diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs
index 831445d37181..61ea35bba7d5 100644
--- a/rust/kernel/block/mq.rs
+++ b/rust/kernel/block/mq.rs
@@ -82,7 +82,7 @@
//! Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?;
//! let mut disk = gen_disk::GenDiskBuilder::new()
//! .capacity_sectors(4096)
-//! .build(format_args!("myblk"), tagset)?;
+//! .build(fmt!("myblk"), tagset)?;
//!
//! # Ok::<(), kernel::error::Error>(())
//! ```
diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs
index cd54cd64ea88..be92d0e5f031 100644
--- a/rust/kernel/block/mq/gen_disk.rs
+++ b/rust/kernel/block/mq/gen_disk.rs
@@ -3,12 +3,12 @@
//! Generic disk abstraction.
//!
//! C header: [`include/linux/blkdev.h`](srctree/include/linux/blkdev.h)
-//! C header: [`include/linux/blk_mq.h`](srctree/include/linux/blk_mq.h)
+//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use crate::block::mq::{raw_writer::RawWriter, Operations, TagSet};
+use crate::fmt::{self, Write};
use crate::{bindings, error::from_err_ptr, error::Result, sync::Arc};
use crate::{error, static_lock_class};
-use core::fmt::{self, Write};
/// A builder for [`GenDisk`].
///
diff --git a/rust/kernel/block/mq/raw_writer.rs b/rust/kernel/block/mq/raw_writer.rs
index 7e2159e4f6a6..d311e24e2595 100644
--- a/rust/kernel/block/mq/raw_writer.rs
+++ b/rust/kernel/block/mq/raw_writer.rs
@@ -1,8 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
-use core::fmt::{self, Write};
-
use crate::error::Result;
+use crate::fmt::{self, Write};
use crate::prelude::EINVAL;
/// A mutable reference to a byte buffer where a string can be written into.
diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs
index 2736b798cdc6..9fb5ef825e41 100644
--- a/rust/kernel/configfs.rs
+++ b/rust/kernel/configfs.rs
@@ -263,7 +263,7 @@ impl<Data> Group<Data> {
try_pin_init!(Self {
group <- pin_init::init_zeroed().chain(|v: &mut Opaque<bindings::config_group>| {
let place = v.get();
- let name = name.as_bytes_with_nul().as_ptr();
+ let name = name.to_bytes_with_nul().as_ptr();
// SAFETY: It is safe to initialize a group once it has been zeroed.
unsafe {
bindings::config_group_init_type_name(place, name.cast(), item_type.as_ptr())
@@ -613,7 +613,7 @@ where
pub const fn new(name: &'static CStr) -> Self {
Self {
attribute: Opaque::new(bindings::configfs_attribute {
- ca_name: name.as_char_ptr(),
+ ca_name: crate::str::as_char_ptr_in_const_context(name),
ca_owner: core::ptr::null_mut(),
ca_mode: 0o660,
show: Some(Self::show),
diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs
index 5de730c8d817..cb6c0338ef5a 100644
--- a/rust/kernel/cpu.rs
+++ b/rust/kernel/cpu.rs
@@ -109,6 +109,7 @@ impl CpuId {
/// unexpectedly due to preemption or CPU migration. It should only be
/// used when the context ensures that the task remains on the same CPU
/// or the users could use a stale (yet valid) CPU ID.
+ #[inline]
pub fn current() -> Self {
// SAFETY: raw_smp_processor_id() always returns a valid CPU ID.
unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) }
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index afc15e72a7c3..86c02e81729e 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -27,7 +27,6 @@ use crate::clk::Clk;
use core::{
cell::UnsafeCell,
marker::PhantomData,
- mem::MaybeUninit,
ops::{Deref, DerefMut},
pin::Pin,
ptr,
@@ -1013,12 +1012,11 @@ impl<T: Driver> Registration<T> {
} else {
None
},
- // SAFETY: All zeros is a valid value for `bindings::cpufreq_driver`.
- ..unsafe { MaybeUninit::zeroed().assume_init() }
+ ..pin_init::zeroed()
};
const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] {
- let src = name.as_bytes_with_nul();
+ let src = name.to_bytes_with_nul();
let mut dst = [0; CPUFREQ_NAME_LEN];
build_assert!(src.len() <= CPUFREQ_NAME_LEN);
diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs
index a1db49eb159a..9cefa0810e0b 100644
--- a/rust/kernel/device.rs
+++ b/rust/kernel/device.rs
@@ -5,10 +5,10 @@
//! C header: [`include/linux/device.h`](srctree/include/linux/device.h)
use crate::{
- bindings,
+ bindings, fmt,
types::{ARef, ForeignOwnable, Opaque},
};
-use core::{fmt, marker::PhantomData, ptr};
+use core::{marker::PhantomData, ptr};
#[cfg(CONFIG_PRINTK)]
use crate::c_str;
@@ -596,7 +596,7 @@ macro_rules! impl_device_context_into_aref {
macro_rules! dev_printk {
($method:ident, $dev:expr, $($f:tt)*) => {
{
- ($dev).$method(::core::format_args!($($f)*));
+ ($dev).$method($crate::prelude::fmt!($($f)*));
}
}
}
diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs
index 49ee12a906db..3a332a8c53a9 100644
--- a/rust/kernel/device/property.rs
+++ b/rust/kernel/device/property.rs
@@ -11,6 +11,7 @@ use crate::{
alloc::KVec,
bindings,
error::{to_result, Result},
+ fmt,
prelude::*,
str::{CStr, CString},
types::{ARef, Opaque},
@@ -68,16 +69,16 @@ impl FwNode {
unsafe { bindings::is_of_node(self.as_raw()) }
}
- /// Returns an object that implements [`Display`](core::fmt::Display) for
+ /// Returns an object that implements [`Display`](fmt::Display) for
/// printing the name of a node.
///
/// This is an alternative to the default `Display` implementation, which
/// prints the full path.
- pub fn display_name(&self) -> impl core::fmt::Display + '_ {
+ pub fn display_name(&self) -> impl fmt::Display + '_ {
struct FwNodeDisplayName<'a>(&'a FwNode);
- impl core::fmt::Display for FwNodeDisplayName<'_> {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ impl fmt::Display for FwNodeDisplayName<'_> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// SAFETY: `self` is valid by its type invariant.
let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) };
if name.is_null() {
@@ -87,7 +88,7 @@ impl FwNode {
// - `fwnode_get_name` returns null or a valid C string.
// - `name` was checked to be non-null.
let name = unsafe { CStr::from_char_ptr(name) };
- write!(f, "{name}")
+ fmt::Display::fmt(name, f)
}
}
@@ -351,8 +352,8 @@ impl FwNodeReferenceArgs {
}
}
-impl core::fmt::Debug for FwNodeReferenceArgs {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+impl fmt::Debug for FwNodeReferenceArgs {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self.as_slice())
}
}
@@ -377,8 +378,8 @@ enum Node<'a> {
Owned(ARef<FwNode>),
}
-impl core::fmt::Display for FwNode {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+impl fmt::Display for FwNode {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// The logic here is the same as the one in lib/vsprintf.c
// (fwnode_full_name_string).
@@ -413,9 +414,9 @@ impl core::fmt::Display for FwNode {
// SAFETY: `fwnode_get_name_prefix` returns null or a
// valid C string.
let prefix = unsafe { CStr::from_char_ptr(prefix) };
- write!(f, "{prefix}")?;
+ fmt::Display::fmt(prefix, f)?;
}
- write!(f, "{}", fwnode.display_name())?;
+ fmt::Display::fmt(&fwnode.display_name(), f)?;
}
Ok(())
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 2bc8ab51ec28..68fe67624424 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -9,8 +9,8 @@ use crate::{
device::{Bound, Core},
error::{to_result, Result},
prelude::*,
+ sync::aref::ARef,
transmute::{AsBytes, FromBytes},
- types::ARef,
};
/// Trait to be implemented by DMA capable bus devices.
diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs
index d29c477e89a8..0956ba0f64de 100644
--- a/rust/kernel/drm/device.rs
+++ b/rust/kernel/drm/device.rs
@@ -2,7 +2,7 @@
//! DRM device.
//!
-//! C header: [`include/linux/drm/drm_device.h`](srctree/include/linux/drm/drm_device.h)
+//! C header: [`include/drm/drm_device.h`](srctree/include/drm/drm_device.h)
use crate::{
alloc::allocator::Kmalloc,
@@ -82,8 +82,8 @@ impl<T: drm::Driver> Device<T> {
major: T::INFO.major,
minor: T::INFO.minor,
patchlevel: T::INFO.patchlevel,
- name: T::INFO.name.as_char_ptr().cast_mut(),
- desc: T::INFO.desc.as_char_ptr().cast_mut(),
+ name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(),
+ desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(),
driver_features: drm::driver::FEAT_GEM,
ioctls: T::IOCTLS.as_ptr(),
diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs
index fe7e8d06961a..d2dad77274c4 100644
--- a/rust/kernel/drm/driver.rs
+++ b/rust/kernel/drm/driver.rs
@@ -2,7 +2,7 @@
//! DRM driver core.
//!
-//! C header: [`include/linux/drm/drm_drv.h`](srctree/include/linux/drm/drm_drv.h)
+//! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h)
use crate::{
bindings, device, devres, drm,
diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs
index e8789c9110d6..8c46f8d51951 100644
--- a/rust/kernel/drm/file.rs
+++ b/rust/kernel/drm/file.rs
@@ -2,7 +2,7 @@
//! DRM File objects.
//!
-//! C header: [`include/linux/drm/drm_file.h`](srctree/include/linux/drm/drm_file.h)
+//! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h)
use crate::{bindings, drm, error::Result, prelude::*, types::Opaque};
use core::marker::PhantomData;
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index b71821cfb5ea..b9f3248876ba 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -2,7 +2,7 @@
//! DRM GEM API
//!
-//! C header: [`include/linux/drm/drm_gem.h`](srctree/include/linux/drm/drm_gem.h)
+//! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h)
use crate::{
alloc::flags::*,
diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs
index fdec01c37168..8431cdcd3ae0 100644
--- a/rust/kernel/drm/ioctl.rs
+++ b/rust/kernel/drm/ioctl.rs
@@ -2,7 +2,7 @@
//! DRM IOCTL definitions.
//!
-//! C header: [`include/linux/drm/drm_ioctl.h`](srctree/include/linux/drm/drm_ioctl.h)
+//! C header: [`include/drm/drm_ioctl.h`](srctree/include/drm/drm_ioctl.h)
use crate::ioctl;
diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
index a41de293dcd1..1c0e0e241daa 100644
--- a/rust/kernel/error.rs
+++ b/rust/kernel/error.rs
@@ -2,7 +2,9 @@
//! Kernel errors.
//!
-//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)
+//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)\
+//! C header: [`include/uapi/asm-generic/errno.h`](srctree/include/uapi/asm-generic/errno.h)\
+//! C header: [`include/linux/errno.h`](srctree/include/linux/errno.h)
use crate::{
alloc::{layout::LayoutError, AllocError},
@@ -101,8 +103,23 @@ pub struct Error(NonZeroI32);
impl Error {
/// Creates an [`Error`] from a kernel error code.
///
- /// It is a bug to pass an out-of-range `errno`. `EINVAL` would
- /// be returned in such a case.
+ /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`).
+ ///
+ /// It is a bug to pass an out-of-range `errno`. [`code::EINVAL`] is returned in such a case.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// assert_eq!(Error::from_errno(-1), EPERM);
+ /// assert_eq!(Error::from_errno(-2), ENOENT);
+ /// ```
+ ///
+ /// The following calls are considered a bug:
+ ///
+ /// ```
+ /// assert_eq!(Error::from_errno(0), EINVAL);
+ /// assert_eq!(Error::from_errno(-1000000), EINVAL);
+ /// ```
pub fn from_errno(errno: crate::ffi::c_int) -> Error {
if let Some(error) = Self::try_from_errno(errno) {
error
@@ -158,7 +175,7 @@ impl Error {
}
/// Returns a string representing the error, if one exists.
- #[cfg(not(any(test, testlib)))]
+ #[cfg(not(testlib))]
pub fn name(&self) -> Option<&'static CStr> {
// SAFETY: Just an FFI call, there are no extra safety requirements.
let ptr = unsafe { bindings::errname(-self.0.get()) };
@@ -175,7 +192,7 @@ impl Error {
/// When `testlib` is configured, this always returns `None` to avoid the dependency on a
/// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still
/// run in userspace.
- #[cfg(any(test, testlib))]
+ #[cfg(testlib)]
pub fn name(&self) -> Option<&'static CStr> {
None
}
@@ -375,8 +392,43 @@ impl From<core::convert::Infallible> for Error {
/// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html
pub type Result<T = (), E = Error> = core::result::Result<T, E>;
-/// Converts an integer as returned by a C kernel function to an error if it's negative, and
-/// `Ok(())` otherwise.
+/// Converts an integer as returned by a C kernel function to a [`Result`].
+///
+/// If the integer is negative, an [`Err`] with an [`Error`] as given by [`Error::from_errno`] is
+/// returned. This means the integer must be `>= -MAX_ERRNO`.
+///
+/// Otherwise, it returns [`Ok`].
+///
+/// It is a bug to pass an out-of-range negative integer. `Err(EINVAL)` is returned in such a case.
+///
+/// # Examples
+///
+/// This function may be used to easily perform early returns with the [`?`] operator when working
+/// with C APIs within Rust abstractions:
+///
+/// ```
+/// # use kernel::error::to_result;
+/// # mod bindings {
+/// # #![expect(clippy::missing_safety_doc)]
+/// # use kernel::prelude::*;
+/// # pub(super) unsafe fn f1() -> c_int { 0 }
+/// # pub(super) unsafe fn f2() -> c_int { EINVAL.to_errno() }
+/// # }
+/// fn f() -> Result {
+/// // SAFETY: ...
+/// to_result(unsafe { bindings::f1() })?;
+///
+/// // SAFETY: ...
+/// to_result(unsafe { bindings::f2() })?;
+///
+/// // ...
+///
+/// Ok(())
+/// }
+/// # assert_eq!(f(), Err(EINVAL));
+/// ```
+///
+/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator
pub fn to_result(err: crate::ffi::c_int) -> Result {
if err < 0 {
Err(Error::from_errno(err))
diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs
index 1abab5b2f052..94e6bb88b903 100644
--- a/rust/kernel/firmware.rs
+++ b/rust/kernel/firmware.rs
@@ -291,7 +291,7 @@ impl<const N: usize> ModInfoBuilder<N> {
let module_name = this.module_name;
if !this.module_name.is_empty() {
- this = this.push_internal(module_name.as_bytes_with_nul());
+ this = this.push_internal(module_name.to_bytes_with_nul());
if N != 0 {
// Re-use the space taken by the NULL terminator and swap it with the '.' separator.
diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs
index f1a3fa698745..cf06e73a6da0 100644
--- a/rust/kernel/fs/file.rs
+++ b/rust/kernel/fs/file.rs
@@ -11,6 +11,7 @@ use crate::{
bindings,
cred::Credential,
error::{code::*, to_result, Error, Result},
+ fmt,
sync::aref::{ARef, AlwaysRefCounted},
types::{NotThreadSafe, Opaque},
};
@@ -460,8 +461,8 @@ impl From<BadFdError> for Error {
}
}
-impl core::fmt::Debug for BadFdError {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+impl fmt::Debug for BadFdError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.pad("EBADF")
}
}
diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs
index 41efd87595d6..3a43886cc14e 100644
--- a/rust/kernel/kunit.rs
+++ b/rust/kernel/kunit.rs
@@ -6,8 +6,8 @@
//!
//! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html>
+use crate::fmt;
use crate::prelude::*;
-use core::fmt;
#[cfg(CONFIG_PRINTK)]
use crate::c_str;
@@ -74,14 +74,14 @@ macro_rules! kunit_assert {
// mistake (it is hidden to prevent that).
//
// This mimics KUnit's failed assertion format.
- $crate::kunit::err(format_args!(
+ $crate::kunit::err($crate::prelude::fmt!(
" # {}: ASSERTION FAILED at {FILE}:{LINE}\n",
$name
));
- $crate::kunit::err(format_args!(
+ $crate::kunit::err($crate::prelude::fmt!(
" Expected {CONDITION} to be true, but is false\n"
));
- $crate::kunit::err(format_args!(
+ $crate::kunit::err($crate::prelude::fmt!(
" Failure not reported to KUnit since this is a non-KUnit task\n"
));
break 'out;
@@ -102,12 +102,12 @@ macro_rules! kunit_assert {
unsafe impl Sync for UnaryAssert {}
static LOCATION: Location = Location($crate::bindings::kunit_loc {
- file: FILE.as_char_ptr(),
+ file: $crate::str::as_char_ptr_in_const_context(FILE),
line: LINE,
});
static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert {
assert: $crate::bindings::kunit_assert {},
- condition: CONDITION.as_char_ptr(),
+ condition: $crate::str::as_char_ptr_in_const_context(CONDITION),
expected_true: true,
});
@@ -202,7 +202,7 @@ pub const fn kunit_case(
) -> kernel::bindings::kunit_case {
kernel::bindings::kunit_case {
run_case: Some(run_case),
- name: name.as_char_ptr(),
+ name: kernel::str::as_char_ptr_in_const_context(name),
attr: kernel::bindings::kunit_attributes {
speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL,
},
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index fef97f2a5098..f910a5ab80ba 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -17,6 +17,7 @@
// the unstable features in use.
//
// Stable since Rust 1.79.0.
+#![feature(generic_nonzero)]
#![feature(inline_const)]
//
// Stable since Rust 1.81.0.
@@ -28,6 +29,7 @@
// Stable since Rust 1.83.0.
#![feature(const_maybe_uninit_as_mut_ptr)]
#![feature(const_mut_refs)]
+#![feature(const_option)]
#![feature(const_ptr_write)]
#![feature(const_refs_to_cell)]
//
@@ -110,6 +112,7 @@ pub mod pid_namespace;
pub mod platform;
pub mod prelude;
pub mod print;
+pub mod ptr;
pub mod rbtree;
pub mod regulator;
pub mod revocable;
@@ -206,7 +209,7 @@ impl ThisModule {
}
}
-#[cfg(not(any(testlib, test)))]
+#[cfg(not(testlib))]
#[panic_handler]
fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
pr_emerg!("{}\n", info);
diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs
index 44e5219cfcbc..7355bbac16a7 100644
--- a/rust/kernel/list.rs
+++ b/rust/kernel/list.rs
@@ -38,6 +38,8 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField};
///
/// # Examples
///
+/// Use [`ListLinks`] as the type of the intrusive field.
+///
/// ```
/// use kernel::list::*;
///
@@ -140,6 +142,124 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField};
/// }
/// # Result::<(), Error>::Ok(())
/// ```
+///
+/// Use [`ListLinksSelfPtr`] as the type of the intrusive field. This allows a list of trait object
+/// type.
+///
+/// ```
+/// use kernel::list::*;
+///
+/// trait Foo {
+/// fn foo(&self) -> (&'static str, i32);
+/// }
+///
+/// #[pin_data]
+/// struct DTWrap<T: ?Sized> {
+/// #[pin]
+/// links: ListLinksSelfPtr<DTWrap<dyn Foo>>,
+/// value: T,
+/// }
+///
+/// impl<T> DTWrap<T> {
+/// fn new(value: T) -> Result<ListArc<Self>> {
+/// ListArc::pin_init(try_pin_init!(Self {
+/// value,
+/// links <- ListLinksSelfPtr::new(),
+/// }), GFP_KERNEL)
+/// }
+/// }
+///
+/// impl_list_arc_safe! {
+/// impl{T: ?Sized} ListArcSafe<0> for DTWrap<T> { untracked; }
+/// }
+/// impl_list_item! {
+/// impl ListItem<0> for DTWrap<dyn Foo> { using ListLinksSelfPtr { self.links }; }
+/// }
+///
+/// // Create a new empty list.
+/// let mut list = List::<DTWrap<dyn Foo>>::new();
+/// {
+/// assert!(list.is_empty());
+/// }
+///
+/// struct A(i32);
+/// // `A` returns the inner value for `foo`.
+/// impl Foo for A { fn foo(&self) -> (&'static str, i32) { ("a", self.0) } }
+///
+/// struct B;
+/// // `B` always returns 42.
+/// impl Foo for B { fn foo(&self) -> (&'static str, i32) { ("b", 42) } }
+///
+/// // Insert 3 element using `push_back()`.
+/// list.push_back(DTWrap::new(A(15))?);
+/// list.push_back(DTWrap::new(A(32))?);
+/// list.push_back(DTWrap::new(B)?);
+///
+/// // Iterate over the list to verify the nodes were inserted correctly.
+/// // [A(15), A(32), B]
+/// {
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42));
+/// assert!(iter.next().is_none());
+///
+/// // Verify the length of the list.
+/// assert_eq!(list.iter().count(), 3);
+/// }
+///
+/// // Pop the items from the list using `pop_back()` and verify the content.
+/// {
+/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("b", 42));
+/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32));
+/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15));
+/// }
+///
+/// // Insert 3 elements using `push_front()`.
+/// list.push_front(DTWrap::new(A(15))?);
+/// list.push_front(DTWrap::new(A(32))?);
+/// list.push_front(DTWrap::new(B)?);
+///
+/// // Iterate over the list to verify the nodes were inserted correctly.
+/// // [B, A(32), A(15)]
+/// {
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15));
+/// assert!(iter.next().is_none());
+///
+/// // Verify the length of the list.
+/// assert_eq!(list.iter().count(), 3);
+/// }
+///
+/// // Pop the items from the list using `pop_front()` and verify the content.
+/// {
+/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15));
+/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32));
+/// }
+///
+/// // Push `list2` to `list` through `push_all_back()`.
+/// // list: [B]
+/// // list2: [B, A(25)]
+/// {
+/// let mut list2 = List::<DTWrap<dyn Foo>>::new();
+/// list2.push_back(DTWrap::new(B)?);
+/// list2.push_back(DTWrap::new(A(25))?);
+///
+/// list.push_all_back(&mut list2);
+///
+/// // list: [B, B, A(25)]
+/// // list2: []
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42));
+/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 25));
+/// assert!(iter.next().is_none());
+/// assert!(list2.is_empty());
+/// }
+/// # Result::<(), Error>::Ok(())
+/// ```
pub struct List<T: ?Sized + ListItem<ID>, const ID: u64 = 0> {
first: *mut ListLinksFields,
_ty: PhantomData<ListArc<T, ID>>,
diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs
index 6373fe183b27..d3aa7d25afad 100644
--- a/rust/kernel/miscdevice.rs
+++ b/rust/kernel/miscdevice.rs
@@ -34,7 +34,7 @@ impl MiscDeviceOptions {
// SAFETY: All zeros is valid for this C type.
let mut result: bindings::miscdevice = unsafe { MaybeUninit::zeroed().assume_init() };
result.minor = bindings::MISC_DYNAMIC_MINOR as ffi::c_int;
- result.name = self.name.as_char_ptr();
+ result.name = crate::str::as_char_ptr_in_const_context(self.name);
result.fops = MiscdeviceVTable::<T>::build();
result
}
diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs
index 7de5cc7a0eee..be1027b7961b 100644
--- a/rust/kernel/net/phy.rs
+++ b/rust/kernel/net/phy.rs
@@ -497,7 +497,7 @@ unsafe impl Sync for DriverVTable {}
pub const fn create_phy_driver<T: Driver>() -> DriverVTable {
// INVARIANT: All the fields of `struct phy_driver` are initialized properly.
DriverVTable(Opaque::new(bindings::phy_driver {
- name: T::NAME.as_char_ptr().cast_mut(),
+ name: crate::str::as_char_ptr_in_const_context(T::NAME).cast_mut(),
flags: T::FLAGS,
phy_id: T::PHY_DEVICE_ID.id(),
phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(),
diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs
index b76b35265df2..58b20c367f99 100644
--- a/rust/kernel/of.rs
+++ b/rust/kernel/of.rs
@@ -34,7 +34,7 @@ unsafe impl RawDeviceIdIndex for DeviceId {
impl DeviceId {
/// Create a new device id from an OF 'compatible' string.
pub const fn new(compatible: &'static CStr) -> Self {
- let src = compatible.as_bytes_with_nul();
+ let src = compatible.to_bytes_with_nul();
// Replace with `bindings::of_device_id::default()` once stabilized for `const`.
// SAFETY: FFI type is valid to be zero-initialized.
let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() };
diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
index 25fe97aafd02..198d09a31449 100644
--- a/rust/kernel/prelude.rs
+++ b/rust/kernel/prelude.rs
@@ -12,7 +12,10 @@
//! ```
#[doc(no_inline)]
-pub use core::pin::Pin;
+pub use core::{
+ mem::{align_of, align_of_val, size_of, size_of_val},
+ pin::Pin,
+};
pub use ::ffi::{
c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong,
diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs
new file mode 100644
index 000000000000..2e5e2a090480
--- /dev/null
+++ b/rust/kernel/ptr.rs
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Types and functions to work with pointers and addresses.
+
+use core::fmt::Debug;
+use core::mem::align_of;
+use core::num::NonZero;
+
+use crate::build_assert;
+
+/// Type representing an alignment, which is always a power of two.
+///
+/// It is used to validate that a given value is a valid alignment, and to perform masking and
+/// alignment operations.
+///
+/// This is a temporary substitute for the [`Alignment`] nightly type from the standard library,
+/// and to be eventually replaced by it.
+///
+/// [`Alignment`]: https://github.com/rust-lang/rust/issues/102070
+///
+/// # Invariants
+///
+/// An alignment is always a power of two.
+#[repr(transparent)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Alignment(NonZero<usize>);
+
+impl Alignment {
+ /// Validates that `ALIGN` is a power of two at build-time, and returns an [`Alignment`] of the
+ /// same value.
+ ///
+ /// A build error is triggered if `ALIGN` is not a power of two.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// let v = Alignment::new::<16>();
+ /// assert_eq!(v.as_usize(), 16);
+ /// ```
+ #[inline(always)]
+ pub const fn new<const ALIGN: usize>() -> Self {
+ build_assert!(
+ ALIGN.is_power_of_two(),
+ "Provided alignment is not a power of two."
+ );
+
+ // INVARIANT: `align` is a power of two.
+ // SAFETY: `align` is a power of two, and thus non-zero.
+ Self(unsafe { NonZero::new_unchecked(ALIGN) })
+ }
+
+ /// Validates that `align` is a power of two at runtime, and returns an
+ /// [`Alignment`] of the same value.
+ ///
+ /// Returns [`None`] if `align` is not a power of two.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// assert_eq!(Alignment::new_checked(16), Some(Alignment::new::<16>()));
+ /// assert_eq!(Alignment::new_checked(15), None);
+ /// assert_eq!(Alignment::new_checked(1), Some(Alignment::new::<1>()));
+ /// assert_eq!(Alignment::new_checked(0), None);
+ /// ```
+ #[inline(always)]
+ pub const fn new_checked(align: usize) -> Option<Self> {
+ if align.is_power_of_two() {
+ // INVARIANT: `align` is a power of two.
+ // SAFETY: `align` is a power of two, and thus non-zero.
+ Some(Self(unsafe { NonZero::new_unchecked(align) }))
+ } else {
+ None
+ }
+ }
+
+ /// Returns the alignment of `T`.
+ ///
+ /// This is equivalent to [`align_of`], but with the return value provided as an [`Alignment`].
+ #[inline(always)]
+ pub const fn of<T>() -> Self {
+ #![allow(clippy::incompatible_msrv)]
+ // This cannot panic since alignments are always powers of two.
+ //
+ // We unfortunately cannot use `new` as it would require the `generic_const_exprs` feature.
+ const { Alignment::new_checked(align_of::<T>()).unwrap() }
+ }
+
+ /// Returns this alignment as a [`usize`].
+ ///
+ /// It is guaranteed to be a power of two.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// assert_eq!(Alignment::new::<16>().as_usize(), 16);
+ /// ```
+ #[inline(always)]
+ pub const fn as_usize(self) -> usize {
+ self.as_nonzero().get()
+ }
+
+ /// Returns this alignment as a [`NonZero`].
+ ///
+ /// It is guaranteed to be a power of two.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// assert_eq!(Alignment::new::<16>().as_nonzero().get(), 16);
+ /// ```
+ #[inline(always)]
+ pub const fn as_nonzero(self) -> NonZero<usize> {
+ // Allow the compiler to know that the value is indeed a power of two. This can help
+ // optimize some operations down the line, like e.g. replacing divisions by bit shifts.
+ if !self.0.is_power_of_two() {
+ // SAFETY: Per the invariants, `self.0` is always a power of two so this block will
+ // never be reached.
+ unsafe { core::hint::unreachable_unchecked() }
+ }
+ self.0
+ }
+
+ /// Returns the base-2 logarithm of the alignment.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// assert_eq!(Alignment::of::<u8>().log2(), 0);
+ /// assert_eq!(Alignment::new::<16>().log2(), 4);
+ /// ```
+ #[inline(always)]
+ pub const fn log2(self) -> u32 {
+ self.0.ilog2()
+ }
+
+ /// Returns the mask for this alignment.
+ ///
+ /// This is equivalent to `!(self.as_usize() - 1)`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::Alignment;
+ ///
+ /// assert_eq!(Alignment::new::<0x10>().mask(), !0xf);
+ /// ```
+ #[inline(always)]
+ pub const fn mask(self) -> usize {
+ // No underflow can occur as the alignment is guaranteed to be a power of two, and thus is
+ // non-zero.
+ !(self.as_usize() - 1)
+ }
+}
+
+/// Trait for items that can be aligned against an [`Alignment`].
+pub trait Alignable: Sized {
+ /// Aligns `self` down to `alignment`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::{Alignable, Alignment};
+ ///
+ /// assert_eq!(0x2f_usize.align_down(Alignment::new::<0x10>()), 0x20);
+ /// assert_eq!(0x30usize.align_down(Alignment::new::<0x10>()), 0x30);
+ /// assert_eq!(0xf0u8.align_down(Alignment::new::<0x1000>()), 0x0);
+ /// ```
+ fn align_down(self, alignment: Alignment) -> Self;
+
+ /// Aligns `self` up to `alignment`, returning `None` if aligning would result in an overflow.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::ptr::{Alignable, Alignment};
+ ///
+ /// assert_eq!(0x4fusize.align_up(Alignment::new::<0x10>()), Some(0x50));
+ /// assert_eq!(0x40usize.align_up(Alignment::new::<0x10>()), Some(0x40));
+ /// assert_eq!(0x0usize.align_up(Alignment::new::<0x10>()), Some(0x0));
+ /// assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None);
+ /// assert_eq!(0x10u8.align_up(Alignment::new::<0x100>()), None);
+ /// assert_eq!(0x0u8.align_up(Alignment::new::<0x100>()), Some(0x0));
+ /// ```
+ fn align_up(self, alignment: Alignment) -> Option<Self>;
+}
+
+/// Implement [`Alignable`] for unsigned integer types.
+macro_rules! impl_alignable_uint {
+ ($($t:ty),*) => {
+ $(
+ impl Alignable for $t {
+ #[inline(always)]
+ fn align_down(self, alignment: Alignment) -> Self {
+ // The operands of `&` need to be of the same type so convert the alignment to
+ // `Self`. This means we need to compute the mask ourselves.
+ ::core::num::NonZero::<Self>::try_from(alignment.as_nonzero())
+ .map(|align| self & !(align.get() - 1))
+ // An alignment larger than `Self` always aligns down to `0`.
+ .unwrap_or(0)
+ }
+
+ #[inline(always)]
+ fn align_up(self, alignment: Alignment) -> Option<Self> {
+ let aligned_down = self.align_down(alignment);
+ if self == aligned_down {
+ Some(aligned_down)
+ } else {
+ Self::try_from(alignment.as_usize())
+ .ok()
+ .and_then(|align| aligned_down.checked_add(align))
+ }
+ }
+ }
+ )*
+ };
+}
+
+impl_alignable_uint!(u8, u16, u32, u64, usize);
diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs
index 8f199b1a3bb1..59fbfc2473f8 100644
--- a/rust/kernel/seq_file.rs
+++ b/rust/kernel/seq_file.rs
@@ -4,7 +4,7 @@
//!
//! C header: [`include/linux/seq_file.h`](srctree/include/linux/seq_file.h)
-use crate::{bindings, c_str, types::NotThreadSafe, types::Opaque};
+use crate::{bindings, c_str, fmt, types::NotThreadSafe, types::Opaque};
/// A utility for generating the contents of a seq file.
#[repr(transparent)]
@@ -31,7 +31,7 @@ impl SeqFile {
/// Used by the [`seq_print`] macro.
#[inline]
- pub fn call_printf(&self, args: core::fmt::Arguments<'_>) {
+ pub fn call_printf(&self, args: fmt::Arguments<'_>) {
// SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`.
unsafe {
bindings::seq_printf(
@@ -47,7 +47,7 @@ impl SeqFile {
#[macro_export]
macro_rules! seq_print {
($m:expr, $($arg:tt)+) => (
- $m.call_printf(format_args!($($arg)+))
+ $m.call_printf($crate::prelude::fmt!($($arg)+))
);
}
pub use seq_print;
diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs
index 9298993ea7d8..289f77abf415 100644
--- a/rust/kernel/sync/arc.rs
+++ b/rust/kernel/sync/arc.rs
@@ -19,6 +19,7 @@
use crate::{
alloc::{AllocError, Flags, KBox},
ffi::c_void,
+ fmt,
init::InPlaceInit,
sync::Refcount,
try_init,
@@ -27,7 +28,6 @@ use crate::{
use core::{
alloc::Layout,
borrow::{Borrow, BorrowMut},
- fmt,
marker::PhantomData,
mem::{ManuallyDrop, MaybeUninit},
ops::{Deref, DerefMut},
@@ -367,10 +367,10 @@ impl<T: ?Sized> Arc<T> {
}
}
-// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
-// pointer to `ArcInner<T>`.
+// SAFETY: The pointer returned by `into_foreign` was originally allocated as an
+// `KBox<ArcInner<T>>`, so that type is what determines the alignment.
unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
- const FOREIGN_ALIGN: usize = core::mem::align_of::<ArcInner<T>>();
+ const FOREIGN_ALIGN: usize = <KBox<ArcInner<T>> as ForeignOwnable>::FOREIGN_ALIGN;
type Borrowed<'a> = ArcBorrow<'a, T>;
type BorrowedMut<'a> = Self::Borrowed<'a>;
diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs
index dbd77bb68617..0d24a0432015 100644
--- a/rust/kernel/sync/aref.rs
+++ b/rust/kernel/sync/aref.rs
@@ -1,6 +1,21 @@
// SPDX-License-Identifier: GPL-2.0
//! Internal reference counting support.
+//!
+//! Many C types already have their own reference counting mechanism (e.g. by storing a
+//! `refcount_t`). This module provides support for directly using their internal reference count
+//! from Rust; instead of making users have to use an additional Rust-reference count in the form of
+//! [`Arc`].
+//!
+//! The smart pointer [`ARef<T>`] acts similarly to [`Arc<T>`] in that it holds a refcount on the
+//! underlying object, but this refcount is internal to the object. It essentially is a Rust
+//! implementation of the `get_` and `put_` pattern used in C for reference counting.
+//!
+//! To make use of [`ARef<MyType>`], `MyType` needs to implement [`AlwaysRefCounted`]. It is a trait
+//! for accessing the internal reference count of an object of the `MyType` type.
+//!
+//! [`Arc`]: crate::sync::Arc
+//! [`Arc<T>`]: crate::sync::Arc
use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull};
@@ -97,7 +112,7 @@ impl<T: AlwaysRefCounted> ARef<T> {
///
/// ```
/// use core::ptr::NonNull;
- /// use kernel::types::{ARef, AlwaysRefCounted};
+ /// use kernel::sync::aref::{ARef, AlwaysRefCounted};
///
/// struct Empty {}
///
diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs
index 7d0935bc325c..49fad6de0674 100644
--- a/rust/kernel/task.rs
+++ b/rust/kernel/task.rs
@@ -9,7 +9,8 @@ use crate::{
ffi::{c_int, c_long, c_uint},
mm::MmWithUser,
pid_namespace::PidNamespace,
- types::{ARef, NotThreadSafe, Opaque},
+ sync::aref::ARef,
+ types::{NotThreadSafe, Opaque},
};
use core::{
cmp::{Eq, PartialEq},
@@ -76,7 +77,7 @@ macro_rules! current {
/// incremented when creating `State` and decremented when it is dropped:
///
/// ```
-/// use kernel::{task::Task, types::ARef};
+/// use kernel::{task::Task, sync::aref::ARef};
///
/// struct State {
/// creator: ARef<Task>,
@@ -347,7 +348,7 @@ impl CurrentTask {
}
// SAFETY: The type invariants guarantee that `Task` is always refcounted.
-unsafe impl crate::types::AlwaysRefCounted for Task {
+unsafe impl crate::sync::aref::AlwaysRefCounted for Task {
#[inline]
fn inc_ref(&self) {
// SAFETY: The existence of a shared reference means that the refcount is nonzero.
diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs
index 64c8dcf548d6..6ea98dfcd027 100644
--- a/rust/kernel/time.rs
+++ b/rust/kernel/time.rs
@@ -25,6 +25,7 @@
//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).
use core::marker::PhantomData;
+use core::ops;
pub mod delay;
pub mod hrtimer;
@@ -200,9 +201,31 @@ impl<C: ClockSource> Instant<C> {
pub(crate) fn as_nanos(&self) -> i64 {
self.inner
}
+
+ /// Create an [`Instant`] from a `ktime_t` without checking if it is non-negative.
+ ///
+ /// # Panics
+ ///
+ /// On debug builds, this function will panic if `ktime` is not in the range from 0 to
+ /// `KTIME_MAX`.
+ ///
+ /// # Safety
+ ///
+ /// The caller promises that `ktime` is in the range from 0 to `KTIME_MAX`.
+ #[inline]
+ pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self {
+ debug_assert!(ktime >= 0);
+
+ // INVARIANT: Our safety contract ensures that `ktime` is in the range from 0 to
+ // `KTIME_MAX`.
+ Self {
+ inner: ktime,
+ _c: PhantomData,
+ }
+ }
}
-impl<C: ClockSource> core::ops::Sub for Instant<C> {
+impl<C: ClockSource> ops::Sub for Instant<C> {
type Output = Delta;
// By the type invariant, it never overflows.
@@ -214,6 +237,46 @@ impl<C: ClockSource> core::ops::Sub for Instant<C> {
}
}
+impl<T: ClockSource> ops::Add<Delta> for Instant<T> {
+ type Output = Self;
+
+ #[inline]
+ fn add(self, rhs: Delta) -> Self::Output {
+ // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow
+ // (e.g. go above `KTIME_MAX`)
+ let res = self.inner + rhs.nanos;
+
+ // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0
+ #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)]
+ assert!(res >= 0);
+
+ Self {
+ inner: res,
+ _c: PhantomData,
+ }
+ }
+}
+
+impl<T: ClockSource> ops::Sub<Delta> for Instant<T> {
+ type Output = Self;
+
+ #[inline]
+ fn sub(self, rhs: Delta) -> Self::Output {
+ // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow
+ // (e.g. go above `KTIME_MAX`)
+ let res = self.inner - rhs.nanos;
+
+ // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0
+ #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)]
+ assert!(res >= 0);
+
+ Self {
+ inner: res,
+ _c: PhantomData,
+ }
+ }
+}
+
/// A span of time.
///
/// This struct represents a span of time, with its value stored as nanoseconds.
@@ -224,6 +287,78 @@ pub struct Delta {
nanos: i64,
}
+impl ops::Add for Delta {
+ type Output = Self;
+
+ #[inline]
+ fn add(self, rhs: Self) -> Self {
+ Self {
+ nanos: self.nanos + rhs.nanos,
+ }
+ }
+}
+
+impl ops::AddAssign for Delta {
+ #[inline]
+ fn add_assign(&mut self, rhs: Self) {
+ self.nanos += rhs.nanos;
+ }
+}
+
+impl ops::Sub for Delta {
+ type Output = Self;
+
+ #[inline]
+ fn sub(self, rhs: Self) -> Self::Output {
+ Self {
+ nanos: self.nanos - rhs.nanos,
+ }
+ }
+}
+
+impl ops::SubAssign for Delta {
+ #[inline]
+ fn sub_assign(&mut self, rhs: Self) {
+ self.nanos -= rhs.nanos;
+ }
+}
+
+impl ops::Mul<i64> for Delta {
+ type Output = Self;
+
+ #[inline]
+ fn mul(self, rhs: i64) -> Self::Output {
+ Self {
+ nanos: self.nanos * rhs,
+ }
+ }
+}
+
+impl ops::MulAssign<i64> for Delta {
+ #[inline]
+ fn mul_assign(&mut self, rhs: i64) {
+ self.nanos *= rhs;
+ }
+}
+
+impl ops::Div for Delta {
+ type Output = i64;
+
+ #[inline]
+ fn div(self, rhs: Self) -> Self::Output {
+ #[cfg(CONFIG_64BIT)]
+ {
+ self.nanos / rhs.nanos
+ }
+
+ #[cfg(not(CONFIG_64BIT))]
+ {
+ // SAFETY: This function is always safe to call regardless of the input values
+ unsafe { bindings::div64_s64(self.nanos, rhs.nanos) }
+ }
+ }
+}
+
impl Delta {
/// A span of time equal to zero.
pub const ZERO: Self = Self { nanos: 0 };
@@ -312,4 +447,30 @@ impl Delta {
bindings::ktime_to_ms(self.as_nanos())
}
}
+
+ /// Return `self % dividend` where `dividend` is in nanoseconds.
+ ///
+ /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is
+ /// limited to 32 bit dividends.
+ #[inline]
+ pub fn rem_nanos(self, dividend: i32) -> Self {
+ #[cfg(CONFIG_64BIT)]
+ {
+ Self {
+ nanos: self.as_nanos() % i64::from(dividend),
+ }
+ }
+
+ #[cfg(not(CONFIG_64BIT))]
+ {
+ let mut rem = 0;
+
+ // SAFETY: `rem` is in the stack, so we can always provide a valid pointer to it.
+ unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) };
+
+ Self {
+ nanos: i64::from(rem),
+ }
+ }
+ }
}
diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs
index 144e3b57cc78..856d2d929a00 100644
--- a/rust/kernel/time/hrtimer.rs
+++ b/rust/kernel/time/hrtimer.rs
@@ -69,9 +69,14 @@
use super::{ClockSource, Delta, Instant};
use crate::{prelude::*, types::Opaque};
-use core::marker::PhantomData;
+use core::{marker::PhantomData, ptr::NonNull};
use pin_init::PinInit;
+/// A type-alias to refer to the [`Instant<C>`] for a given `T` from [`HrTimer<T>`].
+///
+/// Where `C` is the [`ClockSource`] of the [`HrTimer`].
+pub type HrTimerInstant<T> = Instant<<<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock>;
+
/// A timer backed by a C `struct hrtimer`.
///
/// # Invariants
@@ -163,6 +168,84 @@ impl<T> HrTimer<T> {
// handled on the C side.
unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 }
}
+
+ /// Forward the timer expiry for a given timer pointer.
+ ///
+ /// # Safety
+ ///
+ /// - `self_ptr` must point to a valid `Self`.
+ /// - The caller must either have exclusive access to the data pointed at by `self_ptr`, or be
+ /// within the context of the timer callback.
+ #[inline]
+ unsafe fn raw_forward(self_ptr: *mut Self, now: HrTimerInstant<T>, interval: Delta) -> u64
+ where
+ T: HasHrTimer<T>,
+ {
+ // SAFETY:
+ // * The C API requirements for this function are fulfilled by our safety contract.
+ // * `self_ptr` is guaranteed to point to a valid `Self` via our safety contract
+ unsafe {
+ bindings::hrtimer_forward(Self::raw_get(self_ptr), now.as_nanos(), interval.as_nanos())
+ }
+ }
+
+ /// Conditionally forward the timer.
+ ///
+ /// If the timer expires after `now`, this function does nothing and returns 0. If the timer
+ /// expired at or before `now`, this function forwards the timer by `interval` until the timer
+ /// expires after `now` and then returns the number of times the timer was forwarded by
+ /// `interval`.
+ ///
+ /// This function is mainly useful for timer types which can provide exclusive access to the
+ /// timer when the timer is not running. For forwarding the timer from within the timer callback
+ /// context, see [`HrTimerCallbackContext::forward()`].
+ ///
+ /// Returns the number of overruns that occurred as a result of the timer expiry change.
+ pub fn forward(self: Pin<&mut Self>, now: HrTimerInstant<T>, interval: Delta) -> u64
+ where
+ T: HasHrTimer<T>,
+ {
+ // SAFETY: `raw_forward` does not move `Self`
+ let this = unsafe { self.get_unchecked_mut() };
+
+ // SAFETY: By existence of `Pin<&mut Self>`, the pointer passed to `raw_forward` points to a
+ // valid `Self` that we have exclusive access to.
+ unsafe { Self::raw_forward(this, now, interval) }
+ }
+
+ /// Conditionally forward the timer.
+ ///
+ /// This is a variant of [`forward()`](Self::forward) that uses an interval after the current
+ /// time of the base clock for the [`HrTimer`].
+ pub fn forward_now(self: Pin<&mut Self>, interval: Delta) -> u64
+ where
+ T: HasHrTimer<T>,
+ {
+ self.forward(HrTimerInstant::<T>::now(), interval)
+ }
+
+ /// Return the time expiry for this [`HrTimer`].
+ ///
+ /// This value should only be used as a snapshot, as the actual expiry time could change after
+ /// this function is called.
+ pub fn expires(&self) -> HrTimerInstant<T>
+ where
+ T: HasHrTimer<T>,
+ {
+ // SAFETY: `self` is an immutable reference and thus always points to a valid `HrTimer`.
+ let c_timer_ptr = unsafe { HrTimer::raw_get(self) };
+
+ // SAFETY:
+ // - Timers cannot have negative ktime_t values as their expiration time.
+ // - There's no actual locking here, a racy read is fine and expected
+ unsafe {
+ Instant::from_ktime(
+ // This `read_volatile` is intended to correspond to a READ_ONCE call.
+ // FIXME(read_once): Replace with `read_once` when available on the Rust side.
+ core::ptr::read_volatile(&raw const ((*c_timer_ptr).node.expires)),
+ )
+ }
+ }
}
/// Implemented by pointer types that point to structs that contain a [`HrTimer`].
@@ -300,9 +383,13 @@ pub trait HrTimerCallback {
type Pointer<'a>: RawHrTimerCallback;
/// Called by the timer logic when the timer fires.
- fn run(this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>) -> HrTimerRestart
+ fn run(
+ this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>,
+ ctx: HrTimerCallbackContext<'_, Self>,
+ ) -> HrTimerRestart
where
- Self: Sized;
+ Self: Sized,
+ Self: HasHrTimer<Self>;
}
/// A handle representing a potentially running timer.
@@ -324,6 +411,8 @@ pub unsafe trait HrTimerHandle {
/// Note that the timer might be started by a concurrent start operation. If
/// so, the timer might not be in the **stopped** state when this function
/// returns.
+ ///
+ /// Returns `true` if the timer was running.
fn cancel(&mut self) -> bool;
}
@@ -585,6 +674,63 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> {
type Expires = Delta;
}
+/// Privileged smart-pointer for a [`HrTimer`] callback context.
+///
+/// Many [`HrTimer`] methods can only be called in two situations:
+///
+/// * When the caller has exclusive access to the `HrTimer` and the `HrTimer` is guaranteed not to
+/// be running.
+/// * From within the context of an `HrTimer`'s callback method.
+///
+/// This type provides access to said methods from within a timer callback context.
+///
+/// # Invariants
+///
+/// * The existence of this type means the caller is currently within the callback for an
+/// [`HrTimer`].
+/// * `self.0` always points to a live instance of [`HrTimer<T>`].
+pub struct HrTimerCallbackContext<'a, T: HasHrTimer<T>>(NonNull<HrTimer<T>>, PhantomData<&'a ()>);
+
+impl<'a, T: HasHrTimer<T>> HrTimerCallbackContext<'a, T> {
+ /// Create a new [`HrTimerCallbackContext`].
+ ///
+ /// # Safety
+ ///
+ /// This function relies on the caller being within the context of a timer callback, so it must
+ /// not be used anywhere except for within implementations of [`RawHrTimerCallback::run`]. The
+ /// caller promises that `timer` points to a valid initialized instance of
+ /// [`bindings::hrtimer`].
+ ///
+ /// The returned `Self` must not outlive the function context of [`RawHrTimerCallback::run`]
+ /// where this function is called.
+ pub(crate) unsafe fn from_raw(timer: *mut HrTimer<T>) -> Self {
+ // SAFETY: The caller guarantees `timer` is a valid pointer to an initialized
+ // `bindings::hrtimer`
+ // INVARIANT: Our safety contract ensures that we're within the context of a timer callback
+ // and that `timer` points to a live instance of `HrTimer<T>`.
+ Self(unsafe { NonNull::new_unchecked(timer) }, PhantomData)
+ }
+
+ /// Conditionally forward the timer.
+ ///
+ /// This function is identical to [`HrTimer::forward()`] except that it may only be used from
+ /// within the context of a [`HrTimer`] callback.
+ pub fn forward(&mut self, now: HrTimerInstant<T>, interval: Delta) -> u64 {
+ // SAFETY:
+ // - We are guaranteed to be within the context of a timer callback by our type invariants
+ // - By our type invariants, `self.0` always points to a valid `HrTimer<T>`
+ unsafe { HrTimer::<T>::raw_forward(self.0.as_ptr(), now, interval) }
+ }
+
+ /// Conditionally forward the timer.
+ ///
+ /// This is a variant of [`HrTimerCallbackContext::forward()`] that uses an interval after the
+ /// current time of the base clock for the [`HrTimer`].
+ pub fn forward_now(&mut self, duration: Delta) -> u64 {
+ self.forward(HrTimerInstant::<T>::now(), duration)
+ }
+}
+
/// Use to implement the [`HasHrTimer<T>`] trait.
///
/// See [`module`] documentation for an example.
diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs
index ed490a7a8950..7be82bcb352a 100644
--- a/rust/kernel/time/hrtimer/arc.rs
+++ b/rust/kernel/time/hrtimer/arc.rs
@@ -3,6 +3,7 @@
use super::HasHrTimer;
use super::HrTimer;
use super::HrTimerCallback;
+use super::HrTimerCallbackContext;
use super::HrTimerHandle;
use super::HrTimerMode;
use super::HrTimerPointer;
@@ -99,6 +100,12 @@ where
// allocation from other `Arc` clones.
let receiver = unsafe { ArcBorrow::from_raw(data_ptr) };
- T::run(receiver).into_c()
+ // SAFETY:
+ // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
+ // it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
+ // - We are within `RawHrTimerCallback::run`
+ let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };
+
+ T::run(receiver, context).into_c()
}
}
diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs
index aef16d9ee2f0..4d39ef781697 100644
--- a/rust/kernel/time/hrtimer/pin.rs
+++ b/rust/kernel/time/hrtimer/pin.rs
@@ -3,6 +3,7 @@
use super::HasHrTimer;
use super::HrTimer;
use super::HrTimerCallback;
+use super::HrTimerCallbackContext;
use super::HrTimerHandle;
use super::HrTimerMode;
use super::RawHrTimerCallback;
@@ -103,6 +104,12 @@ where
// here.
let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };
- T::run(receiver_pin).into_c()
+ // SAFETY:
+ // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
+ // it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
+ // - We are within `RawHrTimerCallback::run`
+ let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };
+
+ T::run(receiver_pin, context).into_c()
}
}
diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs
index 767d0a4e8a2c..9d9447d4d57e 100644
--- a/rust/kernel/time/hrtimer/pin_mut.rs
+++ b/rust/kernel/time/hrtimer/pin_mut.rs
@@ -1,8 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
use super::{
- HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, HrTimerMode, RawHrTimerCallback,
- UnsafeHrTimerPointer,
+ HasHrTimer, HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerHandle, HrTimerMode,
+ RawHrTimerCallback, UnsafeHrTimerPointer,
};
use core::{marker::PhantomData, pin::Pin, ptr::NonNull};
@@ -107,6 +107,12 @@ where
// here.
let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };
- T::run(receiver_pin).into_c()
+ // SAFETY:
+ // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
+ // it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
+ // - We are within `RawHrTimerCallback::run`
+ let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };
+
+ T::run(receiver_pin, context).into_c()
}
}
diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs
index ec08303315f2..aa1ee31a7195 100644
--- a/rust/kernel/time/hrtimer/tbox.rs
+++ b/rust/kernel/time/hrtimer/tbox.rs
@@ -3,6 +3,7 @@
use super::HasHrTimer;
use super::HrTimer;
use super::HrTimerCallback;
+use super::HrTimerCallbackContext;
use super::HrTimerHandle;
use super::HrTimerMode;
use super::HrTimerPointer;
@@ -119,6 +120,12 @@ where
// `data_ptr` exist.
let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) };
- T::run(data_mut_ref).into_c()
+ // SAFETY:
+ // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
+ // it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
+ // - We are within `RawHrTimerCallback::run`
+ let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };
+
+ T::run(data_mut_ref, context).into_c()
}
}