diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2025-09-30 19:12:49 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2025-09-30 19:12:49 -0700 |
commit | f4e0ff7e45c30f4665cfbbe2f0538e9c5789bebc (patch) | |
tree | 6703541ef1fec9d20dafc751d2e971879c429884 /rust/kernel | |
parent | ae28ed4578e6d5a481e39c5a9827f27048661fdd (diff) | |
parent | f3f6b3664302e16ef1c6b91034a72df5564d6b8a (diff) |
Merge tag 'rust-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/ojeda/linux
Pull rust updates from Miguel Ojeda:
"Toolchain and infrastructure:
- Derive 'Zeroable' for all structs and unions generated by 'bindgen'
where possible and corresponding cleanups. To do so, add the
'pin-init' crate as a dependency to 'bindings' and 'uapi'.
It also includes its first use in the 'cpufreq' module, with more
to come in the next cycle.
- Add warning to the 'rustdoc' target to detect broken 'srctree/'
links and fix existing cases.
- Remove support for unused (since v6.16) host '#[test]'s,
simplifying the 'rusttest' target. Tests should generally run
within KUnit.
'kernel' crate:
- Add 'ptr' module with a new 'Alignment' type, which is always a
power of two and is used to validate that a given value is a valid
alignment and to perform masking and alignment operations:
// Checked at build time.
assert_eq!(Alignment::new::<16>().as_usize(), 16);
// Checked at runtime.
assert_eq!(Alignment::new_checked(15), None);
assert_eq!(Alignment::of::<u8>().log2(), 0);
assert_eq!(0x25u8.align_down(Alignment::new::<0x10>()), 0x20);
assert_eq!(0x5u8.align_up(Alignment::new::<0x10>()), Some(0x10));
assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None);
It also includes its first use in Nova.
- Add 'core::mem::{align,size}_of{,_val}' to the prelude, matching
Rust 1.80.0.
- Keep going with the steps on our migration to the standard library
'core::ffi::CStr' type (use 'kernel::{fmt, prelude::fmt!}' and use
upstream method names).
- 'error' module: improve 'Error::from_errno' and 'to_result'
documentation, including examples/tests.
- 'sync' module: extend 'aref' submodule documentation now that it
exists, and more updates to complete the ongoing move of 'ARef' and
'AlwaysRefCounted' to 'sync::aref'.
- 'list' module: add an example/test for 'ListLinksSelfPtr' usage.
- 'alloc' module:
- Implement 'Box::pin_slice()', which constructs a pinned slice of
elements.
- Provide information about the minimum alignment guarantees of
'Kmalloc', 'Vmalloc' and 'KVmalloc'.
- Take minimum alignment guarantees of allocators for
'ForeignOwnable' into account.
- Remove the 'allocator_test' (including 'Cmalloc').
- Add doctest for 'Vec::as_slice()'.
- Constify various methods.
- 'time' module:
- Add methods on 'HrTimer' that can only be called with exclusive
access to an unarmed timer, or from timer callback context.
- Add arithmetic operations to 'Instant' and 'Delta'.
- Add a few convenience and access methods to 'HrTimer' and
'Instant'.
'macros' crate:
- Reduce collections in 'quote!' macro.
And a few other cleanups and improvements"
* tag 'rust-6.18' of git://git.kernel.org/pub/scm/linux/kernel/git/ojeda/linux: (58 commits)
gpu: nova-core: use Alignment for alignment-related operations
rust: add `Alignment` type
rust: macros: reduce collections in `quote!` macro
rust: acpi: use `core::ffi::CStr` method names
rust: of: use `core::ffi::CStr` method names
rust: net: use `core::ffi::CStr` method names
rust: miscdevice: use `core::ffi::CStr` method names
rust: kunit: use `core::ffi::CStr` method names
rust: firmware: use `core::ffi::CStr` method names
rust: drm: use `core::ffi::CStr` method names
rust: cpufreq: use `core::ffi::CStr` method names
rust: configfs: use `core::ffi::CStr` method names
rust: auxiliary: use `core::ffi::CStr` method names
drm/panic: use `core::ffi::CStr` method names
rust: device: use `kernel::{fmt,prelude::fmt!}`
rust: sync: use `kernel::{fmt,prelude::fmt!}`
rust: seq_file: use `kernel::{fmt,prelude::fmt!}`
rust: kunit: use `kernel::{fmt,prelude::fmt!}`
rust: file: use `kernel::{fmt,prelude::fmt!}`
rust: device: use `kernel::{fmt,prelude::fmt!}`
...
Diffstat (limited to 'rust/kernel')
44 files changed, 958 insertions, 228 deletions
diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs index 7ae317368b00..37e1161c1298 100644 --- a/rust/kernel/acpi.rs +++ b/rust/kernel/acpi.rs @@ -37,11 +37,8 @@ impl DeviceId { /// Create a new device id from an ACPI 'id' string. #[inline(always)] pub const fn new(id: &'static CStr) -> Self { - build_assert!( - id.len_with_nul() <= Self::ACPI_ID_LEN, - "ID exceeds 16 bytes" - ); - let src = id.as_bytes_with_nul(); + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::ACPI_ID_LEN, "ID exceeds 16 bytes"); // Replace with `bindings::acpi_device_id::default()` once stabilized for `const`. // SAFETY: FFI type is valid to be zero-initialized. let mut acpi: bindings::acpi_device_id = unsafe { core::mem::zeroed() }; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index a2c49e5494d3..9c154209423c 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -2,18 +2,11 @@ //! Implementation of the kernel's memory allocation infrastructure. -#[cfg(not(any(test, testlib)))] pub mod allocator; pub mod kbox; pub mod kvec; pub mod layout; -#[cfg(any(test, testlib))] -pub mod allocator_test; - -#[cfg(any(test, testlib))] -pub use self::allocator_test as allocator; - pub use self::kbox::Box; pub use self::kbox::KBox; pub use self::kbox::KVBox; @@ -137,6 +130,14 @@ pub mod flags { /// - Implementers must ensure that all trait functions abide by the guarantees documented in the /// `# Guarantees` sections. pub unsafe trait Allocator { + /// The minimum alignment satisfied by all allocations from this allocator. + /// + /// # Guarantees + /// + /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if + /// the requested layout has a smaller alignment. + const MIN_ALIGN: usize; + /// Allocate memory based on `layout` and `flags`. /// /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 2692cf90c948..869d9fd69527 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -17,6 +17,8 @@ use crate::alloc::{AllocError, Allocator}; use crate::bindings; use crate::pr_warn; +const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; + /// The contiguous kernel allocator. /// /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also @@ -128,6 +130,8 @@ impl Kmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Kmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, @@ -147,6 +151,8 @@ unsafe impl Allocator for Kmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Vmalloc { + const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, @@ -171,6 +177,8 @@ unsafe impl Allocator for Vmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for KVmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option<NonNull<u8>>, diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs deleted file mode 100644 index 90dd987d40e4..000000000000 --- a/rust/kernel/alloc/allocator_test.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users -//! of those types (e.g. `CString`) use kernel allocators for instantiation. -//! -//! In order to allow userspace test cases to make use of such types as well, implement the -//! `Cmalloc` allocator within the `allocator_test` module and type alias all kernel allocators to -//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend. - -#![allow(missing_docs)] - -use super::{flags::*, AllocError, Allocator, Flags}; -use core::alloc::Layout; -use core::cmp; -use core::ptr; -use core::ptr::NonNull; - -/// The userspace allocator based on libc. -pub struct Cmalloc; - -pub type Kmalloc = Cmalloc; -pub type Vmalloc = Kmalloc; -pub type KVmalloc = Kmalloc; - -impl Cmalloc { - /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of - /// `layout`. - pub fn aligned_layout(layout: Layout) -> Layout { - // Note that `layout.size()` (after padding) is guaranteed to be a multiple of - // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return - // a properly aligned object (see comments in `kmalloc()` for more information). - layout.pad_to_align() - } -} - -extern "C" { - #[link_name = "aligned_alloc"] - fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void; - - #[link_name = "free"] - fn libc_free(ptr: *mut crate::ffi::c_void); -} - -// SAFETY: -// - memory remains valid until it is explicitly freed, -// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, -// - `realloc` provides the guarantees as provided in the `# Guarantees` section. -unsafe impl Allocator for Cmalloc { - unsafe fn realloc( - ptr: Option<NonNull<u8>>, - layout: Layout, - old_layout: Layout, - flags: Flags, - ) -> Result<NonNull<[u8]>, AllocError> { - let src = match ptr { - Some(src) => { - if old_layout.size() == 0 { - ptr::null_mut() - } else { - src.as_ptr() - } - } - None => ptr::null_mut(), - }; - - if layout.size() == 0 { - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - return Ok(NonNull::slice_from_raw_parts( - crate::alloc::dangling_from_layout(layout), - 0, - )); - } - - // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`: - // - // > The value of alignment shall be a valid alignment supported by the implementation - // [...]. - // - // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE - // 1003.1-2001) defines `posix_memalign`: - // - // > The value of alignment shall be a power of two multiple of sizeof (void *). - // - // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time - // of writing, this is known to be the case on macOS (but not in glibc). - // - // Satisfy the stricter requirement to avoid spurious test failures on some platforms. - let min_align = core::mem::size_of::<*const crate::ffi::c_void>(); - let layout = layout.align_to(min_align).map_err(|_| AllocError)?; - let layout = layout.pad_to_align(); - - // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or - // exceeds the given size and alignment requirements. - let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) }.cast::<u8>(); - let dst = NonNull::new(dst).ok_or(AllocError)?; - - if flags.contains(__GFP_ZERO) { - // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` - // guarantee that `dst` points to memory of at least `layout.size()` bytes. - unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; - } - - if !src.is_null() { - // SAFETY: - // - `src` has previously been allocated with this `Allocator`; `dst` has just been - // newly allocated, hence the memory regions do not overlap. - // - both` src` and `dst` are properly aligned and valid for reads and writes - unsafe { - ptr::copy_nonoverlapping( - src, - dst.as_ptr(), - cmp::min(layout.size(), old_layout.size()), - ) - }; - } - - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - Ok(NonNull::slice_from_raw_parts(dst, layout.size())) - } -} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 856d05aa60f1..27c4b5a9b61d 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -7,7 +7,6 @@ use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; use super::{AllocError, Allocator, Flags}; use core::alloc::Layout; use core::borrow::{Borrow, BorrowMut}; -use core::fmt; use core::marker::PhantomData; use core::mem::ManuallyDrop; use core::mem::MaybeUninit; @@ -17,6 +16,7 @@ use core::ptr::NonNull; use core::result::Result; use crate::ffi::c_void; +use crate::fmt; use crate::init::InPlaceInit; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; @@ -290,6 +290,83 @@ where Ok(Self::new(x, flags)?.into()) } + /// Construct a pinned slice of elements `Pin<Box<[T], A>>`. + /// + /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks + /// or mutexes. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{new_spinlock, SpinLock}; + /// + /// struct Inner { + /// a: u32, + /// b: u32, + /// } + /// + /// #[pin_data] + /// struct Example { + /// c: u32, + /// #[pin] + /// d: SpinLock<Inner>, + /// } + /// + /// impl Example { + /// fn new() -> impl PinInit<Self, Error> { + /// try_pin_init!(Self { + /// c: 10, + /// d <- new_spinlock!(Inner { a: 20, b: 30 }), + /// }) + /// } + /// } + /// + /// // Allocate a boxed slice of 10 `Example`s. + /// let s = KBox::pin_slice( + /// | _i | Example::new(), + /// 10, + /// GFP_KERNEL + /// )?; + /// + /// assert_eq!(s[5].c, 10); + /// assert_eq!(s[3].d.lock().a, 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pin_slice<Func, Item, E>( + mut init: Func, + len: usize, + flags: Flags, + ) -> Result<Pin<Box<[T], A>>, E> + where + Func: FnMut(usize) -> Item, + Item: PinInit<T, E>, + E: From<AllocError>, + { + let mut buffer = super::Vec::<T, A>::with_capacity(len, flags)?; + for i in 0..len { + let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast(); + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory. + // - `ptr` is not used if an error is returned. + // - `ptr` won't be moved until it is dropped, i.e. it is pinned. + unsafe { init(i).__pinned_init(ptr)? }; + + // SAFETY: + // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to + // `with_capacity()` above. + // - The new value at index buffer.len() + 1 is the only element being added here, and + // it has been initialized above by `init(i).__pinned_init(ptr)`. + unsafe { buffer.inc_len(1) }; + } + + let (ptr, _, _) = buffer.into_raw_parts(); + let slice = core::ptr::slice_from_raw_parts_mut(ptr, len); + + // SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid + // `[T]`. + Ok(Pin::from(unsafe { Box::from_raw(slice) })) + } + /// Convert a [`Box<T,A>`] to a [`Pin<Box<T,A>>`]. If `T` does not implement /// [`Unpin`], then `x` will be pinned in memory and can't be moved. pub fn into_pin(this: Self) -> Pin<Self> { @@ -401,12 +478,17 @@ where } // SAFETY: The pointer returned by `into_foreign` comes from a well aligned -// pointer to `T`. +// pointer to `T` allocated by `A`. unsafe impl<T: 'static, A> ForeignOwnable for Box<T, A> where A: Allocator, { - const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); + const FOREIGN_ALIGN: usize = if core::mem::align_of::<T>() < A::MIN_ALIGN { + A::MIN_ALIGN + } else { + core::mem::align_of::<T>() + }; + type Borrowed<'a> = &'a T; type BorrowedMut<'a> = &'a mut T; @@ -435,12 +517,12 @@ where } // SAFETY: The pointer returned by `into_foreign` comes from a well aligned -// pointer to `T`. +// pointer to `T` allocated by `A`. unsafe impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>> where A: Allocator, { - const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); + const FOREIGN_ALIGN: usize = <Box<T, A> as ForeignOwnable>::FOREIGN_ALIGN; type Borrowed<'a> = Pin<&'a T>; type BorrowedMut<'a> = Pin<&'a mut T>; diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 3c72e0bdddb8..dfc101e03f35 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -7,9 +7,9 @@ use super::{ layout::ArrayLayout, AllocError, Allocator, Box, Flags, }; +use crate::fmt; use core::{ borrow::{Borrow, BorrowMut}, - fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::Deref, @@ -175,7 +175,7 @@ where /// Returns the number of elements that can be stored within the vector without allocating /// additional memory. - pub fn capacity(&self) -> usize { + pub const fn capacity(&self) -> usize { if const { Self::is_zst() } { usize::MAX } else { @@ -185,7 +185,7 @@ where /// Returns the number of elements stored within the vector. #[inline] - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.len } @@ -196,7 +196,7 @@ where /// - `additional` must be less than or equal to `self.capacity - self.len`. /// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized. #[inline] - pub unsafe fn inc_len(&mut self, additional: usize) { + pub const unsafe fn inc_len(&mut self, additional: usize) { // Guaranteed by the type invariant to never underflow. debug_assert!(additional <= self.capacity() - self.len()); // INVARIANT: By the safety requirements of this method this represents the exact number of @@ -224,6 +224,16 @@ where } /// Returns a slice of the entire vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(v.as_slice(), &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` #[inline] pub fn as_slice(&self) -> &[T] { self @@ -245,7 +255,7 @@ where /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw /// pointer. #[inline] - pub fn as_ptr(&self) -> *const T { + pub const fn as_ptr(&self) -> *const T { self.ptr.as_ptr() } @@ -261,7 +271,7 @@ where /// assert!(!v.is_empty()); /// ``` #[inline] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.len() == 0 } @@ -1294,7 +1304,7 @@ impl<'vec, T> Drop for DrainAll<'vec, T> { } } -#[macros::kunit_tests(rust_kvec_kunit)] +#[macros::kunit_tests(rust_kvec)] mod tests { use super::*; use crate::prelude::*; diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs index 348b8d27e102..21a920a4b09b 100644 --- a/rust/kernel/alloc/kvec/errors.rs +++ b/rust/kernel/alloc/kvec/errors.rs @@ -2,7 +2,7 @@ //! Errors for the [`Vec`] type. -use core::fmt::{self, Debug, Formatter}; +use kernel::fmt::{self, Debug, Formatter}; use kernel::prelude::*; /// Error type for [`Vec::push_within_capacity`]. diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 93ed514f7cc7..52cbf61c4539 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -80,7 +80,7 @@ impl<T> ArrayLayout<T> { /// # Safety /// /// `len` must be a value, for which `len * size_of::<T>() <= isize::MAX` is true. - pub unsafe fn new_unchecked(len: usize) -> Self { + pub const unsafe fn new_unchecked(len: usize) -> Self { // INVARIANT: By the safety requirements of this function // `len * size_of::<T>() <= isize::MAX`. Self { diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 4749fb6bffef..58be09871397 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -105,8 +105,8 @@ pub struct DeviceId(bindings::auxiliary_device_id); impl DeviceId { /// Create a new [`DeviceId`] from name. pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self { - let name = name.as_bytes_with_nul(); - let modname = modname.as_bytes_with_nul(); + let name = name.to_bytes_with_nul(); + let modname = modname.to_bytes_with_nul(); // TODO: Replace with `bindings::auxiliary_device_id::default()` once stabilized for // `const`. diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs index 831445d37181..61ea35bba7d5 100644 --- a/rust/kernel/block/mq.rs +++ b/rust/kernel/block/mq.rs @@ -82,7 +82,7 @@ //! Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?; //! let mut disk = gen_disk::GenDiskBuilder::new() //! .capacity_sectors(4096) -//! .build(format_args!("myblk"), tagset)?; +//! .build(fmt!("myblk"), tagset)?; //! //! # Ok::<(), kernel::error::Error>(()) //! ``` diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs index cd54cd64ea88..be92d0e5f031 100644 --- a/rust/kernel/block/mq/gen_disk.rs +++ b/rust/kernel/block/mq/gen_disk.rs @@ -3,12 +3,12 @@ //! Generic disk abstraction. //! //! C header: [`include/linux/blkdev.h`](srctree/include/linux/blkdev.h) -//! C header: [`include/linux/blk_mq.h`](srctree/include/linux/blk_mq.h) +//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) use crate::block::mq::{raw_writer::RawWriter, Operations, TagSet}; +use crate::fmt::{self, Write}; use crate::{bindings, error::from_err_ptr, error::Result, sync::Arc}; use crate::{error, static_lock_class}; -use core::fmt::{self, Write}; /// A builder for [`GenDisk`]. /// diff --git a/rust/kernel/block/mq/raw_writer.rs b/rust/kernel/block/mq/raw_writer.rs index 7e2159e4f6a6..d311e24e2595 100644 --- a/rust/kernel/block/mq/raw_writer.rs +++ b/rust/kernel/block/mq/raw_writer.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 -use core::fmt::{self, Write}; - use crate::error::Result; +use crate::fmt::{self, Write}; use crate::prelude::EINVAL; /// A mutable reference to a byte buffer where a string can be written into. diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 2736b798cdc6..9fb5ef825e41 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -263,7 +263,7 @@ impl<Data> Group<Data> { try_pin_init!(Self { group <- pin_init::init_zeroed().chain(|v: &mut Opaque<bindings::config_group>| { let place = v.get(); - let name = name.as_bytes_with_nul().as_ptr(); + let name = name.to_bytes_with_nul().as_ptr(); // SAFETY: It is safe to initialize a group once it has been zeroed. unsafe { bindings::config_group_init_type_name(place, name.cast(), item_type.as_ptr()) @@ -613,7 +613,7 @@ where pub const fn new(name: &'static CStr) -> Self { Self { attribute: Opaque::new(bindings::configfs_attribute { - ca_name: name.as_char_ptr(), + ca_name: crate::str::as_char_ptr_in_const_context(name), ca_owner: core::ptr::null_mut(), ca_mode: 0o660, show: Some(Self::show), diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs index 5de730c8d817..cb6c0338ef5a 100644 --- a/rust/kernel/cpu.rs +++ b/rust/kernel/cpu.rs @@ -109,6 +109,7 @@ impl CpuId { /// unexpectedly due to preemption or CPU migration. It should only be /// used when the context ensures that the task remains on the same CPU /// or the users could use a stale (yet valid) CPU ID. + #[inline] pub fn current() -> Self { // SAFETY: raw_smp_processor_id() always returns a valid CPU ID. unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index afc15e72a7c3..86c02e81729e 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -27,7 +27,6 @@ use crate::clk::Clk; use core::{ cell::UnsafeCell, marker::PhantomData, - mem::MaybeUninit, ops::{Deref, DerefMut}, pin::Pin, ptr, @@ -1013,12 +1012,11 @@ impl<T: Driver> Registration<T> { } else { None }, - // SAFETY: All zeros is a valid value for `bindings::cpufreq_driver`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }; const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { - let src = name.as_bytes_with_nul(); + let src = name.to_bytes_with_nul(); let mut dst = [0; CPUFREQ_NAME_LEN]; build_assert!(src.len() <= CPUFREQ_NAME_LEN); diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index a1db49eb159a..9cefa0810e0b 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -5,10 +5,10 @@ //! C header: [`include/linux/device.h`](srctree/include/linux/device.h) use crate::{ - bindings, + bindings, fmt, types::{ARef, ForeignOwnable, Opaque}, }; -use core::{fmt, marker::PhantomData, ptr}; +use core::{marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; @@ -596,7 +596,7 @@ macro_rules! impl_device_context_into_aref { macro_rules! dev_printk { ($method:ident, $dev:expr, $($f:tt)*) => { { - ($dev).$method(::core::format_args!($($f)*)); + ($dev).$method($crate::prelude::fmt!($($f)*)); } } } diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs index 49ee12a906db..3a332a8c53a9 100644 --- a/rust/kernel/device/property.rs +++ b/rust/kernel/device/property.rs @@ -11,6 +11,7 @@ use crate::{ alloc::KVec, bindings, error::{to_result, Result}, + fmt, prelude::*, str::{CStr, CString}, types::{ARef, Opaque}, @@ -68,16 +69,16 @@ impl FwNode { unsafe { bindings::is_of_node(self.as_raw()) } } - /// Returns an object that implements [`Display`](core::fmt::Display) for + /// Returns an object that implements [`Display`](fmt::Display) for /// printing the name of a node. /// /// This is an alternative to the default `Display` implementation, which /// prints the full path. - pub fn display_name(&self) -> impl core::fmt::Display + '_ { + pub fn display_name(&self) -> impl fmt::Display + '_ { struct FwNodeDisplayName<'a>(&'a FwNode); - impl core::fmt::Display for FwNodeDisplayName<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + impl fmt::Display for FwNodeDisplayName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: `self` is valid by its type invariant. let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) }; if name.is_null() { @@ -87,7 +88,7 @@ impl FwNode { // - `fwnode_get_name` returns null or a valid C string. // - `name` was checked to be non-null. let name = unsafe { CStr::from_char_ptr(name) }; - write!(f, "{name}") + fmt::Display::fmt(name, f) } } @@ -351,8 +352,8 @@ impl FwNodeReferenceArgs { } } -impl core::fmt::Debug for FwNodeReferenceArgs { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Debug for FwNodeReferenceArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.as_slice()) } } @@ -377,8 +378,8 @@ enum Node<'a> { Owned(ARef<FwNode>), } -impl core::fmt::Display for FwNode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Display for FwNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // The logic here is the same as the one in lib/vsprintf.c // (fwnode_full_name_string). @@ -413,9 +414,9 @@ impl core::fmt::Display for FwNode { // SAFETY: `fwnode_get_name_prefix` returns null or a // valid C string. let prefix = unsafe { CStr::from_char_ptr(prefix) }; - write!(f, "{prefix}")?; + fmt::Display::fmt(prefix, f)?; } - write!(f, "{}", fwnode.display_name())?; + fmt::Display::fmt(&fwnode.display_name(), f)?; } Ok(()) diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2bc8ab51ec28..68fe67624424 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -9,8 +9,8 @@ use crate::{ device::{Bound, Core}, error::{to_result, Result}, prelude::*, + sync::aref::ARef, transmute::{AsBytes, FromBytes}, - types::ARef, }; /// Trait to be implemented by DMA capable bus devices. diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index d29c477e89a8..0956ba0f64de 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -2,7 +2,7 @@ //! DRM device. //! -//! C header: [`include/linux/drm/drm_device.h`](srctree/include/linux/drm/drm_device.h) +//! C header: [`include/drm/drm_device.h`](srctree/include/drm/drm_device.h) use crate::{ alloc::allocator::Kmalloc, @@ -82,8 +82,8 @@ impl<T: drm::Driver> Device<T> { major: T::INFO.major, minor: T::INFO.minor, patchlevel: T::INFO.patchlevel, - name: T::INFO.name.as_char_ptr().cast_mut(), - desc: T::INFO.desc.as_char_ptr().cast_mut(), + name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), + desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), driver_features: drm::driver::FEAT_GEM, ioctls: T::IOCTLS.as_ptr(), diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index fe7e8d06961a..d2dad77274c4 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -2,7 +2,7 @@ //! DRM driver core. //! -//! C header: [`include/linux/drm/drm_drv.h`](srctree/include/linux/drm/drm_drv.h) +//! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) use crate::{ bindings, device, devres, drm, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index e8789c9110d6..8c46f8d51951 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -2,7 +2,7 @@ //! DRM File objects. //! -//! C header: [`include/linux/drm/drm_file.h`](srctree/include/linux/drm/drm_file.h) +//! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) use crate::{bindings, drm, error::Result, prelude::*, types::Opaque}; use core::marker::PhantomData; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index b71821cfb5ea..b9f3248876ba 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -2,7 +2,7 @@ //! DRM GEM API //! -//! C header: [`include/linux/drm/drm_gem.h`](srctree/include/linux/drm/drm_gem.h) +//! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) use crate::{ alloc::flags::*, diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index fdec01c37168..8431cdcd3ae0 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -2,7 +2,7 @@ //! DRM IOCTL definitions. //! -//! C header: [`include/linux/drm/drm_ioctl.h`](srctree/include/linux/drm/drm_ioctl.h) +//! C header: [`include/drm/drm_ioctl.h`](srctree/include/drm/drm_ioctl.h) use crate::ioctl; diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index a41de293dcd1..1c0e0e241daa 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -2,7 +2,9 @@ //! Kernel errors. //! -//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h) +//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)\ +//! C header: [`include/uapi/asm-generic/errno.h`](srctree/include/uapi/asm-generic/errno.h)\ +//! C header: [`include/linux/errno.h`](srctree/include/linux/errno.h) use crate::{ alloc::{layout::LayoutError, AllocError}, @@ -101,8 +103,23 @@ pub struct Error(NonZeroI32); impl Error { /// Creates an [`Error`] from a kernel error code. /// - /// It is a bug to pass an out-of-range `errno`. `EINVAL` would - /// be returned in such a case. + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + /// + /// It is a bug to pass an out-of-range `errno`. [`code::EINVAL`] is returned in such a case. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Error::from_errno(-1), EPERM); + /// assert_eq!(Error::from_errno(-2), ENOENT); + /// ``` + /// + /// The following calls are considered a bug: + /// + /// ``` + /// assert_eq!(Error::from_errno(0), EINVAL); + /// assert_eq!(Error::from_errno(-1000000), EINVAL); + /// ``` pub fn from_errno(errno: crate::ffi::c_int) -> Error { if let Some(error) = Self::try_from_errno(errno) { error @@ -158,7 +175,7 @@ impl Error { } /// Returns a string representing the error, if one exists. - #[cfg(not(any(test, testlib)))] + #[cfg(not(testlib))] pub fn name(&self) -> Option<&'static CStr> { // SAFETY: Just an FFI call, there are no extra safety requirements. let ptr = unsafe { bindings::errname(-self.0.get()) }; @@ -175,7 +192,7 @@ impl Error { /// When `testlib` is configured, this always returns `None` to avoid the dependency on a /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still /// run in userspace. - #[cfg(any(test, testlib))] + #[cfg(testlib)] pub fn name(&self) -> Option<&'static CStr> { None } @@ -375,8 +392,43 @@ impl From<core::convert::Infallible> for Error { /// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html pub type Result<T = (), E = Error> = core::result::Result<T, E>; -/// Converts an integer as returned by a C kernel function to an error if it's negative, and -/// `Ok(())` otherwise. +/// Converts an integer as returned by a C kernel function to a [`Result`]. +/// +/// If the integer is negative, an [`Err`] with an [`Error`] as given by [`Error::from_errno`] is +/// returned. This means the integer must be `>= -MAX_ERRNO`. +/// +/// Otherwise, it returns [`Ok`]. +/// +/// It is a bug to pass an out-of-range negative integer. `Err(EINVAL)` is returned in such a case. +/// +/// # Examples +/// +/// This function may be used to easily perform early returns with the [`?`] operator when working +/// with C APIs within Rust abstractions: +/// +/// ``` +/// # use kernel::error::to_result; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn f1() -> c_int { 0 } +/// # pub(super) unsafe fn f2() -> c_int { EINVAL.to_errno() } +/// # } +/// fn f() -> Result { +/// // SAFETY: ... +/// to_result(unsafe { bindings::f1() })?; +/// +/// // SAFETY: ... +/// to_result(unsafe { bindings::f2() })?; +/// +/// // ... +/// +/// Ok(()) +/// } +/// # assert_eq!(f(), Err(EINVAL)); +/// ``` +/// +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator pub fn to_result(err: crate::ffi::c_int) -> Result { if err < 0 { Err(Error::from_errno(err)) diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 1abab5b2f052..94e6bb88b903 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -291,7 +291,7 @@ impl<const N: usize> ModInfoBuilder<N> { let module_name = this.module_name; if !this.module_name.is_empty() { - this = this.push_internal(module_name.as_bytes_with_nul()); + this = this.push_internal(module_name.to_bytes_with_nul()); if N != 0 { // Re-use the space taken by the NULL terminator and swap it with the '.' separator. diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs index f1a3fa698745..cf06e73a6da0 100644 --- a/rust/kernel/fs/file.rs +++ b/rust/kernel/fs/file.rs @@ -11,6 +11,7 @@ use crate::{ bindings, cred::Credential, error::{code::*, to_result, Error, Result}, + fmt, sync::aref::{ARef, AlwaysRefCounted}, types::{NotThreadSafe, Opaque}, }; @@ -460,8 +461,8 @@ impl From<BadFdError> for Error { } } -impl core::fmt::Debug for BadFdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Debug for BadFdError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("EBADF") } } diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 41efd87595d6..3a43886cc14e 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -6,8 +6,8 @@ //! //! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html> +use crate::fmt; use crate::prelude::*; -use core::fmt; #[cfg(CONFIG_PRINTK)] use crate::c_str; @@ -74,14 +74,14 @@ macro_rules! kunit_assert { // mistake (it is hidden to prevent that). // // This mimics KUnit's failed assertion format. - $crate::kunit::err(format_args!( + $crate::kunit::err($crate::prelude::fmt!( " # {}: ASSERTION FAILED at {FILE}:{LINE}\n", $name )); - $crate::kunit::err(format_args!( + $crate::kunit::err($crate::prelude::fmt!( " Expected {CONDITION} to be true, but is false\n" )); - $crate::kunit::err(format_args!( + $crate::kunit::err($crate::prelude::fmt!( " Failure not reported to KUnit since this is a non-KUnit task\n" )); break 'out; @@ -102,12 +102,12 @@ macro_rules! kunit_assert { unsafe impl Sync for UnaryAssert {} static LOCATION: Location = Location($crate::bindings::kunit_loc { - file: FILE.as_char_ptr(), + file: $crate::str::as_char_ptr_in_const_context(FILE), line: LINE, }); static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert { assert: $crate::bindings::kunit_assert {}, - condition: CONDITION.as_char_ptr(), + condition: $crate::str::as_char_ptr_in_const_context(CONDITION), expected_true: true, }); @@ -202,7 +202,7 @@ pub const fn kunit_case( ) -> kernel::bindings::kunit_case { kernel::bindings::kunit_case { run_case: Some(run_case), - name: name.as_char_ptr(), + name: kernel::str::as_char_ptr_in_const_context(name), attr: kernel::bindings::kunit_attributes { speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, }, diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index fef97f2a5098..f910a5ab80ba 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -17,6 +17,7 @@ // the unstable features in use. // // Stable since Rust 1.79.0. +#![feature(generic_nonzero)] #![feature(inline_const)] // // Stable since Rust 1.81.0. @@ -28,6 +29,7 @@ // Stable since Rust 1.83.0. #![feature(const_maybe_uninit_as_mut_ptr)] #![feature(const_mut_refs)] +#![feature(const_option)] #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] // @@ -110,6 +112,7 @@ pub mod pid_namespace; pub mod platform; pub mod prelude; pub mod print; +pub mod ptr; pub mod rbtree; pub mod regulator; pub mod revocable; @@ -206,7 +209,7 @@ impl ThisModule { } } -#[cfg(not(any(testlib, test)))] +#[cfg(not(testlib))] #[panic_handler] fn panic(info: &core::panic::PanicInfo<'_>) -> ! { pr_emerg!("{}\n", info); diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 44e5219cfcbc..7355bbac16a7 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -38,6 +38,8 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// /// # Examples /// +/// Use [`ListLinks`] as the type of the intrusive field. +/// /// ``` /// use kernel::list::*; /// @@ -140,6 +142,124 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// } /// # Result::<(), Error>::Ok(()) /// ``` +/// +/// Use [`ListLinksSelfPtr`] as the type of the intrusive field. This allows a list of trait object +/// type. +/// +/// ``` +/// use kernel::list::*; +/// +/// trait Foo { +/// fn foo(&self) -> (&'static str, i32); +/// } +/// +/// #[pin_data] +/// struct DTWrap<T: ?Sized> { +/// #[pin] +/// links: ListLinksSelfPtr<DTWrap<dyn Foo>>, +/// value: T, +/// } +/// +/// impl<T> DTWrap<T> { +/// fn new(value: T) -> Result<ListArc<Self>> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinksSelfPtr::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// impl_list_arc_safe! { +/// impl{T: ?Sized} ListArcSafe<0> for DTWrap<T> { untracked; } +/// } +/// impl_list_item! { +/// impl ListItem<0> for DTWrap<dyn Foo> { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// // Create a new empty list. +/// let mut list = List::<DTWrap<dyn Foo>>::new(); +/// { +/// assert!(list.is_empty()); +/// } +/// +/// struct A(i32); +/// // `A` returns the inner value for `foo`. +/// impl Foo for A { fn foo(&self) -> (&'static str, i32) { ("a", self.0) } } +/// +/// struct B; +/// // `B` always returns 42. +/// impl Foo for B { fn foo(&self) -> (&'static str, i32) { ("b", 42) } } +/// +/// // Insert 3 element using `push_back()`. +/// list.push_back(DTWrap::new(A(15))?); +/// list.push_back(DTWrap::new(A(32))?); +/// list.push_back(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [A(15), A(32), B] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_back()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// } +/// +/// // Insert 3 elements using `push_front()`. +/// list.push_front(DTWrap::new(A(15))?); +/// list.push_front(DTWrap::new(A(32))?); +/// list.push_front(DTWrap::new(B)?); +/// +/// // Iterate over the list to verify the nodes were inserted correctly. +/// // [B, A(32), A(15)] +/// { +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert!(iter.next().is_none()); +/// +/// // Verify the length of the list. +/// assert_eq!(list.iter().count(), 3); +/// } +/// +/// // Pop the items from the list using `pop_front()` and verify the content. +/// { +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); +/// } +/// +/// // Push `list2` to `list` through `push_all_back()`. +/// // list: [B] +/// // list2: [B, A(25)] +/// { +/// let mut list2 = List::<DTWrap<dyn Foo>>::new(); +/// list2.push_back(DTWrap::new(B)?); +/// list2.push_back(DTWrap::new(A(25))?); +/// +/// list.push_all_back(&mut list2); +/// +/// // list: [B, B, A(25)] +/// // list2: [] +/// let mut iter = list.iter(); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("b", 42)); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value.foo(), ("a", 25)); +/// assert!(iter.next().is_none()); +/// assert!(list2.is_empty()); +/// } +/// # Result::<(), Error>::Ok(()) +/// ``` pub struct List<T: ?Sized + ListItem<ID>, const ID: u64 = 0> { first: *mut ListLinksFields, _ty: PhantomData<ListArc<T, ID>>, diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 6373fe183b27..d3aa7d25afad 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -34,7 +34,7 @@ impl MiscDeviceOptions { // SAFETY: All zeros is valid for this C type. let mut result: bindings::miscdevice = unsafe { MaybeUninit::zeroed().assume_init() }; result.minor = bindings::MISC_DYNAMIC_MINOR as ffi::c_int; - result.name = self.name.as_char_ptr(); + result.name = crate::str::as_char_ptr_in_const_context(self.name); result.fops = MiscdeviceVTable::<T>::build(); result } diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 7de5cc7a0eee..be1027b7961b 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -497,7 +497,7 @@ unsafe impl Sync for DriverVTable {} pub const fn create_phy_driver<T: Driver>() -> DriverVTable { // INVARIANT: All the fields of `struct phy_driver` are initialized properly. DriverVTable(Opaque::new(bindings::phy_driver { - name: T::NAME.as_char_ptr().cast_mut(), + name: crate::str::as_char_ptr_in_const_context(T::NAME).cast_mut(), flags: T::FLAGS, phy_id: T::PHY_DEVICE_ID.id(), phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index b76b35265df2..58b20c367f99 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -34,7 +34,7 @@ unsafe impl RawDeviceIdIndex for DeviceId { impl DeviceId { /// Create a new device id from an OF 'compatible' string. pub const fn new(compatible: &'static CStr) -> Self { - let src = compatible.as_bytes_with_nul(); + let src = compatible.to_bytes_with_nul(); // Replace with `bindings::of_device_id::default()` once stabilized for `const`. // SAFETY: FFI type is valid to be zero-initialized. let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() }; diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 25fe97aafd02..198d09a31449 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -12,7 +12,10 @@ //! ``` #[doc(no_inline)] -pub use core::pin::Pin; +pub use core::{ + mem::{align_of, align_of_val, size_of, size_of_val}, + pin::Pin, +}; pub use ::ffi::{ c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong, diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs new file mode 100644 index 000000000000..2e5e2a090480 --- /dev/null +++ b/rust/kernel/ptr.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types and functions to work with pointers and addresses. + +use core::fmt::Debug; +use core::mem::align_of; +use core::num::NonZero; + +use crate::build_assert; + +/// Type representing an alignment, which is always a power of two. +/// +/// It is used to validate that a given value is a valid alignment, and to perform masking and +/// alignment operations. +/// +/// This is a temporary substitute for the [`Alignment`] nightly type from the standard library, +/// and to be eventually replaced by it. +/// +/// [`Alignment`]: https://github.com/rust-lang/rust/issues/102070 +/// +/// # Invariants +/// +/// An alignment is always a power of two. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Alignment(NonZero<usize>); + +impl Alignment { + /// Validates that `ALIGN` is a power of two at build-time, and returns an [`Alignment`] of the + /// same value. + /// + /// A build error is triggered if `ALIGN` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// let v = Alignment::new::<16>(); + /// assert_eq!(v.as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn new<const ALIGN: usize>() -> Self { + build_assert!( + ALIGN.is_power_of_two(), + "Provided alignment is not a power of two." + ); + + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Self(unsafe { NonZero::new_unchecked(ALIGN) }) + } + + /// Validates that `align` is a power of two at runtime, and returns an + /// [`Alignment`] of the same value. + /// + /// Returns [`None`] if `align` is not a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new_checked(16), Some(Alignment::new::<16>())); + /// assert_eq!(Alignment::new_checked(15), None); + /// assert_eq!(Alignment::new_checked(1), Some(Alignment::new::<1>())); + /// assert_eq!(Alignment::new_checked(0), None); + /// ``` + #[inline(always)] + pub const fn new_checked(align: usize) -> Option<Self> { + if align.is_power_of_two() { + // INVARIANT: `align` is a power of two. + // SAFETY: `align` is a power of two, and thus non-zero. + Some(Self(unsafe { NonZero::new_unchecked(align) })) + } else { + None + } + } + + /// Returns the alignment of `T`. + /// + /// This is equivalent to [`align_of`], but with the return value provided as an [`Alignment`]. + #[inline(always)] + pub const fn of<T>() -> Self { + #![allow(clippy::incompatible_msrv)] + // This cannot panic since alignments are always powers of two. + // + // We unfortunately cannot use `new` as it would require the `generic_const_exprs` feature. + const { Alignment::new_checked(align_of::<T>()).unwrap() } + } + + /// Returns this alignment as a [`usize`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_usize(), 16); + /// ``` + #[inline(always)] + pub const fn as_usize(self) -> usize { + self.as_nonzero().get() + } + + /// Returns this alignment as a [`NonZero`]. + /// + /// It is guaranteed to be a power of two. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<16>().as_nonzero().get(), 16); + /// ``` + #[inline(always)] + pub const fn as_nonzero(self) -> NonZero<usize> { + // Allow the compiler to know that the value is indeed a power of two. This can help + // optimize some operations down the line, like e.g. replacing divisions by bit shifts. + if !self.0.is_power_of_two() { + // SAFETY: Per the invariants, `self.0` is always a power of two so this block will + // never be reached. + unsafe { core::hint::unreachable_unchecked() } + } + self.0 + } + + /// Returns the base-2 logarithm of the alignment. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::of::<u8>().log2(), 0); + /// assert_eq!(Alignment::new::<16>().log2(), 4); + /// ``` + #[inline(always)] + pub const fn log2(self) -> u32 { + self.0.ilog2() + } + + /// Returns the mask for this alignment. + /// + /// This is equivalent to `!(self.as_usize() - 1)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::Alignment; + /// + /// assert_eq!(Alignment::new::<0x10>().mask(), !0xf); + /// ``` + #[inline(always)] + pub const fn mask(self) -> usize { + // No underflow can occur as the alignment is guaranteed to be a power of two, and thus is + // non-zero. + !(self.as_usize() - 1) + } +} + +/// Trait for items that can be aligned against an [`Alignment`]. +pub trait Alignable: Sized { + /// Aligns `self` down to `alignment`. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x2f_usize.align_down(Alignment::new::<0x10>()), 0x20); + /// assert_eq!(0x30usize.align_down(Alignment::new::<0x10>()), 0x30); + /// assert_eq!(0xf0u8.align_down(Alignment::new::<0x1000>()), 0x0); + /// ``` + fn align_down(self, alignment: Alignment) -> Self; + + /// Aligns `self` up to `alignment`, returning `None` if aligning would result in an overflow. + /// + /// # Examples + /// + /// ``` + /// use kernel::ptr::{Alignable, Alignment}; + /// + /// assert_eq!(0x4fusize.align_up(Alignment::new::<0x10>()), Some(0x50)); + /// assert_eq!(0x40usize.align_up(Alignment::new::<0x10>()), Some(0x40)); + /// assert_eq!(0x0usize.align_up(Alignment::new::<0x10>()), Some(0x0)); + /// assert_eq!(u8::MAX.align_up(Alignment::new::<0x10>()), None); + /// assert_eq!(0x10u8.align_up(Alignment::new::<0x100>()), None); + /// assert_eq!(0x0u8.align_up(Alignment::new::<0x100>()), Some(0x0)); + /// ``` + fn align_up(self, alignment: Alignment) -> Option<Self>; +} + +/// Implement [`Alignable`] for unsigned integer types. +macro_rules! impl_alignable_uint { + ($($t:ty),*) => { + $( + impl Alignable for $t { + #[inline(always)] + fn align_down(self, alignment: Alignment) -> Self { + // The operands of `&` need to be of the same type so convert the alignment to + // `Self`. This means we need to compute the mask ourselves. + ::core::num::NonZero::<Self>::try_from(alignment.as_nonzero()) + .map(|align| self & !(align.get() - 1)) + // An alignment larger than `Self` always aligns down to `0`. + .unwrap_or(0) + } + + #[inline(always)] + fn align_up(self, alignment: Alignment) -> Option<Self> { + let aligned_down = self.align_down(alignment); + if self == aligned_down { + Some(aligned_down) + } else { + Self::try_from(alignment.as_usize()) + .ok() + .and_then(|align| aligned_down.checked_add(align)) + } + } + } + )* + }; +} + +impl_alignable_uint!(u8, u16, u32, u64, usize); diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs index 8f199b1a3bb1..59fbfc2473f8 100644 --- a/rust/kernel/seq_file.rs +++ b/rust/kernel/seq_file.rs @@ -4,7 +4,7 @@ //! //! C header: [`include/linux/seq_file.h`](srctree/include/linux/seq_file.h) -use crate::{bindings, c_str, types::NotThreadSafe, types::Opaque}; +use crate::{bindings, c_str, fmt, types::NotThreadSafe, types::Opaque}; /// A utility for generating the contents of a seq file. #[repr(transparent)] @@ -31,7 +31,7 @@ impl SeqFile { /// Used by the [`seq_print`] macro. #[inline] - pub fn call_printf(&self, args: core::fmt::Arguments<'_>) { + pub fn call_printf(&self, args: fmt::Arguments<'_>) { // SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`. unsafe { bindings::seq_printf( @@ -47,7 +47,7 @@ impl SeqFile { #[macro_export] macro_rules! seq_print { ($m:expr, $($arg:tt)+) => ( - $m.call_printf(format_args!($($arg)+)) + $m.call_printf($crate::prelude::fmt!($($arg)+)) ); } pub use seq_print; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 9298993ea7d8..289f77abf415 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -19,6 +19,7 @@ use crate::{ alloc::{AllocError, Flags, KBox}, ffi::c_void, + fmt, init::InPlaceInit, sync::Refcount, try_init, @@ -27,7 +28,6 @@ use crate::{ use core::{ alloc::Layout, borrow::{Borrow, BorrowMut}, - fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, @@ -367,10 +367,10 @@ impl<T: ?Sized> Arc<T> { } } -// SAFETY: The pointer returned by `into_foreign` comes from a well aligned -// pointer to `ArcInner<T>`. +// SAFETY: The pointer returned by `into_foreign` was originally allocated as an +// `KBox<ArcInner<T>>`, so that type is what determines the alignment. unsafe impl<T: 'static> ForeignOwnable for Arc<T> { - const FOREIGN_ALIGN: usize = core::mem::align_of::<ArcInner<T>>(); + const FOREIGN_ALIGN: usize = <KBox<ArcInner<T>> as ForeignOwnable>::FOREIGN_ALIGN; type Borrowed<'a> = ArcBorrow<'a, T>; type BorrowedMut<'a> = Self::Borrowed<'a>; diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index dbd77bb68617..0d24a0432015 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -1,6 +1,21 @@ // SPDX-License-Identifier: GPL-2.0 //! Internal reference counting support. +//! +//! Many C types already have their own reference counting mechanism (e.g. by storing a +//! `refcount_t`). This module provides support for directly using their internal reference count +//! from Rust; instead of making users have to use an additional Rust-reference count in the form of +//! [`Arc`]. +//! +//! The smart pointer [`ARef<T>`] acts similarly to [`Arc<T>`] in that it holds a refcount on the +//! underlying object, but this refcount is internal to the object. It essentially is a Rust +//! implementation of the `get_` and `put_` pattern used in C for reference counting. +//! +//! To make use of [`ARef<MyType>`], `MyType` needs to implement [`AlwaysRefCounted`]. It is a trait +//! for accessing the internal reference count of an object of the `MyType` type. +//! +//! [`Arc`]: crate::sync::Arc +//! [`Arc<T>`]: crate::sync::Arc use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; @@ -97,7 +112,7 @@ impl<T: AlwaysRefCounted> ARef<T> { /// /// ``` /// use core::ptr::NonNull; - /// use kernel::types::{ARef, AlwaysRefCounted}; + /// use kernel::sync::aref::{ARef, AlwaysRefCounted}; /// /// struct Empty {} /// diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 7d0935bc325c..49fad6de0674 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -9,7 +9,8 @@ use crate::{ ffi::{c_int, c_long, c_uint}, mm::MmWithUser, pid_namespace::PidNamespace, - types::{ARef, NotThreadSafe, Opaque}, + sync::aref::ARef, + types::{NotThreadSafe, Opaque}, }; use core::{ cmp::{Eq, PartialEq}, @@ -76,7 +77,7 @@ macro_rules! current { /// incremented when creating `State` and decremented when it is dropped: /// /// ``` -/// use kernel::{task::Task, types::ARef}; +/// use kernel::{task::Task, sync::aref::ARef}; /// /// struct State { /// creator: ARef<Task>, @@ -347,7 +348,7 @@ impl CurrentTask { } // SAFETY: The type invariants guarantee that `Task` is always refcounted. -unsafe impl crate::types::AlwaysRefCounted for Task { +unsafe impl crate::sync::aref::AlwaysRefCounted for Task { #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 64c8dcf548d6..6ea98dfcd027 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -25,6 +25,7 @@ //! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). use core::marker::PhantomData; +use core::ops; pub mod delay; pub mod hrtimer; @@ -200,9 +201,31 @@ impl<C: ClockSource> Instant<C> { pub(crate) fn as_nanos(&self) -> i64 { self.inner } + + /// Create an [`Instant`] from a `ktime_t` without checking if it is non-negative. + /// + /// # Panics + /// + /// On debug builds, this function will panic if `ktime` is not in the range from 0 to + /// `KTIME_MAX`. + /// + /// # Safety + /// + /// The caller promises that `ktime` is in the range from 0 to `KTIME_MAX`. + #[inline] + pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self { + debug_assert!(ktime >= 0); + + // INVARIANT: Our safety contract ensures that `ktime` is in the range from 0 to + // `KTIME_MAX`. + Self { + inner: ktime, + _c: PhantomData, + } + } } -impl<C: ClockSource> core::ops::Sub for Instant<C> { +impl<C: ClockSource> ops::Sub for Instant<C> { type Output = Delta; // By the type invariant, it never overflows. @@ -214,6 +237,46 @@ impl<C: ClockSource> core::ops::Sub for Instant<C> { } } +impl<T: ClockSource> ops::Add<Delta> for Instant<T> { + type Output = Self; + + #[inline] + fn add(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner + rhs.nanos; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + +impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { + type Output = Self; + + #[inline] + fn sub(self, rhs: Delta) -> Self::Output { + // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow + // (e.g. go above `KTIME_MAX`) + let res = self.inner - rhs.nanos; + + // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res >= 0); + + Self { + inner: res, + _c: PhantomData, + } + } +} + /// A span of time. /// /// This struct represents a span of time, with its value stored as nanoseconds. @@ -224,6 +287,78 @@ pub struct Delta { nanos: i64, } +impl ops::Add for Delta { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> Self { + Self { + nanos: self.nanos + rhs.nanos, + } + } +} + +impl ops::AddAssign for Delta { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.nanos += rhs.nanos; + } +} + +impl ops::Sub for Delta { + type Output = Self; + + #[inline] + fn sub(self, rhs: Self) -> Self::Output { + Self { + nanos: self.nanos - rhs.nanos, + } + } +} + +impl ops::SubAssign for Delta { + #[inline] + fn sub_assign(&mut self, rhs: Self) { + self.nanos -= rhs.nanos; + } +} + +impl ops::Mul<i64> for Delta { + type Output = Self; + + #[inline] + fn mul(self, rhs: i64) -> Self::Output { + Self { + nanos: self.nanos * rhs, + } + } +} + +impl ops::MulAssign<i64> for Delta { + #[inline] + fn mul_assign(&mut self, rhs: i64) { + self.nanos *= rhs; + } +} + +impl ops::Div for Delta { + type Output = i64; + + #[inline] + fn div(self, rhs: Self) -> Self::Output { + #[cfg(CONFIG_64BIT)] + { + self.nanos / rhs.nanos + } + + #[cfg(not(CONFIG_64BIT))] + { + // SAFETY: This function is always safe to call regardless of the input values + unsafe { bindings::div64_s64(self.nanos, rhs.nanos) } + } + } +} + impl Delta { /// A span of time equal to zero. pub const ZERO: Self = Self { nanos: 0 }; @@ -312,4 +447,30 @@ impl Delta { bindings::ktime_to_ms(self.as_nanos()) } } + + /// Return `self % dividend` where `dividend` is in nanoseconds. + /// + /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is + /// limited to 32 bit dividends. + #[inline] + pub fn rem_nanos(self, dividend: i32) -> Self { + #[cfg(CONFIG_64BIT)] + { + Self { + nanos: self.as_nanos() % i64::from(dividend), + } + } + + #[cfg(not(CONFIG_64BIT))] + { + let mut rem = 0; + + // SAFETY: `rem` is in the stack, so we can always provide a valid pointer to it. + unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; + + Self { + nanos: i64::from(rem), + } + } + } } diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 144e3b57cc78..856d2d929a00 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -69,9 +69,14 @@ use super::{ClockSource, Delta, Instant}; use crate::{prelude::*, types::Opaque}; -use core::marker::PhantomData; +use core::{marker::PhantomData, ptr::NonNull}; use pin_init::PinInit; +/// A type-alias to refer to the [`Instant<C>`] for a given `T` from [`HrTimer<T>`]. +/// +/// Where `C` is the [`ClockSource`] of the [`HrTimer`]. +pub type HrTimerInstant<T> = Instant<<<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock>; + /// A timer backed by a C `struct hrtimer`. /// /// # Invariants @@ -163,6 +168,84 @@ impl<T> HrTimer<T> { // handled on the C side. unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 } } + + /// Forward the timer expiry for a given timer pointer. + /// + /// # Safety + /// + /// - `self_ptr` must point to a valid `Self`. + /// - The caller must either have exclusive access to the data pointed at by `self_ptr`, or be + /// within the context of the timer callback. + #[inline] + unsafe fn raw_forward(self_ptr: *mut Self, now: HrTimerInstant<T>, interval: Delta) -> u64 + where + T: HasHrTimer<T>, + { + // SAFETY: + // * The C API requirements for this function are fulfilled by our safety contract. + // * `self_ptr` is guaranteed to point to a valid `Self` via our safety contract + unsafe { + bindings::hrtimer_forward(Self::raw_get(self_ptr), now.as_nanos(), interval.as_nanos()) + } + } + + /// Conditionally forward the timer. + /// + /// If the timer expires after `now`, this function does nothing and returns 0. If the timer + /// expired at or before `now`, this function forwards the timer by `interval` until the timer + /// expires after `now` and then returns the number of times the timer was forwarded by + /// `interval`. + /// + /// This function is mainly useful for timer types which can provide exclusive access to the + /// timer when the timer is not running. For forwarding the timer from within the timer callback + /// context, see [`HrTimerCallbackContext::forward()`]. + /// + /// Returns the number of overruns that occurred as a result of the timer expiry change. + pub fn forward(self: Pin<&mut Self>, now: HrTimerInstant<T>, interval: Delta) -> u64 + where + T: HasHrTimer<T>, + { + // SAFETY: `raw_forward` does not move `Self` + let this = unsafe { self.get_unchecked_mut() }; + + // SAFETY: By existence of `Pin<&mut Self>`, the pointer passed to `raw_forward` points to a + // valid `Self` that we have exclusive access to. + unsafe { Self::raw_forward(this, now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`forward()`](Self::forward) that uses an interval after the current + /// time of the base clock for the [`HrTimer`]. + pub fn forward_now(self: Pin<&mut Self>, interval: Delta) -> u64 + where + T: HasHrTimer<T>, + { + self.forward(HrTimerInstant::<T>::now(), interval) + } + + /// Return the time expiry for this [`HrTimer`]. + /// + /// This value should only be used as a snapshot, as the actual expiry time could change after + /// this function is called. + pub fn expires(&self) -> HrTimerInstant<T> + where + T: HasHrTimer<T>, + { + // SAFETY: `self` is an immutable reference and thus always points to a valid `HrTimer`. + let c_timer_ptr = unsafe { HrTimer::raw_get(self) }; + + // SAFETY: + // - Timers cannot have negative ktime_t values as their expiration time. + // - There's no actual locking here, a racy read is fine and expected + unsafe { + Instant::from_ktime( + // This `read_volatile` is intended to correspond to a READ_ONCE call. + // FIXME(read_once): Replace with `read_once` when available on the Rust side. + core::ptr::read_volatile(&raw const ((*c_timer_ptr).node.expires)), + ) + } + } } /// Implemented by pointer types that point to structs that contain a [`HrTimer`]. @@ -300,9 +383,13 @@ pub trait HrTimerCallback { type Pointer<'a>: RawHrTimerCallback; /// Called by the timer logic when the timer fires. - fn run(this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>) -> HrTimerRestart + fn run( + this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>, + ctx: HrTimerCallbackContext<'_, Self>, + ) -> HrTimerRestart where - Self: Sized; + Self: Sized, + Self: HasHrTimer<Self>; } /// A handle representing a potentially running timer. @@ -324,6 +411,8 @@ pub unsafe trait HrTimerHandle { /// Note that the timer might be started by a concurrent start operation. If /// so, the timer might not be in the **stopped** state when this function /// returns. + /// + /// Returns `true` if the timer was running. fn cancel(&mut self) -> bool; } @@ -585,6 +674,63 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> { type Expires = Delta; } +/// Privileged smart-pointer for a [`HrTimer`] callback context. +/// +/// Many [`HrTimer`] methods can only be called in two situations: +/// +/// * When the caller has exclusive access to the `HrTimer` and the `HrTimer` is guaranteed not to +/// be running. +/// * From within the context of an `HrTimer`'s callback method. +/// +/// This type provides access to said methods from within a timer callback context. +/// +/// # Invariants +/// +/// * The existence of this type means the caller is currently within the callback for an +/// [`HrTimer`]. +/// * `self.0` always points to a live instance of [`HrTimer<T>`]. +pub struct HrTimerCallbackContext<'a, T: HasHrTimer<T>>(NonNull<HrTimer<T>>, PhantomData<&'a ()>); + +impl<'a, T: HasHrTimer<T>> HrTimerCallbackContext<'a, T> { + /// Create a new [`HrTimerCallbackContext`]. + /// + /// # Safety + /// + /// This function relies on the caller being within the context of a timer callback, so it must + /// not be used anywhere except for within implementations of [`RawHrTimerCallback::run`]. The + /// caller promises that `timer` points to a valid initialized instance of + /// [`bindings::hrtimer`]. + /// + /// The returned `Self` must not outlive the function context of [`RawHrTimerCallback::run`] + /// where this function is called. + pub(crate) unsafe fn from_raw(timer: *mut HrTimer<T>) -> Self { + // SAFETY: The caller guarantees `timer` is a valid pointer to an initialized + // `bindings::hrtimer` + // INVARIANT: Our safety contract ensures that we're within the context of a timer callback + // and that `timer` points to a live instance of `HrTimer<T>`. + Self(unsafe { NonNull::new_unchecked(timer) }, PhantomData) + } + + /// Conditionally forward the timer. + /// + /// This function is identical to [`HrTimer::forward()`] except that it may only be used from + /// within the context of a [`HrTimer`] callback. + pub fn forward(&mut self, now: HrTimerInstant<T>, interval: Delta) -> u64 { + // SAFETY: + // - We are guaranteed to be within the context of a timer callback by our type invariants + // - By our type invariants, `self.0` always points to a valid `HrTimer<T>` + unsafe { HrTimer::<T>::raw_forward(self.0.as_ptr(), now, interval) } + } + + /// Conditionally forward the timer. + /// + /// This is a variant of [`HrTimerCallbackContext::forward()`] that uses an interval after the + /// current time of the base clock for the [`HrTimer`]. + pub fn forward_now(&mut self, duration: Delta) -> u64 { + self.forward(HrTimerInstant::<T>::now(), duration) + } +} + /// Use to implement the [`HasHrTimer<T>`] trait. /// /// See [`module`] documentation for an example. diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs index ed490a7a8950..7be82bcb352a 100644 --- a/rust/kernel/time/hrtimer/arc.rs +++ b/rust/kernel/time/hrtimer/arc.rs @@ -3,6 +3,7 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; +use super::HrTimerCallbackContext; use super::HrTimerHandle; use super::HrTimerMode; use super::HrTimerPointer; @@ -99,6 +100,12 @@ where // allocation from other `Arc` clones. let receiver = unsafe { ArcBorrow::from_raw(data_ptr) }; - T::run(receiver).into_c() + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer<T>` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver, context).into_c() } } diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs index aef16d9ee2f0..4d39ef781697 100644 --- a/rust/kernel/time/hrtimer/pin.rs +++ b/rust/kernel/time/hrtimer/pin.rs @@ -3,6 +3,7 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; +use super::HrTimerCallbackContext; use super::HrTimerHandle; use super::HrTimerMode; use super::RawHrTimerCallback; @@ -103,6 +104,12 @@ where // here. let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; - T::run(receiver_pin).into_c() + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer<T>` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() } } diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs index 767d0a4e8a2c..9d9447d4d57e 100644 --- a/rust/kernel/time/hrtimer/pin_mut.rs +++ b/rust/kernel/time/hrtimer/pin_mut.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 use super::{ - HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, HrTimerMode, RawHrTimerCallback, - UnsafeHrTimerPointer, + HasHrTimer, HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerHandle, HrTimerMode, + RawHrTimerCallback, UnsafeHrTimerPointer, }; use core::{marker::PhantomData, pin::Pin, ptr::NonNull}; @@ -107,6 +107,12 @@ where // here. let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; - T::run(receiver_pin).into_c() + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer<T>` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(receiver_pin, context).into_c() } } diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs index ec08303315f2..aa1ee31a7195 100644 --- a/rust/kernel/time/hrtimer/tbox.rs +++ b/rust/kernel/time/hrtimer/tbox.rs @@ -3,6 +3,7 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; +use super::HrTimerCallbackContext; use super::HrTimerHandle; use super::HrTimerMode; use super::HrTimerPointer; @@ -119,6 +120,12 @@ where // `data_ptr` exist. let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) }; - T::run(data_mut_ref).into_c() + // SAFETY: + // - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so + // it is a valid pointer to a `HrTimer<T>` embedded in a `T`. + // - We are within `RawHrTimerCallback::run` + let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) }; + + T::run(data_mut_ref, context).into_c() } } |