Lines
81.82 %
Functions
100 %
//! Helper types for DMA buffers.
use core::ops::{Deref, DerefMut};
use procmacros::doc_replace;
#[cfg(dma_can_access_psram)]
use crate::soc::is_valid_psram_address;
use crate::{
dma::{DmaAlignmentError, DmaBufError},
soc::is_valid_ram_address,
};
/// DMA appropriate wrapper type for internal memory values.
///
/// The value wrapped in this type is guaranteed to be safely useable
/// as a DMA buffer or descriptor array, meaning DMA or cache management
/// operations will not corrupt surrounding data.
/// [`DmaAlignedMut`] is a reference type that carries this guarantee.
// ESP32-P4 internal memory is cached, enforce alignment to avoid memory
// corruption. Technically only needed for IN buffers and descriptor lists.
#[derive(Debug)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
#[cfg_attr(soc_internal_memory_cached, repr(C, align(64)))] // dcache cache line
#[cfg_attr(not(soc_internal_memory_cached), repr(C, align(4)))] // Worst-case word alignment
#[instability::unstable]
pub struct InternalMemory<T>(T);
impl<T> InternalMemory<T> {
/// Creates a new value aligned for DMA operations in internal memory.
pub const fn new(init: T) -> Self {
Self(init)
}
/// Returns a const pointer to the wrapped value.
/// The returned pointer is aligned appropriately for DMA and points at the
/// same address that the DMA engine will access.
pub const fn as_ptr(&self) -> *const T {
&raw const self.0
/// Returns a [`DmaAlignedMut`] to the underlying value.
/// # Panics
/// Panics if the value is not located at a valid internal RAM address.
pub fn get_mut(&mut self) -> DmaAlignedMut<'_, T> {
assert!(is_valid_ram_address(&raw const self.0 as usize));
DmaAlignedMut(&mut self.0)
/// Returns a [`DmaAlignedRef`] to the underlying value.
/// The reference can be used to read the value and to invalidate its cache
/// lines, but not to mutate or write it back. This allows polling
/// DMA-written state (e.g. a descriptor ownership bit) through a shared
/// borrow.
pub fn get_ref(&self) -> DmaAlignedRef<'_, T> {
DmaAlignedRef(&self.0)
/// Returns the DMA alignment (applying to both the start address and the
/// length) required by the memory region containing `addr`, or `None` if
/// `addr` is not in a DMA-capable region.
/// Both the address and size of a DMA buffer must be a multiple of this value
/// so that cache maintenance on the buffer cannot corrupt neighboring data.
pub(crate) fn region_dma_alignment(addr: usize) -> Option<usize> {
if is_valid_ram_address(addr) {
return Some(core::mem::align_of::<InternalMemory<()>>());
if is_valid_psram_address(addr) {
return Some(cfg_select! {
// TODO(esp32p4): PSRAM is cached through the L2 cache,
// whose line size is configurable (64 or 128
// bytes) and is not encoded anywhere yet. Assume the
// larger, always-safe value until the L2 line size is
// available.
soc_internal_memory_cached => 128,
esp32s31 => 64,
any(esp32, esp32c5, esp32c61) => 32, /* TODO: fixed 32-bytes, metadata-ify */
_ => crate::soc::CONFIG_DATA_CACHE_LINE_SIZE,
});
None
/// Validates that the value at `addr` spanning `size` bytes lives in a
/// DMA-capable memory region and is aligned correctly for that region.
fn validate_dma_alignment(addr: usize, size: usize) -> Result<(), DmaBufError> {
// Zero-sized values never alias a cache line, so any address is fine.
if size == 0 {
return Ok(());
let Some(alignment) = region_dma_alignment(addr) else {
return Err(DmaBufError::UnsupportedMemoryRegion);
if !size.is_multiple_of(alignment) {
return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size));
if !addr.is_multiple_of(alignment) {
return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address));
Ok(())
/// Returns whether the value at `addr` spanning `size` bytes occupies a
/// cached memory region and therefore needs explicit cache maintenance.
#[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
fn region_needs_cache_op(addr: usize, size: usize) -> bool {
// Do not do cache operations on zero-sized values.
return false;
let mut in_cached_region = false;
#[cfg(soc_internal_memory_cached)]
{
in_cached_region |= is_valid_ram_address(addr);
in_cached_region |= is_valid_psram_address(addr);
in_cached_region
/// A mutable reference to an [`InternalMemory`] object.
pub struct DmaAlignedMut<'a, T>(&'a mut T)
where
T: ?Sized;
impl<'a, T: ?Sized> DmaAlignedMut<'a, T> {
#[doc_replace("align_req" => {
cfg(soc_internal_memory_cached) => "64",
_ => "4"
})]
/// Creates a new [`DmaAlignedMut`] from a mutable variable, if it is
/// provably compatible.
/// In internal memory, the address and size of the variable
/// must be at least __align_req__ byte aligned.
pub fn new(ptr: &'a mut T) -> Result<Self, DmaBufError> {
let addr = ptr as *mut T as *mut () as usize;
validate_dma_alignment(addr, core::mem::size_of_val(ptr))?;
Ok(Self(ptr))
/// Creates a new [`DmaAlignedMut`] from *any* mutable slice.
/// # Safety
/// The caller must ensure that the reference is properly aligned for the memory region it
/// occupies and the cachelines occupied by the reference do not overlap with any other data
/// that can be corrupted by a cacheline invalidation operation.
pub const unsafe fn new_unchecked(ptr: &'a mut T) -> Self {
Self(ptr)
fn do_cache_op(&self) -> bool {
region_needs_cache_op(
self.0 as *const T as *const () as usize,
core::mem::size_of_val(self.0),
)
/// Writes back the data from the cache to memory.
pub fn writeback(&mut self) {
if !self.do_cache_op() {
return;
// SAFETY: we own the cachelines and can't trash anything else
unsafe {
crate::soc::cache_writeback_addr(
self.0 as *const T as *const () as u32,
core::mem::size_of_val(self.0) as u32,
);
/// Invalidates the cache lines for this data.
pub fn invalidate(&self) {
crate::soc::cache_invalidate_addr(
/// Converts this object into a mutable reference.
pub fn into_inner(self) -> &'a mut T {
self.0
/// Reborrows this object with a different lifetime.
pub fn reborrow<'b>(&'b mut self) -> DmaAlignedMut<'b, T> {
DmaAlignedMut(self.0)
impl<'a, T, const N: usize> DmaAlignedMut<'a, [T; N]> {
/// Converts this object into a slice reference.
pub fn unsize(self) -> DmaAlignedMut<'a, [T]> {
impl<'a, T: ?Sized> Deref for DmaAlignedMut<'a, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
impl<'a, T: ?Sized> DerefMut for DmaAlignedMut<'a, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
/// A shared reference to an [`InternalMemory`] object.
/// Unlike [`DmaAlignedMut`], a [`DmaAlignedRef`] only allows *reading* the
/// value and *invalidating* its cache lines (discarding the CPU-cached copy so
/// a subsequent read observes data written by DMA). It can neither mutate the
/// value nor write it back, so it can be obtained from a shared borrow and used
/// to poll DMA-written, interior-mutable state (e.g. a descriptor's ownership
/// bit) without exclusive access.
pub struct DmaAlignedRef<'a, T>(&'a T)
impl<'a, T: ?Sized> DmaAlignedRef<'a, T> {
/// Creates a new [`DmaAlignedRef`] from a shared reference, if it is
/// In internal memory, the address and size of the value
pub fn new(ptr: &'a T) -> Result<Self, DmaBufError> {
let addr = ptr as *const T as *const () as usize;
/// Creates a new [`DmaAlignedRef`] from *any* shared reference.
pub const unsafe fn new_unchecked(ptr: &'a T) -> Self {
// SAFETY: the wrapped value owns the cachelines it occupies, so
// invalidating them cannot discard CPU writes to neighbouring data.
/// Writes back the cached copy of this data to memory.
/// This only flushes the CPU cache to memory (so a DMA engine reading from
/// memory observes the latest CPU writes); it does not mutate the value, so
/// it is sound to perform through a shared reference.
pub fn writeback(&self) {
// SAFETY: the wrapped value owns the cachelines it occupies, so writing
// them back cannot affect neighbouring data.
/// Converts this object into a shared reference.
pub fn into_inner(self) -> &'a T {
impl<'a, T: ?Sized> Deref for DmaAlignedRef<'a, T> {