1
#[cfg(dma_can_access_psram)]
2
use core::{mem::MaybeUninit, ops::Range};
3
use core::{
4
    ops::{Deref, DerefMut},
5
    ptr::{NonNull, null_mut},
6
};
7

            
8
use super::*;
9
#[cfg(dma_can_access_psram)]
10
use crate::soc::{is_slice_in_psram, is_valid_psram_address, is_valid_ram_address};
11
use crate::{
12
    dma::aligned::{DmaAlignedMut, InternalMemory},
13
    soc::is_slice_in_dram,
14
};
15

            
16
pub(crate) mod scoped;
17
pub(crate) use scoped::*;
18

            
19
/// Error returned from Dma[Rx|Tx|RxTx]Buf operations.
20
#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
21
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
22
pub enum DmaBufError {
23
    /// The buffer is smaller than the requested size.
24
    BufferTooSmall,
25

            
26
    /// More descriptors are needed for the buffer size.
27
    InsufficientDescriptors,
28

            
29
    /// Descriptors or buffers are not located in a supported memory region.
30
    UnsupportedMemoryRegion,
31

            
32
    /// Buffer address or size is not properly aligned.
33
    InvalidAlignment(DmaAlignmentError),
34

            
35
    /// Invalid chunk size: must be > 0 and <= 4095.
36
    InvalidChunkSize,
37
}
38

            
39
impl core::fmt::Display for DmaBufError {
40
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
41
        match self {
42
            DmaBufError::BufferTooSmall => {
43
                write!(f, "The buffer is smaller than the requested size")
44
            }
45
            DmaBufError::InsufficientDescriptors => {
46
                write!(f, "More descriptors are needed for the buffer size")
47
            }
48
            DmaBufError::UnsupportedMemoryRegion => write!(
49
                f,
50
                "Descriptors or buffers are not located in a supported memory region"
51
            ),
52
            DmaBufError::InvalidAlignment(x) => write!(f, "{x}"),
53
            DmaBufError::InvalidChunkSize => {
54
                write!(f, "Invalid chunk size: must be > 0 and <= 4095")
55
            }
56
        }
57
    }
58
}
59

            
60
impl core::error::Error for DmaBufError {}
61

            
62
/// DMA buffer alignment errors.
63
#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
64
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
65
pub enum DmaAlignmentError {
66
    /// Buffer address is not properly aligned.
67
    Address,
68

            
69
    /// Buffer size is not properly aligned.
70
    Size,
71
}
72

            
73
impl From<DmaAlignmentError> for DmaBufError {
74
    fn from(err: DmaAlignmentError) -> Self {
75
        DmaBufError::InvalidAlignment(err)
76
    }
77
}
78

            
79
impl core::fmt::Display for DmaAlignmentError {
80
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
81
        match self {
82
            DmaAlignmentError::Address => write!(f, "Buffer address is not properly aligned"),
83
            DmaAlignmentError::Size => write!(f, "Buffer size is not properly aligned"),
84
        }
85
    }
86
}
87

            
88
impl core::error::Error for DmaAlignmentError {}
89

            
90
cfg_select! {
91
    dma_can_access_psram => {
92
        /// Burst size used when transferring to and from external memory.
93
        #[derive(Clone, Copy, PartialEq, Eq, Debug)]
94
        #[cfg_attr(feature = "defmt", derive(defmt::Format))]
95
        pub enum ExternalBurstConfig {
96
            /// 16 bytes
97
            Size16 = 16,
98

            
99
            /// 32 bytes
100
            Size32 = 32,
101

            
102
            /// 64 bytes
103
            // TODO: investigate why ext_mem_bk_size = 2 causes corruption on S2
104
            #[cfg(not(esp32s2))]
105
            Size64 = 64,
106
        }
107

            
108
        impl ExternalBurstConfig {
109
            /// The default external memory burst length.
110
            pub const DEFAULT: Self = Self::Size16;
111
        }
112

            
113
        impl Default for ExternalBurstConfig {
114
            fn default() -> Self {
115
                Self::DEFAULT
116
            }
117
        }
118

            
119
        /// Internal memory access burst mode.
120
        #[derive(Clone, Copy, PartialEq, Eq, Debug)]
121
        #[cfg_attr(feature = "defmt", derive(defmt::Format))]
122
        pub enum InternalBurstConfig {
123
            /// Burst mode is disabled.
124
            Disabled,
125

            
126
            /// Burst mode is enabled.
127
            Enabled,
128
        }
129

            
130
        impl InternalBurstConfig {
131
            /// The default internal burst mode configuration.
132
            pub const DEFAULT: Self = Self::Disabled;
133
        }
134

            
135
        impl Default for InternalBurstConfig {
136
            fn default() -> Self {
137
                Self::DEFAULT
138
            }
139
        }
140

            
141
        /// Burst transfer configuration.
142
        #[derive(Clone, Copy, PartialEq, Eq, Debug)]
143
        #[cfg_attr(feature = "defmt", derive(defmt::Format))]
144
        pub struct BurstConfig {
145
            /// Configures the burst size for PSRAM transfers.
146
            ///
147
            /// Burst mode is always enabled for PSRAM transfers.
148
            pub external_memory: ExternalBurstConfig,
149

            
150
            /// Enables or disables the burst mode for internal memory transfers.
151
            ///
152
            /// The burst size is not configurable.
153
            pub internal_memory: InternalBurstConfig,
154
        }
155

            
156
        impl BurstConfig {
157
            /// The default burst mode configuration.
158
            pub const DEFAULT: Self = Self {
159
                external_memory: ExternalBurstConfig::DEFAULT,
160
                internal_memory: InternalBurstConfig::DEFAULT,
161
            };
162
        }
163

            
164
        impl Default for BurstConfig {
165
            fn default() -> Self {
166
                Self::DEFAULT
167
            }
168
        }
169

            
170
        impl From<InternalBurstConfig> for BurstConfig {
171
            fn from(internal_memory: InternalBurstConfig) -> Self {
172
                Self {
173
                    external_memory: ExternalBurstConfig::DEFAULT,
174
                    internal_memory,
175
                }
176
            }
177
        }
178

            
179
        impl From<ExternalBurstConfig> for BurstConfig {
180
            fn from(external_memory: ExternalBurstConfig) -> Self {
181
                Self {
182
                    external_memory,
183
                    internal_memory: InternalBurstConfig::DEFAULT,
184
                }
185
            }
186
        }
187
    }
188
    _ => {
189
        /// Burst transfer configuration.
190
        #[derive(Clone, Copy, PartialEq, Eq, Debug)]
191
        #[cfg_attr(feature = "defmt", derive(defmt::Format))]
192
        pub enum BurstConfig {
193
            /// Burst mode is disabled.
194
            Disabled,
195

            
196
            /// Burst mode is enabled.
197
            Enabled,
198
        }
199

            
200
        impl BurstConfig {
201
            /// The default burst mode configuration.
202
            pub const DEFAULT: Self = Self::Disabled;
203
        }
204

            
205
        impl Default for BurstConfig {
206
            fn default() -> Self {
207
                Self::DEFAULT
208
            }
209
        }
210

            
211
        type InternalBurstConfig = BurstConfig;
212
    }
213
}
214

            
215
#[cfg(dma_can_access_psram)]
216
impl ExternalBurstConfig {
217
    const fn min_psram_alignment(self, direction: TransferDirection) -> usize {
218
        // S2 TRM: Specifically, size and buffer address pointer in receive descriptors
219
        // should be 16-byte, 32-byte or 64-byte aligned. For data frame whose
220
        // length is not a multiple of 16 bytes, 32 bytes, or 64 bytes, EDMA adds
221
        // padding bytes to the end.
222

            
223
        // S3 TRM: Size and Address for IN transfers must be block aligned. For receive
224
        // descriptors, if the data length received are not aligned with block size,
225
        // GDMA will pad the data received with 0 until they are aligned to
226
        // initiate burst transfer. You can read the length field in receive descriptors
227
        // to obtain the length of valid data received
228
        if matches!(direction, TransferDirection::In) {
229
            self as usize
230
        } else {
231
            // S2 TRM: Size, length and buffer address pointer in transmit descriptors are
232
            // not necessarily aligned with block size.
233

            
234
            // S3 TRM: Size, length, and buffer address pointer in transmit descriptors do
235
            // not need to be aligned.
236
            1
237
        }
238
    }
239
}
240

            
241
impl InternalBurstConfig {
242
    pub(super) const fn is_burst_enabled(self) -> bool {
243
        !matches!(self, Self::Disabled)
244
    }
245

            
246
    // Size and address alignment as those come in pairs on current hardware.
247
    const fn min_dram_alignment(self, direction: TransferDirection) -> usize {
248
40
        if matches!(direction, TransferDirection::In) {
249
            if cfg!(esp32) {
250
                // NOTE: The size must be word-aligned.
251
                // NOTE: The buffer address must be word-aligned
252
                4
253
40
            } else if self.is_burst_enabled() {
254
                // As described in "Accessing Internal Memory" paragraphs in the various TRMs.
255
                4
256
            } else {
257
                1
258
            }
259
        } else {
260
            // OUT transfers have no alignment requirements, except for ESP32, which is
261
            // described below.
262
            if cfg!(esp32) {
263
                // SPI DMA: Burst transmission is supported. The data size for
264
                // a single transfer must be four bytes aligned.
265
                // I2S DMA: Burst transfer is supported. However, unlike the
266
                // SPI DMA channels, the data size for a single transfer is
267
                // one word, or four bytes.
268
                4
269
            } else {
270
                1
271
            }
272
        }
273
    }
274
}
275

            
276
const fn max(a: usize, b: usize) -> usize {
277
    if a > b { a } else { b }
278
}
279

            
280
impl BurstConfig {
281
    delegate::delegate! {
282
        to self.internal_memory {
283
            #[cfg(dma_can_access_psram)]
284
            pub(super) const fn min_dram_alignment(self, direction: TransferDirection) -> usize;
285

            
286
            #[cfg(all(dma_can_access_psram, not(esp32s31)))] // Burst always enabled
287
            pub(super) fn is_burst_enabled(self) -> bool;
288
        }
289
    }
290

            
291
    /// Calculates an alignment that is compatible with the current burst
292
    /// configuration.
293
    ///
294
    /// This is an over-estimation so that Descriptors can be safely used with
295
    /// any DMA channel in any direction.
296
    pub const fn min_compatible_alignment(self) -> usize {
297
        let in_alignment = self.min_dram_alignment(TransferDirection::In);
298
        let out_alignment = self.min_dram_alignment(TransferDirection::Out);
299
        let alignment = max(in_alignment, out_alignment);
300

            
301
        #[cfg(dma_can_access_psram)]
302
        let alignment = max(alignment, self.external_memory as usize);
303

            
304
        alignment
305
    }
306

            
307
    const fn chunk_size_for_alignment(alignment: usize) -> usize {
308
        // DMA descriptors have a 12-bit field for the size/length of the buffer they
309
        // point at. As there is no such thing as 0-byte alignment, this means the
310
        // maximum size is 4095 bytes.
311
        4096 - alignment
312
    }
313

            
314
    /// Calculates a chunk size that is compatible with the current burst
315
    /// configuration's alignment requirements.
316
    ///
317
    /// This is an over-estimation so that Descriptors can be safely used with
318
    /// any DMA channel in any direction.
319
    pub const fn max_compatible_chunk_size(self) -> usize {
320
        Self::chunk_size_for_alignment(self.min_compatible_alignment())
321
    }
322

            
323
    fn min_alignment(self, _buffer: &[u8], direction: TransferDirection) -> usize {
324
        let alignment = self.min_dram_alignment(direction);
325

            
326
        cfg_select! {
327
            dma_can_access_psram => {
328
                let mut alignment = alignment;
329
                if is_valid_psram_address(_buffer.as_ptr() as usize) {
330
                    alignment = max(
331
                        alignment,
332
                        self.external_memory.min_psram_alignment(direction),
333
                    );
334
                }
335
            }
336
            _ => {}
337
        }
338

            
339
        alignment
340
    }
341

            
342
    // Note: this function ignores address alignment as we assume the buffers are
343
    // aligned.
344
    fn max_chunk_size_for(self, buffer: &[u8], direction: TransferDirection) -> usize {
345
        Self::chunk_size_for_alignment(self.min_alignment(buffer, direction))
346
    }
347

            
348
    fn ensure_buffer_aligned(
349
        self,
350
        buffer: &[u8],
351
        direction: TransferDirection,
352
    ) -> Result<(), DmaAlignmentError> {
353
        let alignment = self.min_alignment(buffer, direction);
354
40
        if !(buffer.as_ptr() as usize).is_multiple_of(alignment) {
355
            return Err(DmaAlignmentError::Address);
356
        }
357

            
358
        // NB: the TRMs suggest that buffer length don't need to be aligned, but
359
        // for IN transfers, we configure the DMA descriptors' size field, which needs
360
        // to be aligned.
361
        if direction == TransferDirection::In && !buffer.len().is_multiple_of(alignment) {
362
            return Err(DmaAlignmentError::Size);
363
        }
364

            
365
        Ok(())
366
    }
367

            
368
    fn ensure_buffer_compatible(
369
        self,
370
        buffer: &[u8],
371
        direction: TransferDirection,
372
    ) -> Result<(), DmaBufError> {
373
84
        if buffer.is_empty() {
374
            return Ok(());
375
        }
376
        // buffer can be either DRAM or PSRAM (if supported)
377
        let is_in_dram = is_slice_in_dram(buffer);
378
        cfg_select! {
379
            dma_can_access_psram => {
380
                let is_in_psram = is_slice_in_psram(buffer);
381
            }
382
            _ => {
383
                let is_in_psram = false;
384
            }
385
        }
386

            
387
49
        if !(is_in_dram || is_in_psram) {
388
            return Err(DmaBufError::UnsupportedMemoryRegion);
389
        }
390

            
391
        self.ensure_buffer_aligned(buffer, direction)?;
392

            
393
        Ok(())
394
    }
395
}
396

            
397
/// The direction of the DMA transfer.
398
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
399
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
400
pub enum TransferDirection {
401
    /// DMA transfer from peripheral or external memory to memory.
402
    In,
403
    /// DMA transfer from memory to peripheral or external memory.
404
    Out,
405
}
406

            
407
/// Holds all the information needed to configure a DMA channel for a transfer.
408
#[derive(PartialEq, Eq, Debug)]
409
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
410
pub struct Preparation {
411
    /// The descriptor the DMA will start from.
412
    pub start: *mut DmaDescriptor,
413

            
414
    /// Must be `true` if any of the DMA descriptors contain data in PSRAM.
415
    #[cfg(dma_can_access_psram)]
416
    pub accesses_psram: bool,
417

            
418
    /// Configures the DMA to transfer data in bursts.
419
    ///
420
    /// The implementation of the buffer must ensure that buffer size
421
    /// and alignment in each descriptor is compatible with the burst
422
    /// transfer configuration.
423
    ///
424
    /// For details on alignment requirements, refer to the chip's
425
    #[doc = crate::trm_markdown_link!()]
426
    pub burst_transfer: BurstConfig,
427

            
428
    /// Configures the "check owner" feature of the DMA channel.
429
    ///
430
    /// Most DMA channels allow software to configure whether the hardware
431
    /// checks that [DmaDescriptor::owner] is set to [Owner::Dma] before
432
    /// consuming the descriptor. If this check fails, the channel stops
433
    /// operating and fires
434
    /// [DmaRxInterrupt::DescriptorError]/[DmaTxInterrupt::DescriptorError].
435
    ///
436
    /// This field allows buffer implementation to configure this behavior.
437
    /// - `Some(true)`: DMA channel must check the owner bit.
438
    /// - `Some(false)`: DMA channel must NOT check the owner bit.
439
    /// - `None`: DMA channel should check the owner bit if it is supported.
440
    ///
441
    /// Some buffer implementations may require that the DMA channel performs
442
    /// this check before consuming the descriptor to ensure correct
443
    /// behavior. e.g. To prevent wrap-around in a circular transfer.
444
    ///
445
    /// Some buffer implementations may require that the DMA channel does NOT
446
    /// perform this check as the ownership bit will not be set before the
447
    /// channel tries to consume the descriptor.
448
    ///
449
    /// Most implementations do not have any such requirements and work
450
    /// correctly regardless of whether the DMA channel checks or not.
451
    ///
452
    /// If the DMA channel does not support the provided option, preparation
453
    /// fails.
454
    pub check_owner: Option<bool>,
455

            
456
    /// Configures whether the DMA channel automatically clears the
457
    /// [DmaDescriptor::owner] bit after it is done with the buffer pointed
458
    /// to by a descriptor.
459
    ///
460
    /// For RX transfers, this is always true and the value specified here is
461
    /// ignored.
462
    ///
463
    /// SPI_DMA on the ESP32 does not support this and panics if set
464
    /// to true.
465
    pub auto_write_back: bool,
466
}
467

            
468
/// [`DmaTxBuffer`] is a DMA descriptor + memory combo that can be used for
469
/// transmitting data from a DMA channel to a peripheral's FIFO.
470
///
471
/// # Safety
472
///
473
/// The implementing type must keep all its descriptors and the buffers they
474
/// point to valid while the buffer is being transferred.
475
pub unsafe trait DmaTxBuffer {
476
    /// A type providing operations that are safe to perform on the buffer
477
    /// while the DMA is actively using it.
478
    type View;
479

            
480
    /// The type returned when a transfer finishes.
481
    ///
482
    /// Some buffers do not need to be reconstructed.
483
    type Final;
484

            
485
    /// Prepares the buffer for an imminent transfer and returns
486
    /// information required to use this buffer.
487
    ///
488
    /// This operation is idempotent.
489
    fn prepare(&mut self) -> Preparation;
490

            
491
    /// This is called before the DMA starts using the buffer.
492
    fn into_view(self) -> Self::View;
493

            
494
    /// This is called after the DMA is done using the buffer.
495
    fn from_view(view: Self::View) -> Self::Final;
496
}
497

            
498
/// [`DmaRxBuffer`] is a DMA descriptor + memory combo that can be used for
499
/// receiving data from a peripheral's FIFO to a DMA channel.
500
///
501
/// Implementations of this trait may only support having a single EOF bit
502
/// which resides in the last descriptor. There will be a separate trait in
503
/// future to support multiple EOFs.
504
///
505
/// # Safety
506
///
507
/// The implementing type must keep all its descriptors and the buffers they
508
/// point to valid while the buffer is being transferred.
509
pub unsafe trait DmaRxBuffer {
510
    /// A type providing operations that are safe to perform on the buffer
511
    /// while the DMA is actively using it.
512
    type View;
513

            
514
    /// The type returned when a transfer finishes.
515
    ///
516
    /// Some buffers do not need to be reconstructed.
517
    type Final;
518

            
519
    /// Prepares the buffer for an imminent transfer and returns
520
    /// information required to use this buffer.
521
    ///
522
    /// This operation is idempotent.
523
    fn prepare(&mut self) -> Preparation;
524

            
525
    /// This is called before the DMA starts using the buffer.
526
    fn into_view(self) -> Self::View;
527

            
528
    /// This is called after the DMA is done using the buffer.
529
    fn from_view(view: Self::View) -> Self::Final;
530
}
531

            
532
/// An in-progress view into [`DmaRxBuf`]/[`DmaTxBuf`].
533
///
534
/// In the future, this could support peeking into state of the
535
/// descriptors/buffers.
536
pub struct BufView<T>(T);
537

            
538
/// DMA transmit buffer
539
///
540
/// This is a contiguous buffer linked together by DMA descriptors of length
541
/// 4095 at most. It can only be used for transmitting data to a peripheral's
542
/// FIFO. See [`DmaRxBuf`] for receiving data.
543
#[derive(Debug)]
544
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
545
pub struct DmaTxBuf(ScopedDmaTxBuf<'static>);
546

            
547
impl DmaTxBuf {
548
    /// Creates a new [`DmaTxBuf`] from some descriptors and a buffer.
549
36
    pub fn new(
550
        descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
551
        buffer: DmaAlignedMut<'static, [u8]>,
552
    ) -> Result<Self, DmaBufError> {
553
        ScopedDmaTxBuf::new(descriptors, buffer).map(Self)
554
36
    }
555

            
556
    /// Creates a new [`DmaTxBuf`] from some descriptors and a buffer.
557
    ///
558
    /// There must be enough descriptors for the provided buffer.
559
    /// Depending on alignment requirements, each descriptor can handle at most
560
    /// 4095 bytes worth of buffer.
561
    ///
562
    /// Both the descriptors and buffer must be in DMA-capable memory.
563
    /// Only DRAM is supported for descriptors.
564
    pub fn new_with_config(
565
        descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
566
        buffer: DmaAlignedMut<'static, [u8]>,
567
        config: impl Into<BurstConfig>,
568
    ) -> Result<Self, DmaBufError> {
569
        ScopedDmaTxBuf::new_with_config(descriptors, buffer, config).map(Self)
570
    }
571

            
572
    /// Configures the DMA to use burst transfers to access this buffer.
573
    pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> {
574
        self.0.set_burst_config(burst)
575
    }
576

            
577
    /// Consumes the buf, returning the descriptors and buffer.
578
    pub fn split(
579
        self,
580
    ) -> (
581
        DmaAlignedMut<'static, [DmaDescriptor]>,
582
        DmaAlignedMut<'static, [u8]>,
583
    ) {
584
        self.0.split()
585
    }
586

            
587
    /// Returns the size of the underlying buffer.
588
    pub fn capacity(&self) -> usize {
589
        self.0.capacity()
590
    }
591

            
592
    /// Returns the number of bytes that would be transmitted by this buf.
593
    #[allow(clippy::len_without_is_empty)]
594
21
    pub fn len(&self) -> usize {
595
21
        self.0.len()
596
21
    }
597

            
598
    /// Resets the descriptors to only transmit `len` amount of bytes from this
599
    /// buf.
600
    ///
601
    /// The number of bytes in data must be less than or equal to the buffer
602
    /// size.
603
    pub fn set_length(&mut self, len: usize) {
604
        self.0.set_length(len);
605
    }
606

            
607
    /// Fills the TX buffer with the bytes provided in `data` and reset the
608
    /// descriptors to only cover the filled section.
609
    ///
610
    /// The number of bytes in data must be less than or equal to the buffer
611
    /// size.
612
11
    pub fn fill(&mut self, data: &[u8]) {
613
        self.0.fill(data);
614
    }
615

            
616
    /// Returns the buf as a mutable slice that can be written.
617
    pub fn as_mut_slice(&mut self) -> &mut [u8] {
618
        self.0.as_mut_slice()
619
    }
620

            
621
    /// Returns the buf as a slice that can be read.
622
    pub fn as_slice(&self) -> &[u8] {
623
        self.0.as_slice()
624
    }
625

            
626
    /// Consumes the buffer and returns the scoped version.
627
    pub(crate) fn into_scoped(self) -> ScopedDmaTxBuf<'static> {
628
        self.0
629
    }
630
}
631

            
632
unsafe impl DmaTxBuffer for DmaTxBuf {
633
    type View = BufView<DmaTxBuf>;
634
    type Final = DmaTxBuf;
635

            
636
    fn prepare(&mut self) -> Preparation {
637
        self.0.prepare()
638
    }
639

            
640
    fn into_view(self) -> BufView<DmaTxBuf> {
641
        BufView(self)
642
    }
643

            
644
    fn from_view(view: Self::View) -> Self {
645
        view.0
646
    }
647
}
648

            
649
/// DMA receive buffer
650
///
651
/// This is a contiguous buffer linked together by DMA descriptors of length
652
/// 4092. It can only be used for receiving data from a peripheral's FIFO.
653
/// See [`DmaTxBuf`] for transmitting data.
654
#[derive(Debug)]
655
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
656
pub struct DmaRxBuf(ScopedDmaRxBuf<'static>);
657

            
658
impl DmaRxBuf {
659
    /// Creates a new [`DmaRxBuf`] from some descriptors and a buffer.
660
39
    pub fn new(
661
        descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
662
        buffer: DmaAlignedMut<'static, [u8]>,
663
    ) -> Result<Self, DmaBufError> {
664
        ScopedDmaRxBuf::new(descriptors, buffer).map(Self)
665
39
    }
666

            
667
    /// Creates a new [`DmaRxBuf`] from some descriptors and a buffer.
668
    ///
669
    /// There must be enough descriptors for the provided buffer.
670
    /// Depending on alignment requirements, each descriptor can handle at most
671
    /// 4092 bytes worth of buffer.
672
    ///
673
    /// Both the descriptors and buffer must be in DMA-capable memory.
674
    /// Only DRAM is supported for descriptors.
675
    pub fn new_with_config(
676
        descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
677
        buffer: DmaAlignedMut<'static, [u8]>,
678
        config: impl Into<BurstConfig>,
679
    ) -> Result<Self, DmaBufError> {
680
        ScopedDmaRxBuf::new_with_config(descriptors, buffer, config).map(Self)
681
    }
682

            
683
    /// Configures the DMA to use burst transfers to access this buffer.
684
    pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> {
685
        self.0.set_burst_config(burst)
686
    }
687

            
688
    /// Consumes the buf, returning the descriptors and buffer.
689
    pub fn split(
690
        self,
691
    ) -> (
692
        DmaAlignedMut<'static, [DmaDescriptor]>,
693
        DmaAlignedMut<'static, [u8]>,
694
    ) {
695
        self.0.split()
696
    }
697

            
698
    /// Returns the size of the underlying buffer.
699
    pub fn capacity(&self) -> usize {
700
        self.0.capacity()
701
    }
702

            
703
    /// Returns the maximum number of bytes that this buf has been configured to
704
    /// receive.
705
    #[allow(clippy::len_without_is_empty)]
706
37
    pub fn len(&self) -> usize {
707
37
        self.0.len()
708
37
    }
709

            
710
    /// Resets the descriptors to only receive `len` amount of bytes into this
711
    /// buf.
712
    ///
713
    /// The number of bytes in data must be less than or equal to the buffer
714
    /// size.
715
    pub fn set_length(&mut self, len: usize) {
716
        self.0.set_length(len)
717
    }
718

            
719
    /// Returns the entire underlying buffer as a slice that can be read.
720
    pub fn as_slice(&self) -> &[u8] {
721
        self.0.as_slice()
722
    }
723

            
724
    /// Returns the entire underlying buffer as a slice that can be written.
725
    pub fn as_mut_slice(&mut self) -> &mut [u8] {
726
        self.0.as_mut_slice()
727
    }
728

            
729
    /// Returns the number of bytes that was received by this buf.
730
    pub fn number_of_received_bytes(&self) -> usize {
731
        self.0.number_of_received_bytes()
732
    }
733

            
734
    /// Reads the received data into the provided `buf`.
735
    ///
736
    /// If `buf.len()` is less than the amount of received data then only the
737
    /// first `buf.len()` bytes of received data is written into `buf`.
738
    ///
739
    /// Returns the number of bytes written to `buf`.
740
    pub fn read_received_data(&self, buf: &mut [u8]) -> usize {
741
        self.0.read_received_data(buf)
742
    }
743

            
744
    /// Returns the received data as an iterator of slices.
745
    pub fn received_data(&self) -> impl Iterator<Item = &[u8]> {
746
        self.0.received_data()
747
    }
748

            
749
    /// Consumes the buffer and returns the scoped version.
750
    pub(crate) fn into_scoped(self) -> ScopedDmaRxBuf<'static> {
751
        self.0
752
    }
753
}
754

            
755
unsafe impl DmaRxBuffer for DmaRxBuf {
756
    type View = BufView<DmaRxBuf>;
757
    type Final = DmaRxBuf;
758

            
759
46
    fn prepare(&mut self) -> Preparation {
760
        self.0.prepare()
761
46
    }
762

            
763
    fn into_view(self) -> BufView<DmaRxBuf> {
764
        BufView(self)
765
    }
766

            
767
    fn from_view(view: Self::View) -> Self {
768
        view.0
769
    }
770
}
771

            
772
/// DMA transmit and receive buffer.
773
///
774
/// This is a (single) contiguous buffer linked together by two sets of DMA
775
/// descriptors of length 4092 each.
776
/// It can be used for simultaneously transmitting to and receiving from a
777
/// peripheral's FIFO. These are typically full-duplex transfers.
778
#[derive(Debug)]
779
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
780
pub struct DmaRxTxBuf {
781
    rx_descriptors: DescriptorSet<'static>,
782
    tx_descriptors: DescriptorSet<'static>,
783
    buffer: DmaAlignedMut<'static, [u8]>,
784
    burst: BurstConfig,
785
}
786

            
787
impl DmaRxTxBuf {
788
    /// Creates a new [DmaRxTxBuf] from some descriptors and a buffer.
789
    pub fn new(
790
        rx_descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
791
        tx_descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
792
        buffer: DmaAlignedMut<'static, [u8]>,
793
    ) -> Result<Self, DmaBufError> {
794
        let mut buf = Self {
795
            rx_descriptors: DescriptorSet::new(rx_descriptors)?,
796
            tx_descriptors: DescriptorSet::new(tx_descriptors)?,
797
            buffer,
798
            burst: BurstConfig::default(),
799
        };
800

            
801
        let capacity = buf.capacity();
802
        buf.configure(buf.burst, capacity)?;
803

            
804
        Ok(buf)
805
    }
806

            
807
    fn configure(
808
        &mut self,
809
        burst: impl Into<BurstConfig>,
810
        length: usize,
811
    ) -> Result<(), DmaBufError> {
812
        let burst = burst.into();
813
        self.set_length_fallible(length, burst)?;
814

            
815
        let max_chunk_size_in = burst.max_chunk_size_for(&self.buffer, TransferDirection::In);
816
        let max_chunk_size_out = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out);
817
        self.rx_descriptors
818
            .link_with_buffer(&mut self.buffer, max_chunk_size_in)?;
819
        self.tx_descriptors
820
            .link_with_buffer(&mut self.buffer, max_chunk_size_out)?;
821

            
822
        self.burst = burst;
823

            
824
        Ok(())
825
    }
826

            
827
    /// Configures the DMA to use burst transfers to access this buffer.
828
    pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> {
829
        let len = self.len();
830
        self.configure(burst, len)
831
    }
832

            
833
    /// Consumes the buf, returning the rx descriptors, tx descriptors and
834
    /// buffer.
835
    #[allow(clippy::type_complexity)]
836
    pub fn split(
837
        self,
838
    ) -> (
839
        DmaAlignedMut<'static, [DmaDescriptor]>,
840
        DmaAlignedMut<'static, [DmaDescriptor]>,
841
        DmaAlignedMut<'static, [u8]>,
842
    ) {
843
        (
844
            self.rx_descriptors.into_inner(),
845
            self.tx_descriptors.into_inner(),
846
            self.buffer,
847
        )
848
    }
849

            
850
    /// Returns the size of the underlying buffer.
851
    pub fn capacity(&self) -> usize {
852
        self.buffer.len()
853
    }
854

            
855
    /// Returns the number of bytes that would be transmitted by this buf.
856
    #[allow(clippy::len_without_is_empty)]
857
    pub fn len(&self) -> usize {
858
        self.tx_descriptors
859
            .linked_iter()
860
            .map(|d| d.len())
861
            .sum::<usize>()
862
    }
863

            
864
    /// Returns the entire buf as a slice that can be read.
865
    pub fn as_slice(&self) -> &[u8] {
866
        &self.buffer
867
    }
868

            
869
    /// Returns the entire buf as a slice that can be written.
870
    pub fn as_mut_slice(&mut self) -> &mut [u8] {
871
        &mut self.buffer
872
    }
873

            
874
    fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> {
875
        if len > self.capacity() {
876
            return Err(DmaBufError::BufferTooSmall);
877
        }
878
        burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::In)?;
879
        burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::Out)?;
880

            
881
        let max_chunk_size_in = burst.max_chunk_size_for(&self.buffer, TransferDirection::In);
882
        let max_chunk_size_out = burst.max_chunk_size_for(&self.buffer, TransferDirection::Out);
883
        self.rx_descriptors.set_rx_length(len, max_chunk_size_in)?;
884
        self.tx_descriptors.set_tx_length(len, max_chunk_size_out)?;
885

            
886
        Ok(())
887
    }
888

            
889
    /// Resets the descriptors to only transmit/receive `len` amount of bytes
890
    /// with this buf.
891
    ///
892
    /// `len` must be less than or equal to the buffer size.
893
    pub fn set_length(&mut self, len: usize) {
894
        unwrap!(self.set_length_fallible(len, self.burst));
895
    }
896
}
897

            
898
unsafe impl DmaTxBuffer for DmaRxTxBuf {
899
    type View = BufView<DmaRxTxBuf>;
900
    type Final = DmaRxTxBuf;
901

            
902
    fn prepare(&mut self) -> Preparation {
903
        for desc in self.tx_descriptors.linked_iter_mut() {
904
            // In non-circular mode, we only set `suc_eof` for the last descriptor to signal
905
            // the end of the transfer.
906
            desc.reset_for_tx(desc.next.is_null());
907
        }
908

            
909
        #[cfg(dma_can_access_psram)]
910
        let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize);
911

            
912
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
913
        self.buffer.writeback();
914

            
915
        Preparation {
916
            start: self.tx_descriptors.head(),
917
            #[cfg(dma_can_access_psram)]
918
            accesses_psram: is_data_in_psram,
919
            burst_transfer: self.burst,
920
            check_owner: None,
921
            auto_write_back: false,
922
        }
923
    }
924

            
925
    fn into_view(self) -> BufView<DmaRxTxBuf> {
926
        BufView(self)
927
    }
928

            
929
    fn from_view(view: Self::View) -> Self {
930
        view.0
931
    }
932
}
933

            
934
unsafe impl DmaRxBuffer for DmaRxTxBuf {
935
    type View = BufView<DmaRxTxBuf>;
936
    type Final = DmaRxTxBuf;
937

            
938
    fn prepare(&mut self) -> Preparation {
939
        for desc in self.rx_descriptors.linked_iter_mut() {
940
            desc.reset_for_rx();
941
        }
942

            
943
        cfg_select! {
944
            dma_can_access_psram => {
945
                // Optimization: avoid locking for PSRAM range.
946
                let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize);
947
                if is_data_in_psram || cfg!(soc_internal_memory_cached) {
948
                    unsafe {
949
                        crate::soc::cache_invalidate_addr(
950
                            self.buffer.as_ptr() as u32,
951
                            self.buffer.len() as u32,
952
                        )
953
                    };
954
                }
955
            }
956
            _ => {}
957
        }
958

            
959
        Preparation {
960
            start: self.rx_descriptors.head(),
961
            #[cfg(dma_can_access_psram)]
962
            accesses_psram: is_data_in_psram,
963
            burst_transfer: self.burst,
964
            check_owner: None,
965
            auto_write_back: true,
966
        }
967
    }
968

            
969
    fn into_view(self) -> BufView<DmaRxTxBuf> {
970
        BufView(self)
971
    }
972

            
973
    fn from_view(view: Self::View) -> Self {
974
        view.0
975
    }
976
}
977

            
978
/// DMA Streaming Receive Buffer.
979
///
980
/// This is a contiguous buffer linked together by DMA descriptors, and the
981
/// buffer is evenly distributed between each descriptor provided.
982
///
983
/// It is used for continuously streaming data from a peripheral's FIFO.
984
///
985
/// It maintains a sliding window of descriptors that progresses when
986
/// [DmaRxStreamBufView::consume] is called.
987
///
988
/// The list starts out like so `A (empty) -> B (empty) -> C (empty) -> D
989
/// (empty) -> NULL`.
990
///
991
/// As the DMA writes to the buffers the list progresses like so:
992
/// - `A (empty) -> B (empty) -> C (empty) -> D (empty) -> NULL`
993
/// - `A (full)  -> B (empty) -> C (empty) -> D (empty) -> NULL`
994
/// - `A (full)  -> B (full)  -> C (empty) -> D (empty) -> NULL`
995
/// - `A (full)  -> B (full)  -> C (full)  -> D (empty) -> NULL`
996
///
997
/// As [DmaRxStreamBufView::consume] is called, the list (approximately)
998
/// progresses like so:
999
/// - `A (full)  -> B (full)  -> C (full)  -> D (empty) -> NULL`
/// - `B (full)  -> C (full)  -> D (empty) -> A (empty) -> NULL`
/// - `C (full)  -> D (empty) -> A (empty) -> B (empty) -> NULL`
/// - `D (empty) -> A (empty) -> B (empty) -> C (empty) -> NULL`
///
/// If all the descriptors fill up, the [DmaRxInterrupt::DescriptorEmpty]
/// interrupt fires and the DMA stops writing. The transfer must then be resumed
/// or restarted.
///
/// This buffer does not indicate when this condition occurs. Check with the
/// driver to see if the DMA has stopped.
///
/// When constructing this buffer, tune the ratio between the chunk size and
/// buffer size appropriately. Smaller chunk sizes mean data is received more
/// frequently, but the DMA interrupts ([DmaRxInterrupt::Done]) also fire more
/// frequently when they are used.
///
/// See [DmaRxStreamBufView] for APIs available while a transfer is in
/// progress.
#[derive(Debug)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct DmaRxStreamBuf {
    descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
    buffer: DmaAlignedMut<'static, [u8]>,
    burst: BurstConfig,
}
impl DmaRxStreamBuf {
    /// Creates a new [`DmaRxStreamBuf`] evenly distributing the buffer between
    /// the provided descriptors.
44
    pub fn new(
        mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
        mut buffer: DmaAlignedMut<'static, [u8]>,
    ) -> Result<Self, DmaBufError> {
        // see https://github.com/esp-rs/esp-hal/issues/2269#issuecomment-4397953660
        // we can lift that requirement once we sort out this issue
44
        if descriptors.len() < 4 {
            return Err(DmaBufError::InsufficientDescriptors);
        }
        // Evenly distribute the buffer between the descriptors.
43
        let chunk_size = Some(buffer.len() / descriptors.len())
43
            .filter(|x| *x <= 4095)
            .ok_or(DmaBufError::InsufficientDescriptors)?;
        let mut chunks = buffer.chunks_exact_mut(chunk_size);
        for (desc, chunk) in descriptors.iter_mut().zip(chunks.by_ref()) {
368
            desc.buffer = chunk.as_mut_ptr();
            desc.set_size(chunk.len());
        }
        let remainder = chunks.into_remainder();
43
        if !remainder.is_empty() {
            // Append any excess to the last descriptor.
            let last_descriptor = descriptors.last_mut().unwrap();
            let size = last_descriptor.size() + remainder.len();
            if size > 4095 {
                return Err(DmaBufError::InsufficientDescriptors);
            }
            last_descriptor.set_size(size);
        }
43
        Ok(Self {
            descriptors,
            buffer,
            burst: BurstConfig::default(),
        })
43
    }
    /// Consumes the buf, returning the descriptors and buffer.
    pub fn split(
        self,
    ) -> (
        DmaAlignedMut<'static, [DmaDescriptor]>,
        DmaAlignedMut<'static, [u8]>,
    ) {
        (self.descriptors, self.buffer)
    }
}
unsafe impl DmaRxBuffer for DmaRxStreamBuf {
    type View = DmaRxStreamBufView;
    type Final = DmaRxStreamBuf;
17
    fn prepare(&mut self) -> Preparation {
        // Link up all the descriptors (but not in a circle).
        let mut next = null_mut();
17
        for desc in self.descriptors.iter_mut().rev() {
68
            desc.next = next;
            next = desc;
            desc.reset_for_rx();
        }
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.descriptors.writeback();
17
        Preparation {
            start: self.descriptors.as_mut_ptr(),
            #[cfg(dma_can_access_psram)]
            accesses_psram: false,
17
            burst_transfer: self.burst,
            // Whilst we give ownership of the descriptors the DMA, the correctness of this buffer
            // implementation doesn't rely on the DMA checking for descriptor ownership.
            // No descriptor is added back to the end of the stream before it's ready for the DMA
            // to consume it.
            check_owner: None,
            auto_write_back: true,
        }
17
    }
    fn into_view(self) -> DmaRxStreamBufView {
        DmaRxStreamBufView {
            buf: self,
            descriptor_idx: 0,
            descriptor_offset: 0,
        }
    }
    fn from_view(view: Self::View) -> Self {
        view.buf
    }
}
/// A view into a [DmaRxStreamBuf].
pub struct DmaRxStreamBufView {
    buf: DmaRxStreamBuf,
    descriptor_idx: usize,
    descriptor_offset: usize,
}
impl DmaRxStreamBufView {
    /// Returns the number of bytes that are available to read from the buf.
7
    pub fn available_bytes(&mut self) -> usize {
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.buf.descriptors.invalidate();
7
        let (tail, head) = self.buf.descriptors.split_at(self.descriptor_idx);
        let mut result = 0;
        for desc in head.iter().chain(tail) {
12
            if desc.owner() == Owner::Dma {
                break;
            }
5
            result += desc.len();
        }
7
        result - self.descriptor_offset
7
    }
    /// Reads as much as possible into the buf from the available data.
2
    pub fn pop(&mut self, buf: &mut [u8]) -> usize {
2
        if buf.is_empty() {
            return 0;
        }
        let total_bytes = buf.len();
        let mut remaining = buf;
        loop {
            let available = self.peek();
2
            if available.is_empty() {
                break;
            }
2
            if available.len() >= remaining.len() {
                remaining.copy_from_slice(&available[0..remaining.len()]);
1
                self.consume(remaining.len());
                let consumed = remaining.len();
                remaining = &mut remaining[consumed..];
                break;
            } else {
                let to_consume = available.len();
                remaining[0..to_consume].copy_from_slice(available);
1
                self.consume(to_consume);
                remaining = &mut remaining[to_consume..];
            }
        }
2
        total_bytes - remaining.len()
2
    }
    /// Returns a slice into the buffer containing available data.
    /// This will be the longest possible contiguous slice into the buffer that
    /// contains data that is available to read.
    ///
    /// Ignores EOFs. See [Self::peek_until_eof] for EOF support.
5
    pub fn peek(&mut self) -> &[u8] {
5
        let (slice, _) = self.peek_internal(false);
        slice
5
    }
    /// Same as [Self::peek] but will not skip over any EOFs.
    ///
    /// It also returns a boolean indicating whether this slice ends with an EOF
    /// or not.
1
    pub fn peek_until_eof(&mut self) -> (&[u8], bool) {
1
        self.peek_internal(true)
1
    }
    /// Consumes the first `n` bytes from the available data, returning any
    /// fully consumed descriptors back to the DMA.
    /// This is typically called after [Self::peek]/[Self::peek_until_eof].
    ///
    /// Returns the number of bytes that were actually consumed.
4
    pub fn consume(&mut self, n: usize) -> usize {
        let mut remaining_bytes_to_consume = n;
        let mut descriptors_modified = false;
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.buf.descriptors.invalidate();
        loop {
4
            let desc = &mut self.buf.descriptors[self.descriptor_idx];
4
            if desc.owner() == Owner::Dma {
                // Descriptor is still owned by DMA so it can't be read yet.
                // This should only happen when there is no more data available to read.
                break;
            }
4
            let remaining_bytes_in_descriptor = desc.len() - self.descriptor_offset;
4
            if remaining_bytes_to_consume < remaining_bytes_in_descriptor {
1
                self.descriptor_offset += remaining_bytes_to_consume;
                remaining_bytes_to_consume = 0;
                break;
            }
            // Reset the descriptor for reuse.
            desc.set_owner(Owner::Dma);
            desc.set_suc_eof(false);
            desc.set_length(0);
            // Before connecting this descriptor to the end of the list, the next descriptor
            // must be disconnected from this one to prevent the DMA from
            // overtaking.
3
            desc.next = null_mut();
            let desc_ptr: *mut _ = desc;
            let prev_descriptor_index = self
                .descriptor_idx
                .checked_sub(1)
                .unwrap_or(self.buf.descriptors.len() - 1);
            // Connect this consumed descriptor to the end of the chain.
3
            self.buf.descriptors[prev_descriptor_index].next = desc_ptr;
            descriptors_modified = true;
3
            self.descriptor_idx += 1;
3
            if self.descriptor_idx >= self.buf.descriptors.len() {
                self.descriptor_idx = 0;
            }
3
            self.descriptor_offset = 0;
3
            remaining_bytes_to_consume -= remaining_bytes_in_descriptor;
        }
        if descriptors_modified {
            #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
            self.buf.descriptors.writeback();
        }
3
        n - remaining_bytes_to_consume
3
    }
9
    fn peek_internal(&mut self, stop_at_eof: bool) -> (&[u8], bool) {
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.buf.descriptors.invalidate();
9
        let descriptors = &self.buf.descriptors[self.descriptor_idx..];
        // There must be at least one descriptor.
9
        debug_assert!(!descriptors.is_empty());
        if descriptors.len() == 1 {
            let last_descriptor = &descriptors[0];
            if last_descriptor.owner() == Owner::Dma {
                // No data available.
                (&[], false)
            } else {
                let length = last_descriptor.len() - self.descriptor_offset;
                let chunk_size = last_descriptor.size();
                let buffer_start = self.buf.buffer.len() - chunk_size;
                #[cfg(soc_internal_memory_cached)]
                if length != 0 {
                    unsafe {
                        crate::soc::cache_invalidate_addr(
                            self.buf.buffer.as_ptr().add(buffer_start) as u32,
                            length as u32,
                        );
                    }
                }
                (
                    &self.buf.buffer[buffer_start..][..length],
                    last_descriptor.flags.suc_eof(),
                )
            }
        } else {
9
            let chunk_size = descriptors[0].size();
            let mut found_eof = false;
            let mut number_of_contiguous_bytes = 0;
            for desc in descriptors {
10
                if desc.owner() == Owner::Dma {
                    break;
                }
8
                number_of_contiguous_bytes += desc.len();
1
                if stop_at_eof && desc.flags.suc_eof() {
                    found_eof = true;
                    break;
                }
                // If the length is smaller than the size, the contiguous-ness ends here.
8
                if desc.len() < desc.size() {
                    break;
                }
            }
            #[cfg(soc_internal_memory_cached)]
            {
                let buffer_start = chunk_size * self.descriptor_idx + self.descriptor_offset;
                let buffer_len = number_of_contiguous_bytes - self.descriptor_offset;
                if buffer_len != 0 {
                    unsafe {
                        crate::soc::cache_invalidate_addr(
                            self.buf.buffer.as_ptr().add(buffer_start) as u32,
                            buffer_len as u32,
                        );
                    }
                }
            }
9
            (
9
                &self.buf.buffer[chunk_size * self.descriptor_idx..][..number_of_contiguous_bytes]
9
                    [self.descriptor_offset..],
                found_eof,
            )
        }
9
    }
}
/// DMA Streaming Transmit Buffer.
///
/// This is symmetric implementation to [DmaRxStreamBuf], used for continuously
/// streaming data to a peripheral's FIFO.
///
/// The list starts out like so `A(full) -> B(full) -> C(full) -> D(full) -> NULL`.
///
/// As the DMA writes to FIFO, the list progresses like so:
/// - `A(full)  -> B(full)  -> C(full)  -> D(full) -> NULL`
/// - `A(empty) -> B(full)  -> C(full)  -> D(full) -> NULL`
/// - `A(empty) -> B(empty) -> C(full)  -> D(full) -> NULL`
/// - `A(empty) -> B(empty) -> C(empty) -> D(full) -> NULL`
///
/// As [DmaTxStreamBufView::push] is called, the list (approximately) progresses like so:
/// - `A(empty) -> B(empty) -> C(empty) -> D(full) -> NULL`
/// - `B(empty) -> C(empty) -> D(full)  -> A(full) -> NULL`
/// - `C(empty) -> D(full)  -> A(full)  -> B(full) -> NULL`
/// - `D(full)  -> A(full)  -> B(full)  -> C(full) -> NULL`
///
/// If all the descriptors run out, the [DmaTxInterrupt::TotalEof] interrupt fires and the DMA
/// stops writing. The transfer must then be resumed or restarted.
#[derive(Debug)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct DmaTxStreamBuf {
    descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
    buffer: DmaAlignedMut<'static, [u8]>,
    burst: BurstConfig,
    pre_filled: Option<usize>,
    view_descriptor_idx: usize,
    view_descriptor_offset: usize,
}
impl DmaTxStreamBuf {
    /// Creates a new [`DmaTxStreamBuf`] evenly distributing the buffer between
    /// the provided descriptors.
35
    pub fn new(
        mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
        mut buffer: DmaAlignedMut<'static, [u8]>,
    ) -> Result<Self, DmaBufError> {
35
        if descriptors.len() < 4 {
            // see https://github.com/esp-rs/esp-hal/issues/2269#issuecomment-4397953660
            // we can lift that requirement once we sort out this issue
            return Err(DmaBufError::InsufficientDescriptors);
        }
        // Evenly distribute the buffer between the descriptors.
34
        let chunk_size = Some(buffer.len() / descriptors.len())
34
            .filter(|x| *x <= 4095)
            .ok_or(DmaBufError::InsufficientDescriptors)?;
        let mut chunks = buffer.chunks_exact_mut(chunk_size);
        for (desc, chunk) in descriptors.iter_mut().zip(chunks.by_ref()) {
332
            desc.buffer = chunk.as_mut_ptr();
            desc.set_size(chunk.len());
            desc.set_length(chunk.len());
        }
        let remainder = chunks.into_remainder();
34
        if !remainder.is_empty() {
            // Append any excess to the last descriptor.
            let last_descriptor = descriptors.last_mut().unwrap();
            let size = last_descriptor.size() + remainder.len();
            if size > 4095 {
                Err(DmaBufError::InsufficientDescriptors)?;
            }
            last_descriptor.set_size(size);
        }
34
        Ok(Self {
            descriptors,
            buffer,
            burst: Default::default(),
            pre_filled: None,
            view_descriptor_idx: 0,
            view_descriptor_offset: 0,
        })
34
    }
    /// Consumes the buf, returning the descriptors and buffer.
    pub fn split(
        self,
    ) -> (
        DmaAlignedMut<'static, [DmaDescriptor]>,
        DmaAlignedMut<'static, [u8]>,
    ) {
        (self.descriptors, self.buffer)
    }
    /// Pushes the buffer with the given data before DMA transfer starts.
    ///
    /// It is expected to pre-fill at least enough data to fill the first two descriptors' buffers.
    /// The more data is pre-filled, the more head-room is left to push more data.
    ///
    /// Pre-filled data only applies to the next transfer: a buffer returned from a
    /// transfer starts with nothing pre-filled. If nothing is pushed before a
    /// transfer starts, the entire buffer is transmitted with its current contents.
12
    pub fn push(&mut self, data: &[u8]) -> usize {
        self.push_with(|buf| {
            let len = buf.len().min(data.len());
            buf[..len].copy_from_slice(&data[..len]);
            len
        })
12
    }
    /// Pushes the buffer with the given data before DMA transfer starts.
    ///
    /// It is expected to pre-fill at least enough data to fill the first two descriptors' buffers.
    /// The more data is pre-filled, the more head-room is left to push more data.
    ///
    /// Pre-filled data only applies to the next transfer: a buffer returned from a
    /// transfer starts with nothing pre-filled. If nothing is pushed before a
    /// transfer starts, the entire buffer is transmitted with its current contents.
    ///
    /// Returns the number of bytes filled.
    pub fn push_with(&mut self, f: impl FnOnce(&mut [u8]) -> usize) -> usize {
12
        let start = self.pre_filled.unwrap_or(0);
1
        let bytes_pushed = f(&mut self.buffer[start..]);
12
        self.pre_filled = Some(start + bytes_pushed);
        bytes_pushed
    }
    fn setup_view_state(&mut self) {
9
        let pre_filled = self.pre_filled.unwrap_or(self.buffer.len());
        let (idx, offset) = mark_tx_stream_descriptors_ready(&mut self.descriptors, pre_filled);
9
        self.view_descriptor_idx = idx;
9
        self.view_descriptor_offset = offset;
        #[cfg(soc_internal_memory_cached)]
        if pre_filled != 0 {
            unsafe {
                crate::soc::cache_writeback_addr(self.buffer.as_ptr() as u32, pre_filled as u32);
            }
        }
    }
}
/// Marks descriptors containing data that should be transmitted when the DMA
/// channel starts.
fn mark_tx_stream_descriptors_ready(
    descriptors: &mut DmaAlignedMut<'_, [DmaDescriptor]>,
    bytes_pushed: usize,
) -> (usize, usize) {
9
    if bytes_pushed == 0 {
        return (0, 0);
    }
    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
    descriptors.invalidate();
    let num = descriptors.len();
    let mut bytes_filled = 0;
    let mut cursor = (0, 0);
    for d in 0..num {
31
        let remaining = bytes_pushed - bytes_filled;
        let size = descriptors[d].size();
31
        if remaining == 0 {
3
            terminate_tx_stream_at(descriptors, d);
            cursor = (d, 0);
            break;
        }
28
        if remaining < size {
2
            if d == 0 {
                // The transfer needs at least one descriptor; send the partial chunk and
                // continue filling from the next one.
                descriptors[d].set_owner(Owner::Dma);
                descriptors[d].set_length(remaining);
                descriptors[d].set_suc_eof(true);
                if num > 1 {
                    terminate_tx_stream_at(descriptors, 1);
                    cursor = (1, 0);
                } else {
                    descriptors[d].next = null_mut();
                }
            } else {
2
                terminate_tx_stream_at(descriptors, d);
                cursor = (d, remaining);
            }
            break;
        }
26
        bytes_filled += size;
        descriptors[d].set_owner(Owner::Dma);
        descriptors[d].set_length(size);
        descriptors[d].set_suc_eof(true);
    }
    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
    descriptors.writeback();
    cursor
}
5
fn terminate_tx_stream_at(descriptors: &mut DmaAlignedMut<'_, [DmaDescriptor]>, start: usize) {
5
    if start > 0 {
5
        descriptors[start - 1].next = null_mut();
    }
5
    for desc in descriptors.iter_mut().skip(start) {
        desc.set_owner(Owner::Cpu);
    }
5
}
fn advance_tx_stream_descriptors(
    descriptors: &mut DmaAlignedMut<'_, [DmaDescriptor]>,
    descriptor_idx: &mut usize,
    descriptor_offset: &mut usize,
    bytes_pushed: usize,
) {
6
    if bytes_pushed == 0 {
        return;
    }
    let mut bytes_filled = 0;
    let num_descriptors = descriptors.len();
    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
    descriptors.invalidate();
    for i in 0..num_descriptors {
8
        let d = (*descriptor_idx + i) % num_descriptors;
8
        let desc = &mut descriptors[d];
6
        let bytes_in_d = desc.size() - *descriptor_offset;
8
        if bytes_in_d + bytes_filled > bytes_pushed {
3
            *descriptor_idx = d;
5
            *descriptor_offset = *descriptor_offset + bytes_pushed - bytes_filled;
            #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
            descriptors.writeback();
            return;
        }
        bytes_filled += bytes_in_d;
5
        *descriptor_offset = 0;
        // Put the current descriptor at the end of the list
        desc.set_owner(Owner::Dma);
        desc.set_length(desc.size());
        desc.set_suc_eof(true);
5
        let p = d.checked_sub(1).unwrap_or(num_descriptors - 1);
5
        if p != d {
            let [prev, desc] = descriptors.get_disjoint_mut([p, d]).unwrap();
5
            desc.next = null_mut();
5
            prev.next = desc;
        }
    }
    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
    descriptors.writeback();
}
unsafe impl DmaTxBuffer for DmaTxStreamBuf {
    type View = DmaTxStreamBufView;
    type Final = Self;
9
    fn prepare(&mut self) -> Preparation {
        // Link up all the descriptors (but not in a circle).
        let mut next = null_mut();
9
        for desc in self.descriptors.iter_mut().rev() {
36
            desc.next = next;
            desc.set_owner(Owner::Dma);
            next = desc;
        }
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.descriptors.writeback();
        self.setup_view_state();
9
        Preparation {
            start: self.descriptors.as_mut_ptr(),
            #[cfg(dma_can_access_psram)]
            accesses_psram: false,
9
            burst_transfer: self.burst,
            // Whilst we give ownership of the descriptors the DMA, the correctness of this buffer
            // implementation doesn't rely on the DMA checking for descriptor ownership.
            // No descriptor is added back to the end of the stream before it's ready for the DMA
            // to consume it.
            check_owner: None,
            auto_write_back: true,
        }
9
    }
    fn into_view(self) -> Self::View {
1
        DmaTxStreamBufView {
            descriptor_idx: self.view_descriptor_idx,
            descriptor_offset: self.view_descriptor_offset,
            buf: self,
        }
    }
    fn from_view(view: Self::View) -> Self {
        let DmaTxStreamBufView {
1
            mut buf,
            descriptor_idx,
            descriptor_offset,
        } = view;
1
        buf.view_descriptor_idx = descriptor_idx;
1
        buf.view_descriptor_offset = descriptor_offset;
1
        buf.pre_filled = None;
        buf
    }
}
/// A view into a [DmaTxStreamBuf].
pub struct DmaTxStreamBufView {
    buf: DmaTxStreamBuf,
    descriptor_idx: usize,
    descriptor_offset: usize,
}
impl DmaTxStreamBufView {
    /// Returns the number of bytes available for writing.
21
    pub fn available_bytes(&mut self) -> usize {
        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
        self.buf.descriptors.invalidate();
21
        let (tail, head) = self.buf.descriptors.split_at(self.descriptor_idx);
        head.iter()
            .chain(tail)
            .take_while(|d| d.owner() == Owner::Cpu)
            .map(|d| d.size())
            .sum::<usize>()
21
            .saturating_sub(self.descriptor_offset)
21
    }
    fn write_position(&self) -> usize {
3
        let desc = &self.buf.descriptors[self.descriptor_idx];
3
        desc.buffer
            .addr()
            .wrapping_sub(self.buf.buffer.as_ptr().addr())
3
            + self.descriptor_offset
    }
    /// Pushes a buffer into the stream buffer.
    /// Returns the number of bytes pushed.
    pub fn push_with(&mut self, f: impl FnOnce(&mut [u8]) -> usize) -> usize {
        let dma_start = self.write_position();
        let dma_end = dma_start
3
            .saturating_add(self.available_bytes())
            .min(self.buf.buffer.len())
            .max(dma_start);
3
        let bytes_pushed = f(&mut self.buf.buffer[dma_start..dma_end]).min(dma_end - dma_start);
        #[cfg(soc_internal_memory_cached)]
        if bytes_pushed != 0 {
            unsafe {
                crate::soc::cache_writeback_addr(
                    self.buf.buffer.as_ptr().add(dma_start) as u32,
                    bytes_pushed as u32,
                );
            }
        }
3
        self.advance(bytes_pushed);
        bytes_pushed
    }
    /// Advances the first `n` bytes from the available data.
6
    pub fn advance(&mut self, bytes_pushed: usize) {
6
        advance_tx_stream_descriptors(
            &mut self.buf.descriptors,
            &mut self.descriptor_idx,
            &mut self.descriptor_offset,
            bytes_pushed,
        );
5
    }
    /// Pushes a buffer into the stream buffer.
    /// Returns the number of bytes pushed.
3
    pub fn push(&mut self, data: &[u8]) -> usize {
        let total_len = data.len();
        let mut remaining = data;
4
        while !remaining.is_empty() && self.available_bytes() > 0 {
            let written = self.push_with(|buffer| {
                let len = usize::min(buffer.len(), remaining.len());
                buffer[..len].copy_from_slice(&remaining[..len]);
                len
            });
3
            if written == 0 {
                break;
            }
            remaining = &remaining[written..];
        }
3
        total_len - remaining.len()
3
    }
}
static mut EMPTY: InternalMemory<[DmaDescriptor; 1]> = InternalMemory::new([DmaDescriptor::EMPTY]);
/// An empty buffer for transfers that carry no data.
pub struct EmptyBuf;
unsafe impl DmaTxBuffer for EmptyBuf {
    type View = EmptyBuf;
    type Final = EmptyBuf;
    fn prepare(&mut self) -> Preparation {
        #[cfg(soc_internal_memory_cached)]
        #[allow(static_mut_refs)]
        unsafe {
            EMPTY.get_mut().writeback();
        }
        Preparation {
            start: (&raw mut EMPTY).cast(),
            #[cfg(dma_can_access_psram)]
            accesses_psram: false,
            burst_transfer: BurstConfig::default(),
            // As we don't give ownership of the descriptor to the DMA, it's important that the DMA
            // channel does *NOT* check for ownership, otherwise the channel will return an error.
            check_owner: Some(false),
            // The DMA should not write back to the descriptor as it is shared.
            auto_write_back: false,
        }
    }
    fn into_view(self) -> EmptyBuf {
        self
    }
    fn from_view(view: Self::View) -> Self {
        view
    }
}
unsafe impl DmaRxBuffer for EmptyBuf {
    type View = EmptyBuf;
    type Final = EmptyBuf;
    fn prepare(&mut self) -> Preparation {
        #[cfg(soc_internal_memory_cached)]
        #[allow(static_mut_refs)]
        unsafe {
            EMPTY.get_mut().writeback();
        }
        Preparation {
            start: (&raw mut EMPTY).cast(),
            #[cfg(dma_can_access_psram)]
            accesses_psram: false,
            burst_transfer: BurstConfig::default(),
            // As we don't give ownership of the descriptor to the DMA, it's important that the DMA
            // channel does *NOT* check for ownership, otherwise the channel will return an error.
            check_owner: Some(false),
            auto_write_back: true,
        }
    }
    fn into_view(self) -> EmptyBuf {
        self
    }
    fn from_view(view: Self::View) -> Self {
        view
    }
}
/// DMA Loop Buffer
///
/// This consists of a single descriptor that points to itself and points to a
/// single buffer, resulting in the buffer being transmitted over and over
/// again, indefinitely.
///
/// A DMA descriptor is 12 bytes. If the buffer is significantly shorter
/// than this, the DMA channel spends more time reading the descriptor than
/// it does reading the buffer, which may leave it unable to keep up with the
/// bandwidth requirements of some peripherals at high frequencies.
pub struct DmaLoopBuf {
    descriptor: DmaAlignedMut<'static, [DmaDescriptor]>,
    buffer: DmaAlignedMut<'static, [u8]>,
}
impl DmaLoopBuf {
    /// Creates a new [DmaLoopBuf].
    pub fn new(
        mut descriptors: DmaAlignedMut<'static, [DmaDescriptor]>,
        mut buffer: DmaAlignedMut<'static, [u8]>,
    ) -> Result<DmaLoopBuf, DmaBufError> {
        if buffer.len() > BurstConfig::default().max_chunk_size_for(&buffer, TransferDirection::Out)
        {
            return Err(DmaBufError::InsufficientDescriptors);
        }
        descriptors[0].set_owner(Owner::Dma); // Doesn't matter
        descriptors[0].set_suc_eof(false);
        descriptors[0].set_length(buffer.len());
        descriptors[0].set_size(buffer.len());
        descriptors[0].buffer = buffer.as_mut_ptr();
        descriptors[0].next = descriptors.as_mut_ptr();
        Ok(Self {
            descriptor: descriptors,
            buffer,
        })
    }
    /// Consumes the buf, returning the descriptor and buffer.
    pub fn split(
        self,
    ) -> (
        DmaAlignedMut<'static, [DmaDescriptor]>,
        DmaAlignedMut<'static, [u8]>,
    ) {
        (self.descriptor, self.buffer)
    }
}
unsafe impl DmaTxBuffer for DmaLoopBuf {
    type View = DmaLoopBuf;
    type Final = DmaLoopBuf;
    fn prepare(&mut self) -> Preparation {
        Preparation {
            start: self.descriptor.as_mut_ptr(),
            #[cfg(dma_can_access_psram)]
            accesses_psram: false,
            burst_transfer: BurstConfig::default(),
            // The DMA must not check the owner bit, as it is never set.
            check_owner: Some(false),
            // Doesn't matter either way but it is set to true for ESP32 SPI_DMA compatibility.
            auto_write_back: false,
        }
    }
    fn into_view(self) -> Self::View {
        self
    }
    fn from_view(view: Self::View) -> Self {
        view
    }
}
impl Deref for DmaLoopBuf {
    type Target = [u8];
    fn deref(&self) -> &Self::Target {
        &self.buffer
    }
}
impl DerefMut for DmaLoopBuf {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.buffer
    }
}
/// A Preparation that masks itself as a DMA buffer.
///
/// For low-level use, where none of the pre-made buffers really fit.
///
/// Intended for low-level use inside esp-hal only.
pub(crate) struct NoBuffer(pub(crate) Preparation);
impl NoBuffer {
    fn prep(&self) -> Preparation {
1
        Preparation {
24
            start: self.0.start,
            #[cfg(dma_can_access_psram)]
            accesses_psram: self.0.accesses_psram,
24
            burst_transfer: self.0.burst_transfer,
24
            check_owner: self.0.check_owner,
23
            auto_write_back: self.0.auto_write_back,
        }
    }
}
unsafe impl DmaTxBuffer for NoBuffer {
    type View = ();
    type Final = ();
    fn prepare(&mut self) -> Preparation {
        self.prep()
    }
    fn into_view(self) -> Self::View {}
    fn from_view(_view: Self::View) {}
}
unsafe impl DmaRxBuffer for NoBuffer {
    type View = ();
    type Final = ();
    fn prepare(&mut self) -> Preparation {
        self.prep()
    }
    fn into_view(self) -> Self::View {}
    fn from_view(_view: Self::View) {}
}
/// Prepares data unsafely to be transmitted via DMA.
///
/// `block_size` is the requirement imposed by the peripheral that receives the data. It
/// ensures that the DMA will not try to copy a partial block, which would cause the RX DMA (that
/// moves results back into RAM) to never complete.
///
/// The function returns the DMA buffer, and the number of bytes that will be transferred.
///
/// # Safety
///
/// The caller must keep all its descriptors and the buffers they
/// point to valid while the buffer is being transferred.
#[cfg_attr(not(any(aes_supports_dma, spi_master_supports_dma)), expect(unused))]
291
pub(crate) unsafe fn prepare_for_tx(
    descriptors: &mut [DmaDescriptor],
    mut data: NonNull<[u8]>,
    block_size: usize,
) -> Result<(NoBuffer, usize), DmaError> {
    let alignment =
        BurstConfig::DEFAULT.min_alignment(unsafe { data.as_ref() }, TransferDirection::Out);
    if !data.addr().get().is_multiple_of(alignment) {
        // ESP32 has word alignment requirement on the TX descriptors, too.
        return Err(DmaError::InvalidAlignment(DmaAlignmentError::Address));
    }
    // Whichever is stricter, data location or peripheral requirements.
    //
    // This ensures that the RX DMA, if used, can transfer the returned number of bytes using at
    // most N+2 descriptors. While the hardware doesn't require this on the TX DMA side, (the TX DMA
    // can, except on the ESP32, transfer any amount of data), it makes usage MUCH simpler.
    let alignment = alignment.max(block_size);
291
    let chunk_size = 4096 - alignment;
291
    let data_len = data.len().min(chunk_size * descriptors.len());
    cfg_select! {
        dma_can_access_psram => {
            let data_addr = data.addr().get();
            let data_in_psram = crate::psram::psram_range().contains(&data_addr);
            // Make sure input data is in PSRAM instead of cache
            if data_in_psram || cfg!(soc_internal_memory_cached) {
                unsafe { crate::soc::cache_writeback_addr(data_addr as u32, data_len as u32) };
            }
        }
        soc_internal_memory_cached => {
            unsafe { crate::soc::cache_writeback_addr(data.addr().get() as u32, data_len as u32) };
        }
        _ => {}
    }
    let descriptors = unsafe { DmaAlignedMut::new_unchecked(descriptors) };
291
    let mut descriptors = unwrap!(DescriptorSet::new(descriptors));
    // TODO: it would be best if this function returned the amount of data that could be linked
    // up.
291
    unwrap!(descriptors.link_with_buffer(unsafe { data.as_mut() }, chunk_size));
    unwrap!(descriptors.set_tx_length(data_len, chunk_size));
    for desc in descriptors.linked_iter_mut() {
        desc.reset_for_tx(desc.next.is_null());
    }
    #[cfg(soc_internal_memory_cached)]
    descriptors.descriptors.writeback();
291
    Ok((
        NoBuffer(Preparation {
            start: descriptors.head(),
            burst_transfer: BurstConfig::DEFAULT,
            check_owner: None,
            auto_write_back: false,
            #[cfg(dma_can_access_psram)]
            accesses_psram: data_in_psram,
        }),
        data_len,
    ))
291
}
/// Prepares buffers to receive data from DMA.
///
/// The function returns the DMA buffer, and the number of bytes that will be transferred.
///
/// # Safety
///
/// The caller must keep all its descriptors and the buffers they
/// point to valid while the buffer is being transferred.
#[cfg_attr(not(any(aes_supports_dma, spi_master_supports_dma)), expect(unused))]
297
pub(crate) unsafe fn prepare_for_rx(
    descriptors: &mut [DmaDescriptor],
    #[cfg(dma_can_access_psram)] align_buffers: &mut [Option<ManualWritebackBuffer>; 2],
    mut data: NonNull<[u8]>,
) -> (NoBuffer, usize) {
    let chunk_size =
        BurstConfig::DEFAULT.max_chunk_size_for(unsafe { data.as_ref() }, TransferDirection::In);
    // The data we have to process may not be appropriate for the DMA:
    // - it may be improperly aligned for PSRAM
    // - it may not have a length that is a multiple of the external memory block size
    cfg_select! {
        dma_can_access_psram => {
            let data_addr = data.addr().get();
            let data_in_psram = crate::psram::psram_range().contains(&data_addr);
        }
        _ => {
            let data_in_psram = false;
        }
    }
    let descriptors = unsafe { DmaAlignedMut::new_unchecked(descriptors) };
297
    let mut descriptors = unwrap!(DescriptorSet::new(descriptors));
    let data_len = if data_in_psram {
        cfg_select! {
            dma_can_access_psram => {
                // This could use a better API, but right now we'll have to build the descriptor
                // list by hand.
                let consumed_bytes =
                    build_descriptor_list_for_psram(&mut descriptors, align_buffers, data);
                // Invalidate data written by the DMA. As this likely affects more data than we
                // touched, write back first.
                unsafe {
                    crate::soc::cache_writeback_addr(data_addr as u32, consumed_bytes as u32);
                    crate::soc::cache_invalidate_addr(data_addr as u32, consumed_bytes as u32);
                }
                consumed_bytes
            }
            _ => {
                unreachable!()
            }
        }
    } else {
        // Just set up descriptors as usual
        let data_len = data.len();
297
        unwrap!(descriptors.link_with_buffer(unsafe { data.as_mut() }, chunk_size));
        unwrap!(descriptors.set_tx_length(data_len, chunk_size));
        #[cfg(soc_internal_memory_cached)]
        // Invalidate data written by the DMA. As this likely affects more data than we touched,
        // write back first.
        unsafe {
            crate::soc::cache_writeback_addr(data.addr().get() as u32, data_len as u32);
            crate::soc::cache_invalidate_addr(data.addr().get() as u32, data_len as u32);
        }
        data_len
    };
    for desc in descriptors.linked_iter_mut() {
        desc.reset_for_rx();
    }
    #[cfg(soc_internal_memory_cached)]
    descriptors.descriptors.writeback();
297
    (
        NoBuffer(Preparation {
            start: descriptors.head(),
            burst_transfer: BurstConfig::DEFAULT,
            check_owner: None,
            auto_write_back: true,
            #[cfg(dma_can_access_psram)]
            accesses_psram: data_in_psram,
        }),
        data_len,
    )
297
}
#[cfg(dma_can_access_psram)]
fn build_descriptor_list_for_psram(
    descriptors: &mut DescriptorSet<'_>,
    copy_buffers: &mut [Option<ManualWritebackBuffer>; 2],
    data: NonNull<[u8]>,
) -> usize {
    let data_len = data.len();
    let data_addr = data.addr().get();
    let min_alignment = ExternalBurstConfig::DEFAULT.min_psram_alignment(TransferDirection::In);
    let chunk_size = 4096 - min_alignment;
    let mut desciptor_iter = DescriptorChainingIter::new(&mut descriptors.descriptors);
    let mut copy_buffer_iter = copy_buffers.iter_mut();
    // MIN_LAST_DMA_LEN could make this really annoying, so we're just allocating a bit larger
    // buffer and shove edge cases into a single one. If we have >24 bytes on the S2, the 2-buffer
    // alignment algo works fine as one of them can steal 16 bytes, the other will have
    // MIN_LAST_DMA_LEN data to work with.
    let has_aligned_data = data_len > BUF_LEN;
    // Calculate byte offset to the start of the buffer. An already-aligned
    // buffer needs no head at all.
    let offset = data_addr % min_alignment;
    let head_to_copy = (min_alignment - offset) % min_alignment;
    let head_to_copy = if !has_aligned_data {
        BUF_LEN
    } else if head_to_copy > 0 && head_to_copy < MIN_LAST_DMA_LEN {
        head_to_copy + min_alignment
    } else {
        head_to_copy
    };
    let head_to_copy = head_to_copy.min(data_len);
    // Calculate last unaligned part
    let tail_to_copy = (data_len - head_to_copy) % min_alignment;
    let tail_to_copy = if tail_to_copy > 0 && tail_to_copy < MIN_LAST_DMA_LEN {
        tail_to_copy + min_alignment
    } else {
        tail_to_copy
    };
    let mut consumed = 0;
    // Align beginning
    if head_to_copy > 0 {
        let copy_buffer = unwrap!(copy_buffer_iter.next());
        let buffer =
            copy_buffer.insert(ManualWritebackBuffer::new(get_range(data, 0..head_to_copy)));
        buffer.prepare_for_dma();
        let Some(descriptor) = desciptor_iter.next() else {
            return consumed;
        };
        descriptor.set_size(head_to_copy);
        descriptor.buffer = buffer.mut_buffer_ptr();
        consumed += head_to_copy;
    };
    // Chain up descriptors for the main aligned data part.
    let mut aligned_data = get_range(data, head_to_copy..data.len() - tail_to_copy);
    while !aligned_data.is_empty() {
        let Some(descriptor) = desciptor_iter.next() else {
            return consumed;
        };
        let chunk = aligned_data.len().min(chunk_size);
        descriptor.set_size(chunk);
        descriptor.buffer = aligned_data.cast::<u8>().as_ptr();
        consumed += chunk;
        aligned_data = get_range(aligned_data, chunk..aligned_data.len());
    }
    // Align end
    if tail_to_copy > 0 {
        let copy_buffer = unwrap!(copy_buffer_iter.next());
        let buffer = copy_buffer.insert(ManualWritebackBuffer::new(get_range(
            data,
            data.len() - tail_to_copy..data.len(),
        )));
        buffer.prepare_for_dma();
        let Some(descriptor) = desciptor_iter.next() else {
            return consumed;
        };
        descriptor.set_size(tail_to_copy);
        descriptor.buffer = buffer.mut_buffer_ptr();
        consumed += tail_to_copy;
    }
    consumed
}
#[cfg(dma_can_access_psram)]
fn get_range(ptr: NonNull<[u8]>, range: Range<usize>) -> NonNull<[u8]> {
    let len = range.end - range.start;
    NonNull::slice_from_raw_parts(unsafe { ptr.cast().byte_add(range.start) }, len)
}
#[cfg(dma_can_access_psram)]
struct DescriptorChainingIter<'a> {
    /// index of the next element to emit
    index: usize,
    descriptors: &'a mut [DmaDescriptor],
}
#[cfg(dma_can_access_psram)]
impl<'a> DescriptorChainingIter<'a> {
    fn new(descriptors: &'a mut [DmaDescriptor]) -> Self {
        Self {
            descriptors,
            index: 0,
        }
    }
    fn next(&mut self) -> Option<&'_ mut DmaDescriptor> {
        if self.index == 0 {
            self.index += 1;
            self.descriptors.get_mut(0)
        } else if self.index < self.descriptors.len() {
            let index = self.index;
            self.index += 1;
            // Grab a pointer to the current descriptor.
            let ptr = &raw mut self.descriptors[index];
            // Link the descriptor to the previous one.
            self.descriptors[index - 1].next = ptr;
            // Reborrow the pointer so that it doesn't get invalidated by our continued use of the
            // descriptor reference.
            Some(unsafe { &mut *ptr })
        } else {
            None
        }
    }
}
#[cfg(dma_can_access_psram)]
const MIN_LAST_DMA_LEN: usize = if cfg!(esp32s2) { 5 } else { 1 };
#[cfg(dma_can_access_psram)]
const BUF_LEN: usize = 16 + 2 * (MIN_LAST_DMA_LEN - 1); // 2x makes aligning short buffers simpler
/// PSRAM helper. DMA can write data of any alignment into this buffer, and it can be written by
/// the CPU back to PSRAM.
#[cfg(dma_can_access_psram)]
pub(crate) struct ManualWritebackBuffer {
    buffer: InternalMemory<MaybeUninit<[u8; BUF_LEN]>>,
    dst_address: NonNull<u8>,
    n_bytes: u8,
}
#[cfg(dma_can_access_psram)]
impl ManualWritebackBuffer {
    pub fn new(ptr: NonNull<[u8]>) -> Self {
        assert!(ptr.len() <= BUF_LEN);
        Self {
            buffer: InternalMemory::new(MaybeUninit::uninit()),
            dst_address: ptr.cast(),
            n_bytes: ptr.len() as u8,
        }
    }
    pub fn prepare_for_dma(&mut self) {
        // Ensure our cache line is not dirty. A dirty cacheline
        // evicted during DMA operation can clobber received data.
        #[cfg(soc_internal_memory_cached)]
        self.buffer.get_mut().invalidate();
    }
    pub fn write_back(&mut self) {
        // The DMA wrote its data directly to memory, bypassing the CPU cache.
        // Invalidate the cache lines covering the alignment buffer so the CPU
        // reads the fresh DMA data rather than the stale zeros from new().
        #[cfg(soc_internal_memory_cached)]
        self.buffer.get_mut().invalidate();
        let src = self.mut_buffer_ptr().cast_const();
        unsafe {
            self.dst_address
                .as_ptr()
                .copy_from(src, self.n_bytes as usize);
        }
    }
    pub fn mut_buffer_ptr(&mut self) -> *mut u8 {
        self.buffer.get_mut().as_mut_ptr().cast::<u8>()
    }
}