use core::{

    cell::{Cell, UnsafeCell},

    cmp::min,

    mem::{ManuallyDrop, MaybeUninit},

    pin::Pin,

    ptr::NonNull,

    sync::atomic::{Ordering, fence},

    task::{Context, Poll},

};

#[cfg(feature = "unstable")]

use embedded_hal::spi::{ErrorType, SpiBus};

use enumset::EnumSet;

#[cfg(place_spi_master_driver_in_ram)]

use procmacros::ram;

use super::*;

use crate::{

    RegisterToggle,

    dma::{

        CHUNK_SIZE,

        Channel,

        DmaDescriptor,

        DmaEligiblePeripheral,

        DmaRxBuf,

        DmaRxBuffer,

        DmaTxBuf,

        DmaTxBuffer,

        NoBuffer,

        ScopedDmaRxBuf,

        ScopedDmaTxBuf,

        TransferDirection,

        aligned::{DmaAlignedMut, InternalMemory},

        asynch::DmaRxFuture,

        prepare_for_rx,

        prepare_for_tx,

    },

    pac::spi2::RegisterBlock,

    private::DropGuard,

    soc::is_slice_in_dram,

    spi::{DmaError, master::low_level::SpiClockGuard},

};

#[cfg(dma_can_access_psram)]

use crate::{dma::ManualWritebackBuffer, soc::is_slice_in_psram};

const MAX_DMA_SIZE: usize = 32736;

impl<'d> Spi<'d, Blocking> {

    #[doc_replace(

        "dma_channel" => {

            cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",

            cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",

            cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",

        }

    )]

    /// Converts the driver into an [`SpiDma`] driver that uses the specified DMA channel.

    ///

    /// ```rust, no_run

    /// # {before_snippet}

    /// use esp_hal::spi::{

    ///     Mode,

    ///     master::{Config, Spi},

    /// };

    ///

    /// let mut spi_dma = Spi::new(

    ///     peripherals.SPI2,

    ///     Config::default()

    ///         .with_frequency(Rate::from_khz(100))

    ///         .with_mode(Mode::_0),

    /// )?

    /// .with_dma(peripherals.__dma_channel__);

    /// # {after_snippet}

    /// ```

    #[instability::unstable]

    pub fn with_dma(

        self,

        channel: impl SpiMasterDmaChannel<'d, AnySpi<'d>>,

    ) -> SpiDma<'d, crate::Blocking> {

    }

}

#[doc_replace(

    "dma_channel" => {

        cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",

        cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",

        cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",

    }

)]

/// DMA-controlled SPI driver.

///

/// This driver uses DMA to transfer data, allowing the CPU to continue working while the SPI

/// transfer is in progress.

///

/// The driver provides two separate approaches to transferring data:

///

/// - The slice-based API allows transferring data from/to slices of memory. The data may be copied

///   into an internal buffer before the transfer begins. A pair of copy buffers can be set up by

///   passing them to [`with_buffers`](SpiDma::with_buffers) before the first transfer begins. For

///   more details on when copying is necessary, see the documentation of the

///   [`with_buffers`](SpiDma::with_buffers) method.

/// - The buffer API allows transferring externally managed buffers. In this mode, the buffers to be

///   transferred are provided by the caller. The buffer objects ensure that data is located in

///   appropriate memory regions. The buffers and the driver object are moved into transfer objects

///   for the duration of the transfer. These functions take [`DmaRxBuf`] and [`DmaTxBuf`] objects

///   as arguments as well as the number of bytes to transfer, and their names end with `_buffer`

///

/// These approaches provide different trade-offs between memory usage / CPU overhead and ease of

/// use. `embedded-hal` traits are implemented by the slice-based API's functions.

///

/// # Examples

///

/// ```rust, no_run

/// # {before_snippet}

/// use esp_hal::{

///     dma::{DmaRxBuf, DmaTxBuf},

///     dma_rx_buffer,

///     dma_tx_buffer,

///     spi::{

///         Mode,

///         master::{Config, Spi},

///     },

/// };

///

/// // Optional: create and set up copy buffers.

/// let dma_rx_buf = dma_rx_buffer!(32000)?;

/// let dma_tx_buf = dma_tx_buffer!(32000)?;

///

/// let mut spi = Spi::new(

///     peripherals.SPI2,

///     Config::default()

///         .with_frequency(Rate::from_khz(100))

///         .with_mode(Mode::_0),

/// )?

/// .with_dma(peripherals.__dma_channel__)

/// .with_buffers(dma_rx_buf, dma_tx_buf);

/// #

/// # {after_snippet}

/// ```

#[cfg_attr(feature = "defmt", derive(defmt::Format))]

pub struct SpiDma<'d, Dm>

where

    Dm: DriverMode,

{

    spi: SpiWrapper<'d>,

    pub(crate) channel: Channel<Dm, SpiMasterErased<'d>>,

}

impl<Dm> crate::private::Sealed for SpiDma<'_, Dm> where Dm: DriverMode {}

impl<Dm> core::fmt::Debug for SpiDma<'_, Dm>

where

    Dm: DriverMode + core::fmt::Debug,

{

    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {

        f.debug_struct("SpiDma").field("spi", &self.spi).finish()

    }

}

#[instability::unstable]

impl crate::interrupt::InterruptConfigurable for SpiDma<'_, Blocking> {

    fn set_interrupt_handler(&mut self, handler: InterruptHandler) {

        self.set_interrupt_handler(handler);

    }

}

#[instability::unstable]

impl<Dm> embassy_embedded_hal::SetConfig for SpiDma<'_, Dm>

where

    Dm: DriverMode,

{

    type Config = Config;

    type ConfigError = ConfigError;

    fn set_config(&mut self, config: &Self::Config) -> Result<(), Self::ConfigError> {

        self.apply_config(config)

    }

}

#[instability::unstable]

impl<Dm> ErrorType for SpiDma<'_, Dm>

where

    Dm: DriverMode,

{

    type Error = Error;

}

#[instability::unstable]

impl<Dm> SpiBus for SpiDma<'_, Dm>

where

    Dm: DriverMode,

{

    fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {

        self.read(words)

    }

    fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {

        self.write(words)

    }

    fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {

        self.transfer(read, write)

    }

    fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {

        self.transfer_in_place(words)

    }

    fn flush(&mut self) -> Result<(), Self::Error> {

        // DMA limitation - we must ensure the transfers complete before returning

        // to user code, otherwise the user might access the buffers while the transfer

        // is still in progress. Therefore, there is no such thing as "flushing".

        Ok(())

    }

}

#[instability::unstable]

impl embedded_hal_async::spi::SpiBus for SpiDma<'_, Async> {

    async fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {

        self.read_async(words).await

    }

    async fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {

        self.write_async(words).await

    }

    async fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {

        self.transfer_async(read, write).await

    }

    async fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {

        self.transfer_in_place_async(words).await

    }

    async fn flush(&mut self) -> Result<(), Self::Error> {

        // DMA limitation - we must ensure the transfers complete before returning

        // to user code, otherwise the user might access the buffers while the transfer

        // is still in progress. Therefore, there is no such thing as "flushing".

        Ok(())

    }

}

impl<'d> SpiDma<'d, Blocking> {

    /// Converts the SPI driver into async mode.

    #[instability::unstable]

        self.spi

            .set_interrupt_handler(self.spi.info().async_handler);

            channel: self.channel.into_async(),

        }

    fn new_inner(spi: SpiWrapper<'d>, channel: SpiMasterErased<'d>) -> Self {

        let state = spi.spi.dma_state();

        state.tx_transfer_in_progress.set(false);

        state.rx_transfer_in_progress.set(false);

        // Safety: The descriptors occupy their own shared cache line and are updated in a

        // synchronised fashion.

        let (tx_descriptors, rx_descriptors) = unsafe {

            let descriptors = (&mut *state.descriptors.get()).get_mut().into_inner();

            descriptors.fill(DmaDescriptor::EMPTY);

            let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1);

            (

                DmaAlignedMut::new_unchecked(tx_descriptors),

                DmaAlignedMut::new_unchecked(rx_descriptors),

            )

        };

        let tx_buffer = cfg_select! {

            all(spi_master_version = "1", spi_address_workaround) => unsafe {

                (&mut *state.default_tx_buffer.get()).get_mut().unsize()

            },

            _ => unsafe { DmaAlignedMut::new_unchecked(&mut [][..]) },

        };

            DmaAlignedMut::new_unchecked(&mut [])

        }));

        // The buffers must be set up when creating the driver.

        unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer.into_scoped()) };

        unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer.into_scoped()) };

    }

        spi_driver: Spi<'d, Blocking>,

        channel: SpiMasterErased<'d>,

    ) -> Self {

        let spi = spi_driver.spi;

        Self::new_inner(spi, channel)

    /// Listens for the given interrupts.

    #[instability::unstable]

    pub fn listen(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {

        self.driver().enable_listen(interrupts.into(), true);

    }

    /// Unlistens from the given interrupts.

    #[instability::unstable]

    pub fn unlisten(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {

        self.driver().enable_listen(interrupts.into(), false);

    }

    /// Returns the asserted interrupts.

    #[instability::unstable]

    pub fn interrupts(&mut self) -> EnumSet<SpiInterrupt> {

        self.driver().interrupts()

    }

    /// Resets asserted interrupts.

    #[instability::unstable]

    pub fn clear_interrupts(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {

        self.driver().clear_interrupts(interrupts.into());

    }

    #[cfg_attr(

        not(multi_core),

        doc = "Registers an interrupt handler for the peripheral."

    )]

    #[cfg_attr(

        multi_core,

        doc = "Registers an interrupt handler for the peripheral on the current core."

    )]

    #[doc = ""]

    /// Replaces any previously registered interrupt handlers.

    ///

    /// The default/unhandled interrupt handler can be restored with

    /// [crate::interrupt::DEFAULT_INTERRUPT_HANDLER]

    ///

    /// # Panics

    ///

    /// Panics if passed interrupt handler is invalid (e.g. has priority

    /// `None`)

    #[instability::unstable]

    pub fn set_interrupt_handler(&mut self, handler: InterruptHandler) {

        self.spi.set_interrupt_handler(handler);

    }

}

impl<'d> SpiDma<'d, Async> {

    /// Converts the SPI instance into blocking mode.

    #[instability::unstable]

    pub fn into_blocking(self) -> SpiDma<'d, Blocking> {

        self.spi.disable_peri_interrupt_on_all_cores();

        SpiDma {

            spi: self.spi,

            channel: self.channel.into_blocking(),

        }

    }

            self.dma_driver().state.rx_transfer_in_progress.set(false);

        }

        struct Fut(Driver);

        impl Fut {

            const DONE_EVENTS: EnumSet<SpiInterrupt> =

                enumset::enum_set!(SpiInterrupt::TransferDone);

        }

        impl Future for Fut {

            type Output = ();

                    #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]

                    // Need to poll for done-ness even after interrupt fires.

                    if self.0.busy() {

                        cx.waker().wake_by_ref();

                        return Poll::Pending;

                    }

                    self.0.clear_interrupts(Self::DONE_EVENTS);

                    return Poll::Ready(());

                }

                self.0.enable_listen(Self::DONE_EVENTS, true);

                Poll::Pending

        }

        impl Drop for Fut {

            fn drop(&mut self) {

                self.0.enable_listen(Self::DONE_EVENTS, false);

        }

        }

            // In case DMA TX buffer is bigger than what the SPI consumes, stop the DMA.

            }

            self.dma_driver().state.tx_transfer_in_progress.set(false);

        }

    }

    /// Fills the given buffer with data from the bus.

    #[instability::unstable]

    pub async fn read_async(&mut self, words: &mut [u8]) -> Result<(), Error> {

        if words.is_empty() {

            return Ok(());

        }

        let _clock = SpiClockGuard::new(self.spi.info());

        self.driver().setup_full_duplex()?;

        if self.use_blocking_transfer(words.len()) {

            self.dma_driver().disable_dma();

            return self.driver().read(words);

        }

        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {

            DmaOperationKind::Copied => {

                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {

                descriptors: &mut descriptors,

                #[cfg(dma_can_access_psram)]

                align_buffer: [const { None }; 2],

            },

        };

        if maybe_copy_buffer.chunk_size() == 0 {

            return Err(Error::from(DmaError::BufferTooSmall));

        }

        for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {

            let read_bytes = chunk.len();

            let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };

            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };

            self.transfer_buffers_dma_async(read_bytes, 0, rx_buffer, tx_buffer)

                .await?;

            maybe_copy_buffer.finish(chunk);

        }

        Ok(())

    }

    /// Transmits the given buffer to the bus.

    #[instability::unstable]

    pub async fn write_async(&mut self, words: &[u8]) -> Result<(), Error> {

        if words.is_empty() {

            return Ok(());

        }

        let _clock = SpiClockGuard::new(self.spi.info());

        self.driver().setup_full_duplex()?;

        if self.use_blocking_transfer(words.len()) {

            self.dma_driver().disable_dma();

            return self.driver().write(words);

        }

        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {

            DmaOperationKind::Copied => {

                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),

        };

        if maybe_copy_buffer.chunk_size() == 0 {

            return Err(Error::from(DmaError::BufferTooSmall));

        }

        for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {

            let write_bytes = chunk.len();

            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };

            let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };

            self.transfer_buffers_dma_async(0, write_bytes, rx_buffer, tx_buffer)

                .await?;

        }

        Ok(())

    }

    /// Transfers by writing out a buffer and reading the response from

    /// the bus into another buffer.

    #[instability::unstable]

    pub async fn transfer_async(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {

        if read.is_empty() && write.is_empty() {

            return Ok(());

        }

        let _clock = SpiClockGuard::new(self.spi.info());

        self.driver().setup_full_duplex()?;

        if self.use_blocking_transfer(read.len().max(write.len())) {

            self.dma_driver().disable_dma();

            return if read.is_empty() {

                self.driver().write(write)

            } else if write.is_empty() {

                self.driver().read(read)

            } else {

                self.driver().transfer(read, write)

            };

        }

        let common_length = min(read.len(), write.len());

        let (read_common, read_remainder) = read.split_at_mut(common_length);

        let (write_common, write_remainder) = write.split_at(common_length);

        // DmaOperationKind must be determined on the sub-slices actually passed to DMA.

        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {

            DmaOperationKind::Copied => {

                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {

                descriptors: &mut rx_descriptors,

                #[cfg(dma_can_access_psram)]

                align_buffer: [const { None }; 2],

            },

        };

        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {

            DmaOperationKind::Copied => {

                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),

        };

        let chunk_size = min(

            maybe_copy_rx_buffer.chunk_size(),

            maybe_copy_tx_buffer.chunk_size(),

        );

        if chunk_size == 0 {

            return Err(Error::from(DmaError::BufferTooSmall));

        }

        for (read_chunk, write_chunk) in read_common

            .chunks_mut(chunk_size)

            .zip(write_common.chunks(chunk_size))

        {

            let read_bytes = read_chunk.len();

            let write_bytes = write_chunk.len();

            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };

            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };

            self.transfer_buffers_dma_async(read_bytes, write_bytes, rx_buffer, tx_buffer)

                .await?;

            maybe_copy_rx_buffer.finish(read_chunk);

        }

        if !read_remainder.is_empty() {

            self.read_async(read_remainder).await

        } else if !write_remainder.is_empty() {

            self.write_async(write_remainder).await

        } else {

            Ok(())

        }

    }

    /// Transfers by writing out a buffer and reading the response from

    /// the bus into the same buffer.

    #[instability::unstable]

    pub async fn transfer_in_place_async(&mut self, words: &mut [u8]) -> Result<(), Error> {

        if words.is_empty() {

            return Ok(());

        }

        let _clock = SpiClockGuard::new(self.spi.info());

        self.driver().setup_full_duplex()?;

        if self.use_blocking_transfer(words.len()) {

            self.dma_driver().disable_dma();

            return self.driver().transfer_in_place(words);

        }

        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =

            match DmaOperationKind::for_write(words) {

                DmaOperationKind::Copied => (

                    MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),

                    MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),

                ),

                DmaOperationKind::InPlace => (

                    MaybeCopyRxBuf::Direct {

                        descriptors: &mut rx_descriptors,

                        #[cfg(dma_can_access_psram)]

                        align_buffer: [const { None }; 2],

                    },

                    MaybeCopyTxBuf::Direct(&mut tx_descriptors),

                ),

            };

        let chunk_size = min(

            maybe_copy_rx_buffer.chunk_size(),

            maybe_copy_tx_buffer.chunk_size(),

        );

        if chunk_size == 0 {

            return Err(Error::from(DmaError::BufferTooSmall));

        }

        for chunk in words.chunks_mut(chunk_size) {

            let bytes = chunk.len();

            let ptr = NonNull::from(&mut *chunk);

            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };

            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };

            self.transfer_buffers_dma_async(bytes, bytes, rx_buffer, tx_buffer)

                .await?;

            maybe_copy_rx_buffer.finish(chunk);

        }

        Ok(())

    }

    /// Half-duplex read.

    ///

    /// This performs the command, address, dummy, and data phases as a single

    /// SPI transaction. Because command and address phases cannot be split

    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or

    /// in the configured internal RX copy buffer.

    #[instability::unstable]

    pub async fn half_duplex_read_async(

            return Ok(());

        }

        // Transfers below the configured threshold can skip the DMA setup cost entirely.

            return self

        }

            DmaOperationKind::Copied => {

                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {

                descriptors: &mut descriptors,

                #[cfg(dma_can_access_psram)]

                align_buffer: [const { None }; 2],

            },

        };

        let chunk_size = maybe_copy_buffer.chunk_size();

            return Err(Error::from(DmaError::BufferTooSmall));

        }

                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),

                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),

            };

        }

        Ok(())

    }

    /// Half-duplex write.

    ///

    /// This performs the command, address, dummy, and data phases as a single

    /// SPI transaction. Because command and address phases cannot be split

    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or

    /// in the configured internal TX copy buffer.

    #[instability::unstable]

    pub async fn half_duplex_write_async(

        &mut self,

        data_mode: DataMode,

        cmd: Command,

        address: Address,

        dummy: u8,

        buffer: &[u8],

    ) -> Result<(), Error> {

        let _clock = SpiClockGuard::new(self.spi.info());

        if buffer.is_empty() {

            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };

            self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, 0, tx_buffer)

                .await?;

            return Ok(());

        }

        // Transfers below the configured threshold can skip the DMA setup cost entirely.

        if self.use_blocking_transfer(buffer.len()) {

            self.dma_driver().disable_dma();

            return self

                .driver()

                .half_duplex_write(data_mode, cmd, address, dummy, buffer);

        }

        let operation = DmaOperationKind::for_write(buffer);

        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];

        let mut maybe_copy_buffer = match operation {

            DmaOperationKind::Copied => {

                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })

            }

            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),

        };

        let chunk_size = maybe_copy_buffer.chunk_size();

        if chunk_size == 0 {

            return Err(Error::from(DmaError::BufferTooSmall));

        }

        if buffer.len() > chunk_size {

            return match operation {

                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),

                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),

            };

        }

        let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(buffer)) };

        self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)

            .await

    }

    async fn transfer_buffers_dma_async(

        &mut self,

        read_bytes: usize,

        write_bytes: usize,

        mut rx_buffer: impl DmaRxBuffer,

        mut tx_buffer: impl DmaTxBuffer,

    ) -> Result<(), Error> {

        let _clock = SpiClockGuard::new(self.spi.info());

        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());

        unsafe {

            spi.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;

        }

        spi.wait_for_idle_async().await;

        spi.defuse();

        Ok(())

    }

    async fn half_duplex_read_dma_async(

        unsafe {

            spi.start_half_duplex_read(

                data_mode,

                cmd,

                address,

                dummy,

                bytes_to_read,

                &mut rx_buffer,

            )?;

        }

        spi.defuse();

        Ok(())

    async fn half_duplex_write_dma_async(

        &mut self,

        data_mode: DataMode,

        cmd: Command,

        address: Address,

        dummy: u8,

        bytes_to_write: usize,

        mut tx_buffer: impl DmaTxBuffer,

    ) -> Result<(), Error> {

        let _clock = SpiClockGuard::new(self.spi.info());

        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());

        unsafe {

            spi.start_half_duplex_write(

                data_mode,

                cmd,

                address,

                dummy,

                bytes_to_write,

                &mut tx_buffer,

            )?;

        }

        spi.wait_for_idle_async().await;

        spi.defuse();

        Ok(())

    }

}

// +1 to make sure we have enough descriptors to satisfy strict alignment requirements

const LINK_DESCRIPTOR_COUNT: usize = MAX_DMA_SIZE.div_ceil(CHUNK_SIZE) + 2 + 1;

enum MaybeCopyTxBuf<'a> {

    Copy(&'a mut ScopedDmaTxBuf<'static>),

    Direct(&'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT]),

}

impl<'a> MaybeCopyTxBuf<'a> {

            MaybeCopyTxBuf::Copy(tx_buffer) => {

                tx_buffer.as_mut_slice()[..data.len()].copy_from_slice(unsafe { data.as_ref() });

            }

            MaybeCopyTxBuf::Direct(descriptors) => {

                buffer

            }

        }

    fn chunk_size(&self) -> usize {

            MaybeCopyTxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),

            MaybeCopyTxBuf::Direct(_) => MAX_DMA_SIZE,

        }

    }

}

#[allow(clippy::large_enum_variant)]

enum MaybeCopyRxBuf<'a> {

    Copy(&'a mut ScopedDmaRxBuf<'static>),

    Direct {

        descriptors: &'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT],

        #[cfg(dma_can_access_psram)]

        align_buffer: [Option<ManualWritebackBuffer>; 2],

    },

}

impl<'a> MaybeCopyRxBuf<'a> {

            MaybeCopyRxBuf::Direct {

                descriptors,

                #[cfg(dma_can_access_psram)]

                align_buffer,

            } => {

                        #[cfg(dma_can_access_psram)]

                        align_buffer,

                        data,

                    )

                };

                buffer

            }

        }

    fn chunk_size(&self) -> usize {

            MaybeCopyRxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),

            MaybeCopyRxBuf::Direct { .. } => MAX_DMA_SIZE,

        }

    }

            MaybeCopyRxBuf::Copy(buffer) => {

            }

            MaybeCopyRxBuf::Direct {

                #[cfg(dma_can_access_psram)]

                align_buffer,

                ..

            } => {

                #[cfg(soc_internal_memory_cached)]

                unsafe {

                    crate::soc::cache_invalidate_addr(chunk.as_ptr() as u32, chunk.len() as u32);

                }

                #[cfg(dma_can_access_psram)]

                for buffer in align_buffer.iter_mut() {

                    if let Some(buffer) = buffer.as_mut() {

                        buffer.write_back();

                    }

                    *buffer = None;

                }

            }

        }

}

#[derive(Clone, Copy)]

enum DmaOperationKind {

    /// The entire slice must be copied into the internal buffer first.

    Copied,

    /// The slice can be transferred directly, with minimal copying done for alignment.

    InPlace,

}

impl DmaOperationKind {

    fn compute(buffer: &[u8], direction: TransferDirection) -> Self {

        fn is_dma_compatible(buffer: &[u8], _direction: TransferDirection) -> bool {

            // FIXME: lazy workaround for ESP32 TX DMA alignment requirements.

            // `prepare_for_tx` and `prepare_for_rx` should be updated to handle ESP32.

            #[cfg(spi_master_version = "1")]

            if !((buffer.as_ptr() as usize).is_multiple_of(4) && buffer.len().is_multiple_of(4)) {

                return false;

            }

            if is_slice_in_dram(buffer) {

                return true;

            }

            #[cfg(dma_can_access_psram)]

            if is_slice_in_psram(buffer) {

                #[cfg(spi_master_version = "2")]

                if _direction == TransferDirection::In {

                    // For some reason, having tail bytes in internal RAM causes issues, so we

                    // force copying if the end of the PSRAM buffer is not aligned.

                    let tail_bytes = (buffer.as_ptr() as usize + buffer.len()).wrapping_neg() & 15;

                    if tail_bytes > 0 {

                        return false;

                    }

                }

                return true;

            }

            // TODO: C5+ DMA can read from flash

            false

        }

        if is_dma_compatible(buffer, direction) {

            Self::InPlace

        } else {

            Self::Copied

        }

    }

    fn for_read(buffer: &mut [u8]) -> Self {

        Self::compute(buffer, TransferDirection::In)

    fn for_write(buffer: &[u8]) -> Self {

        Self::compute(buffer, TransferDirection::Out)

}

impl<'d, Dm> SpiDma<'d, Dm>

where

    Dm: DriverMode,

{

    fn use_blocking_transfer(&self, transfer_size: usize) -> bool {

        let threshold = self

            .spi

            .state()

            .min_async_transfer_size

            .load(Ordering::Relaxed);

        threshold > 0 && transfer_size < threshold

    }

    fn spi(&self) -> &SpiWrapper<'_> {

        &self.spi

    }

    fn driver(&self) -> Driver {

            info: self.spi.info(),

            state: self.spi.state(),

        }

    }

    fn dma_driver(&self) -> DmaDriver {

            driver: self.driver(),

            state: self.spi().dma_state(),

            dma_peripheral: self.spi.spi.dma_peripheral(),

        }

    }

    fn is_done(&self) -> bool {

            return false;

        }

            // If this is an asymmetric transfer and the RX side is smaller, the RX channel

            // will never be "done" as it won't have enough descriptors/buffer to receive

            // the EOF bit from the SPI. So instead the RX channel will hit

            // a "descriptor empty" which means the DMA is written as much

            // of the received data as possible into the buffer and

            // discarded the rest. The user doesn't care about this discarded data.

                return false;

            }

        }

        true

    }

    fn wait_for_idle(&mut self) {

            // Wait for the SPI to become idle

        }

        self.dma_driver().state.rx_transfer_in_progress.set(false);

        self.dma_driver().state.tx_transfer_in_progress.set(false);

        fence(Ordering::Acquire);

    }

    /// # Safety

    ///

    /// The caller must ensure to not access the buffer contents while the

    /// transfer is in progress. Moving the buffer itself is allowed.

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

        &mut self,

        full_duplex: bool,

        bytes_to_read: usize,

        bytes_to_write: usize,

        rx_buffer: &mut RX,

        tx_buffer: &mut TX,

    ) -> Result<(), Error> {

            return Err(Error::MaxDmaTransferSizeExceeded);

        }

        self.dma_driver()

            .state

            .rx_transfer_in_progress

        self.dma_driver()

            .state

            .tx_transfer_in_progress

        unsafe {

            self.dma_driver().start_transfer_dma(

                full_duplex,

                bytes_to_read,

                bytes_to_write,

                rx_buffer,

                tx_buffer,

                &mut self.channel,

            )

        }

    /// # Safety

    ///

    /// The caller must ensure that the buffers are not accessed while the

    /// transfer is in progress. Moving the buffers is allowed.

    #[cfg(all(spi_master_version = "1", spi_address_workaround))]

    unsafe fn set_up_address_workaround(

        &mut self,

        cmd: Command,

        address: Address,

        dummy: u8,

    ) -> Result<(), Error> {

        if dummy > 0 {

            // FIXME: https://github.com/esp-rs/esp-hal/issues/2240

            error!("Dummy bits are not supported when there is no data to write");

            return Err(Error::Unsupported);

        }

        let buffer = unsafe { self.dma_driver().tx_buffer() };

        let bytes_to_write = address.width().div_ceil(8);

        // The address register is read in big-endian order,

        // we have to prepare the emulated write in the same way.

        let addr_bytes = address.value().to_be_bytes();

        let addr_bytes = &addr_bytes[4 - bytes_to_write..][..bytes_to_write];

        buffer.fill(addr_bytes);

        self.driver().setup_half_duplex(

            true,

            cmd,

            Address::None,

            false,

            dummy,

            bytes_to_write == 0,

            address.mode(),

        )?;

        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };

        unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }

    }

    fn cancel_transfer(&mut self) {

        let state = self.dma_driver().state;

            // We need to stop the DMA transfer, too.

                self.channel.tx.stop_transfer();

                state.tx_transfer_in_progress.set(false);

            }

                self.channel.rx.stop_transfer();

                state.rx_transfer_in_progress.set(false);

            }

        }

    }

    /// # Safety

    ///

    /// The caller must ensure that the buffers are not accessed while the

    /// transfer is in progress. Moving the buffers is allowed.

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    unsafe fn start_dma_write(

        &mut self,

        bytes_to_write: usize,

        buffer: &mut impl DmaTxBuffer,

    ) -> Result<(), Error> {

        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };

        unsafe { self.start_dma_transfer(0, bytes_to_write, rx_buffer, buffer) }

    }

    /// Assigns copy buffers to the SPI driver.

    ///

    /// These buffers will be used to copy data when using the slice-based transfer functions.

    ///

    /// Data is copied in two cases:

    ///   - When the buffer is not located in a memory region that can be accessed by the DMA.

    #[cfg_attr(

        not(spi_master_dma_can_access_flash),

        doc = "The DMA cannot read flash memory."

    )]

    ///   - When the alignment of the buffer does not meet the DMA's requirements, the unaligned

    ///     parts of the buffer are copied.

    #[cfg_attr(

        spi_master_version = "1",

        doc = "On ESP32, transferring from internal SRAM requires copying the entire buffer if it is

not 4-byte aligned. This is a limitation of the current implementation."

    )]

    #[cfg_attr(

        spi_master_version = "2",

        doc = "On ESP32-S2, receiving into PSRAM requires the buffer's _end_ to be 16-byte

aligned, otherwise the driver requires copying the entire buffer."

    )]

    #[doc = ""]

    /// The maximum useful size for these buffers is 32736 bytes, any additional memory will

    /// be wasted.

    ///

    /// For an example of how to create these buffers, see the [`SpiDma`] documentation.

    #[instability::unstable]

    pub fn with_buffers(self, dma_rx_buf: DmaRxBuf, dma_tx_buf: DmaTxBuf) -> SpiDma<'d, Dm> {

        unsafe {

            (&mut *self.spi.dma_state().rx_buffer.get()).write(dma_rx_buf.into_scoped());

            (&mut *self.spi.dma_state().tx_buffer.get()).write(dma_tx_buf.into_scoped());

        }

    }

    /// Performs a DMA write.

    ///

    /// Returns a [`SpiDmaTransfer`] that owns the buffer and the

    /// SPI instance. The maximum amount of data to be sent is 32736

    /// bytes.

    #[allow(clippy::type_complexity)]

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    #[instability::unstable]

        mut self,

        bytes_to_write: usize,

        mut buffer: TX,

    ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {

            return Err((e, self, buffer));

        };

            Err(e) => Err((e, self, buffer)),

        }

    /// # Safety

    ///

    /// The caller must ensure that the buffers are not accessed while the

    /// transfer is in progress. Moving the buffers is allowed.

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    unsafe fn start_dma_read(

        &mut self,

        bytes_to_read: usize,

        buffer: &mut impl DmaRxBuffer,

    ) -> Result<(), Error> {

        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };

        unsafe { self.start_dma_transfer(bytes_to_read, 0, buffer, tx_buffer) }

    }

    /// Performs a DMA read.

    ///

    /// Returns a [`SpiDmaTransfer`] that owns the buffer and

    /// the SPI instance. The maximum amount of data to be

    /// received is 32736 bytes.

    #[allow(clippy::type_complexity)]

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    #[instability::unstable]

    pub fn read_buffer<RX: DmaRxBuffer>(

        mut self,

        bytes_to_read: usize,

        mut buffer: RX,

    ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {

        };

        }

    }

    /// # Safety

    ///

    /// The caller must ensure that the buffers are not accessed while the

    /// transfer is in progress. Moving the buffers is allowed.

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    unsafe fn start_dma_transfer(

        &mut self,

        bytes_to_read: usize,

        bytes_to_write: usize,

        rx_buffer: &mut impl DmaRxBuffer,

        tx_buffer: &mut impl DmaTxBuffer,

    ) -> Result<(), Error> {

        unsafe {

        }

    }

    /// Performs a DMA transfer.

    ///

    /// Returns a [`SpiDmaTransfer`] that owns the buffers and

    /// the SPI instance. The maximum amount of data to be

    /// sent/received is 32736 bytes.

    #[allow(clippy::type_complexity)]

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    #[instability::unstable]

        mut self,

        bytes_to_read: usize,

        mut rx_buffer: RX,

        bytes_to_write: usize,

        mut tx_buffer: TX,

    ) -> Result<SpiDmaTransfer<'d, Dm, (RX, TX)>, (Error, Self, RX, TX)> {

            return Err((e, self, rx_buffer, tx_buffer));

        };

        match unsafe {

                bytes_to_read,

                bytes_to_write,

                &mut rx_buffer,

                &mut tx_buffer,

            )

        } {

            Err(e) => Err((e, self, rx_buffer, tx_buffer)),

        }

    /// # Safety

    ///

    /// The caller must ensure that the buffers are not accessed while the

    /// transfer is in progress. Moving the buffers is allowed.

    #[cfg_attr(place_spi_master_driver_in_ram, ram)]

    unsafe fn start_half_duplex_read(

        &mut self,

        data_mode: DataMode,

        cmd: Command,

        address: Address,

        dummy: u8,

        bytes_to_read: usize,

        buffer: &mut impl DmaRxBuffer,

    ) -> Result<(), Error> {

            false,

            cmd,

            address,

            false,

            dummy,

            data_mode,

        )?;

        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };