diff --git a/esp-hal/CHANGELOG.md b/esp-hal/CHANGELOG.md index ec6cd9ff2cb..0ccd9b5a2c5 100644 --- a/esp-hal/CHANGELOG.md +++ b/esp-hal/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The clock frequency accessor functions no longer need to lock the clock tree (#5461) - SPI: `SpiDmaBus` has been merged into `SpiDma`. `with_buffers` now returns `SpiDma` directly, and the buffer-taking transfer methods have been renamed to `read_buffer`, `write_buffer`, `transfer_buffers`, `half_duplex_read_buffer` and `half_duplex_write_buffer` to avoid conflicts with the `SpiBus` trait methods. (#5272) +- SPI: `SpiDma` will now skip copying into the internal buffers unless necessary (#5290) ### Fixed diff --git a/esp-hal/src/dma/buffers.rs b/esp-hal/src/dma/buffers/mod.rs similarity index 89% rename from esp-hal/src/dma/buffers.rs rename to esp-hal/src/dma/buffers/mod.rs index 6bb55e75620..856f8e6e3bb 100644 --- a/esp-hal/src/dma/buffers.rs +++ b/esp-hal/src/dma/buffers/mod.rs @@ -10,6 +10,9 @@ use crate::soc::is_slice_in_dram; #[cfg(dma_can_access_psram)] use crate::soc::{is_slice_in_psram, is_valid_psram_address, is_valid_ram_address}; +pub(crate) mod scoped; +pub(crate) use scoped::*; + /// Error returned from Dma[Rx|Tx|RxTx]Buf operations. #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] @@ -495,11 +498,7 @@ pub struct BufView(T); /// FIFO. See [DmaRxBuf] for receiving data. #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] -pub struct DmaTxBuf { - descriptors: DescriptorSet<'static>, - buffer: &'static mut [u8], - burst: BurstConfig, -} +pub struct DmaTxBuf(ScopedDmaTxBuf<'static>); impl DmaTxBuf { /// Creates a new [DmaTxBuf] from some descriptors and a buffer. @@ -514,7 +513,7 @@ impl DmaTxBuf { descriptors: &'static mut [DmaDescriptor], buffer: &'static mut [u8], ) -> Result { - Self::new_with_config(descriptors, buffer, BurstConfig::default()) + ScopedDmaTxBuf::new_with_config(descriptors, buffer, BurstConfig::default()).map(Self) } /// Creates a new [DmaTxBuf] from some descriptors and a buffer. @@ -530,80 +529,28 @@ impl DmaTxBuf { buffer: &'static mut [u8], config: impl Into, ) -> Result { - let mut buf = Self { - descriptors: DescriptorSet::new(descriptors)?, - buffer, - burst: BurstConfig::default(), - }; - - let capacity = buf.capacity(); - buf.configure(config, capacity)?; - - Ok(buf) - } - - fn configure( - &mut self, - burst: impl Into, - length: usize, - ) -> Result<(), DmaBufError> { - let burst = burst.into(); - self.set_length_fallible(length, burst)?; - - self.descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; - - self.burst = burst; - Ok(()) + ScopedDmaTxBuf::new_with_config(descriptors, buffer, config).map(Self) } /// Configures the DMA to use burst transfers to access this buffer. pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> { - let len = self.len(); - self.configure(burst, len) + self.0.set_burst_config(burst) } /// Consume the buf, returning the descriptors and buffer. pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { - (self.descriptors.into_inner(), self.buffer) + self.0.split() } /// Returns the size of the underlying buffer pub fn capacity(&self) -> usize { - self.buffer.len() + self.0.capacity() } /// Return the number of bytes that would be transmitted by this buf. #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { - self.descriptors - .linked_iter() - .map(|d| d.len()) - .sum::() - } - - fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> { - if len > self.capacity() { - return Err(DmaBufError::BufferTooSmall); - } - burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::Out)?; - - self.descriptors.set_tx_length( - len, - burst.max_chunk_size_for(self.buffer, TransferDirection::Out), - )?; - - // This only needs to be done once (after every significant length change) as - // Self::prepare sets Preparation::auto_write_back to false. - for desc in self.descriptors.linked_iter_mut() { - // In non-circular mode, we only set `suc_eof` for the last descriptor to signal - // the end of the transfer. - desc.reset_for_tx(desc.next.is_null()); - } - - Ok(()) + self.0.len() } /// Reset the descriptors to only transmit `len` amount of bytes from this @@ -612,7 +559,7 @@ impl DmaTxBuf { /// The number of bytes in data must be less than or equal to the buffer /// size. pub fn set_length(&mut self, len: usize) { - unwrap!(self.set_length_fallible(len, self.burst)) + self.0.set_length(len); } /// Fills the TX buffer with the bytes provided in `data` and reset the @@ -621,18 +568,22 @@ impl DmaTxBuf { /// The number of bytes in data must be less than or equal to the buffer /// size. pub fn fill(&mut self, data: &[u8]) { - self.set_length(data.len()); - self.as_mut_slice()[..data.len()].copy_from_slice(data); + self.0.fill(data); } /// Returns the buf as a mutable slice than can be written. pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.buffer + self.0.as_mut_slice() } /// Returns the buf as a slice than can be read. pub fn as_slice(&self) -> &[u8] { - self.buffer + self.0.as_slice() + } + + /// Consumes the buffer and returns the scoped version. + pub(crate) fn into_scoped(self) -> ScopedDmaTxBuf<'static> { + self.0 } } @@ -641,29 +592,7 @@ unsafe impl DmaTxBuffer for DmaTxBuf { type Final = DmaTxBuf; fn prepare(&mut self) -> Preparation { - cfg_if::cfg_if! { - if #[cfg(dma_can_access_psram)] { - let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); - if is_data_in_psram { - unsafe { - crate::soc::cache_writeback_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } - } - - Preparation { - start: self.descriptors.head(), - direction: TransferDirection::Out, - #[cfg(dma_can_access_psram)] - accesses_psram: is_data_in_psram, - burst_transfer: self.burst, - check_owner: None, - auto_write_back: false, - } + self.0.prepare() } fn into_view(self) -> BufView { @@ -682,11 +611,7 @@ unsafe impl DmaTxBuffer for DmaTxBuf { /// See [DmaTxBuf] for transmitting data. #[derive(Debug)] #[cfg_attr(feature = "defmt", derive(defmt::Format))] -pub struct DmaRxBuf { - descriptors: DescriptorSet<'static>, - buffer: &'static mut [u8], - burst: BurstConfig, -} +pub struct DmaRxBuf(ScopedDmaRxBuf<'static>); impl DmaRxBuf { /// Creates a new [DmaRxBuf] from some descriptors and a buffer. @@ -700,7 +625,7 @@ impl DmaRxBuf { descriptors: &'static mut [DmaDescriptor], buffer: &'static mut [u8], ) -> Result { - Self::new_with_config(descriptors, buffer, BurstConfig::default()) + ScopedDmaRxBuf::new_with_config(descriptors, buffer, BurstConfig::default()).map(Self) } /// Creates a new [DmaRxBuf] from some descriptors and a buffer. @@ -716,70 +641,29 @@ impl DmaRxBuf { buffer: &'static mut [u8], config: impl Into, ) -> Result { - let mut buf = Self { - descriptors: DescriptorSet::new(descriptors)?, - buffer, - burst: BurstConfig::default(), - }; - - buf.configure(config, buf.capacity())?; - - Ok(buf) - } - - fn configure( - &mut self, - burst: impl Into, - length: usize, - ) -> Result<(), DmaBufError> { - let burst = burst.into(); - self.set_length_fallible(length, burst)?; - - self.descriptors.link_with_buffer( - self.buffer, - burst.max_chunk_size_for(self.buffer, TransferDirection::In), - )?; - - self.burst = burst; - Ok(()) + ScopedDmaRxBuf::new_with_config(descriptors, buffer, config).map(Self) } /// Configures the DMA to use burst transfers to access this buffer. pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> { - let len = self.len(); - self.configure(burst, len) + self.0.set_burst_config(burst) } /// Consume the buf, returning the descriptors and buffer. pub fn split(self) -> (&'static mut [DmaDescriptor], &'static mut [u8]) { - (self.descriptors.into_inner(), self.buffer) + self.0.split() } /// Returns the size of the underlying buffer pub fn capacity(&self) -> usize { - self.buffer.len() + self.0.capacity() } /// Returns the maximum number of bytes that this buf has been configured to /// receive. #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { - self.descriptors - .linked_iter() - .map(|d| d.size()) - .sum::() - } - - fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> { - if len > self.capacity() { - return Err(DmaBufError::BufferTooSmall); - } - burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::In)?; - - self.descriptors.set_rx_length( - len, - burst.max_chunk_size_for(&self.buffer[..len], TransferDirection::In), - ) + self.0.len() } /// Reset the descriptors to only receive `len` amount of bytes into this @@ -788,25 +672,22 @@ impl DmaRxBuf { /// The number of bytes in data must be less than or equal to the buffer /// size. pub fn set_length(&mut self, len: usize) { - unwrap!(self.set_length_fallible(len, self.burst)); + self.0.set_length(len) } /// Returns the entire underlying buffer as a slice than can be read. pub fn as_slice(&self) -> &[u8] { - self.buffer + self.0.as_slice() } /// Returns the entire underlying buffer as a slice than can be written. pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.buffer + self.0.as_mut_slice() } /// Return the number of bytes that was received by this buf. pub fn number_of_received_bytes(&self) -> usize { - self.descriptors - .linked_iter() - .map(|d| d.len()) - .sum::() + self.0.number_of_received_bytes() } /// Reads the received data into the provided `buf`. @@ -815,31 +696,18 @@ impl DmaRxBuf { /// first `buf.len()` bytes of received data is written into `buf`. /// /// Returns the number of bytes in written to `buf`. - pub fn read_received_data(&self, mut buf: &mut [u8]) -> usize { - // Note that due to an ESP32 quirk, the last received descriptor may not get - // updated. - let capacity = buf.len(); - for chunk in self.received_data() { - if buf.is_empty() { - break; - } - let to_fill; - (to_fill, buf) = buf.split_at_mut(chunk.len()); - to_fill.copy_from_slice(chunk); - } - - capacity - buf.len() + pub fn read_received_data(&self, buf: &mut [u8]) -> usize { + self.0.read_received_data(buf) } /// Returns the received data as an iterator of slices. pub fn received_data(&self) -> impl Iterator { - self.descriptors.linked_iter().map(|desc| { - // SAFETY: We set up the descriptor to point to a subslice of the buffer, and - // here we are only recreating that slice with a perhaps shorter length. - // We are also not accessing `self.buffer` while this slice is alive, so we - // are not violating any aliasing rules. - unsafe { core::slice::from_raw_parts(desc.buffer.cast_const(), desc.len()) } - }) + self.0.received_data() + } + + /// Consumes the buffer and returns the scoped version. + pub(crate) fn into_scoped(self) -> ScopedDmaRxBuf<'static> { + self.0 } } @@ -848,34 +716,7 @@ unsafe impl DmaRxBuffer for DmaRxBuf { type Final = DmaRxBuf; fn prepare(&mut self) -> Preparation { - for desc in self.descriptors.linked_iter_mut() { - desc.reset_for_rx(); - } - - cfg_if::cfg_if! { - if #[cfg(dma_can_access_psram)] { - // Optimization: avoid locking for PSRAM range. - let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); - if is_data_in_psram { - unsafe { - crate::soc::cache_invalidate_addr( - self.buffer.as_ptr() as u32, - self.buffer.len() as u32, - ) - }; - } - } - } - - Preparation { - start: self.descriptors.head(), - direction: TransferDirection::In, - #[cfg(dma_can_access_psram)] - accesses_psram: is_data_in_psram, - burst_transfer: self.burst, - check_owner: None, - auto_write_back: true, - } + self.0.prepare() } fn into_view(self) -> BufView { @@ -1591,7 +1432,7 @@ impl DerefMut for DmaLoopBuf { /// Fow low level use, where none of the pre-made buffers really fit. /// /// This type likely never should be visible outside of esp-hal. -pub(crate) struct NoBuffer(Preparation); +pub(crate) struct NoBuffer(pub(crate) Preparation); impl NoBuffer { fn prep(&self) -> Preparation { Preparation { @@ -1640,7 +1481,7 @@ unsafe impl DmaRxBuffer for NoBuffer { /// /// The caller must keep all its descriptors and the buffers they /// point to valid while the buffer is being transferred. -#[cfg_attr(not(aes_dma), expect(unused))] +#[cfg_attr(not(any(aes_dma, spi_master_supports_dma)), expect(unused))] pub(crate) unsafe fn prepare_for_tx( descriptors: &mut [DmaDescriptor], mut data: NonNull<[u8]>, @@ -1692,7 +1533,7 @@ pub(crate) unsafe fn prepare_for_tx( direction: TransferDirection::Out, burst_transfer: BurstConfig::DEFAULT, check_owner: None, - auto_write_back: true, + auto_write_back: false, #[cfg(dma_can_access_psram)] accesses_psram: data_in_psram, }), @@ -1708,7 +1549,7 @@ pub(crate) unsafe fn prepare_for_tx( /// /// The caller must keep all its descriptors and the buffers they /// point to valid while the buffer is being transferred. -#[cfg_attr(not(aes_dma), expect(unused))] +#[cfg_attr(not(any(aes_dma, spi_master_supports_dma)), expect(unused))] pub(crate) unsafe fn prepare_for_rx( descriptors: &mut [DmaDescriptor], #[cfg(dma_can_access_psram)] align_buffers: &mut [Option; 2], diff --git a/esp-hal/src/dma/buffers/scoped.rs b/esp-hal/src/dma/buffers/scoped.rs new file mode 100644 index 00000000000..ddeae0d3e15 --- /dev/null +++ b/esp-hal/src/dma/buffers/scoped.rs @@ -0,0 +1,371 @@ +use super::*; + +/// DMA transmit buffer +/// +/// This is a contiguous buffer linked together by DMA descriptors of length +/// 4095 at most. It can only be used for transmitting data to a peripheral's +/// FIFO. See [DmaRxBuf] for receiving data. +#[derive(Debug)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] +pub(crate) struct ScopedDmaTxBuf<'a> { + descriptors: DescriptorSet<'a>, + buffer: &'a mut [u8], + burst: BurstConfig, +} + +impl<'a> ScopedDmaTxBuf<'a> { + /// Creates a new [ScopedDmaTxBuf] from some descriptors and a buffer. + /// + /// There must be enough descriptors for the provided buffer. + /// Depending on alignment requirements, each descriptor can handle at most + /// 4095 bytes worth of buffer. + /// + /// Both the descriptors and buffer must be in DMA-capable memory. + /// Only DRAM is supported for descriptors. + pub fn new_with_config( + descriptors: &'a mut [DmaDescriptor], + buffer: &'a mut [u8], + config: impl Into, + ) -> Result { + let mut buf = Self { + descriptors: DescriptorSet::new(descriptors)?, + buffer, + burst: BurstConfig::default(), + }; + + let capacity = buf.capacity(); + buf.configure(config, capacity)?; + + Ok(buf) + } + + fn configure( + &mut self, + burst: impl Into, + length: usize, + ) -> Result<(), DmaBufError> { + let burst = burst.into(); + self.set_length_fallible(length, burst)?; + + self.descriptors.link_with_buffer( + self.buffer, + burst.max_chunk_size_for(self.buffer, TransferDirection::Out), + )?; + + self.burst = burst; + Ok(()) + } + + /// Configures the DMA to use burst transfers to access this buffer. + pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> { + let len = self.len(); + self.configure(burst, len) + } + + /// Consume the buf, returning the descriptors and buffer. + pub fn split(self) -> (&'a mut [DmaDescriptor], &'a mut [u8]) { + (self.descriptors.into_inner(), self.buffer) + } + + /// Returns the size of the underlying buffer + pub fn capacity(&self) -> usize { + self.buffer.len() + } + + /// Return the number of bytes that would be transmitted by this buf. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.descriptors + .linked_iter() + .map(|d| d.len()) + .sum::() + } + + fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> { + if len > self.capacity() { + return Err(DmaBufError::BufferTooSmall); + } + burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::Out)?; + + self.descriptors.set_tx_length( + len, + burst.max_chunk_size_for(self.buffer, TransferDirection::Out), + )?; + + // This only needs to be done once (after every significant length change) as + // Self::prepare sets Preparation::auto_write_back to false. + for desc in self.descriptors.linked_iter_mut() { + // In non-circular mode, we only set `suc_eof` for the last descriptor to signal + // the end of the transfer. + desc.reset_for_tx(desc.next.is_null()); + } + + Ok(()) + } + + /// Reset the descriptors to only transmit `len` amount of bytes from this + /// buf. + /// + /// The number of bytes in data must be less than or equal to the buffer + /// size. + pub fn set_length(&mut self, len: usize) { + unwrap!(self.set_length_fallible(len, self.burst)) + } + + /// Fills the TX buffer with the bytes provided in `data` and reset the + /// descriptors to only cover the filled section. + /// + /// The number of bytes in data must be less than or equal to the buffer + /// size. + pub fn fill(&mut self, data: &[u8]) { + self.set_length(data.len()); + self.as_mut_slice()[..data.len()].copy_from_slice(data); + } + + /// Returns the buf as a mutable slice than can be written. + pub fn as_mut_slice(&mut self) -> &mut [u8] { + self.buffer + } + + /// Returns the buf as a slice than can be read. + pub fn as_slice(&self) -> &[u8] { + self.buffer + } +} + +unsafe impl<'a> DmaTxBuffer for ScopedDmaTxBuf<'a> { + type View = BufView>; + type Final = ScopedDmaTxBuf<'a>; + + fn prepare(&mut self) -> Preparation { + cfg_if::cfg_if! { + if #[cfg(dma_can_access_psram)] { + let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); + if is_data_in_psram { + unsafe { + crate::soc::cache_writeback_addr( + self.buffer.as_ptr() as u32, + self.buffer.len() as u32, + ) + }; + } + } + } + + Preparation { + start: self.descriptors.head(), + direction: TransferDirection::Out, + #[cfg(dma_can_access_psram)] + accesses_psram: is_data_in_psram, + burst_transfer: self.burst, + check_owner: None, + auto_write_back: false, + } + } + + fn into_view(self) -> BufView> { + BufView(self) + } + + fn from_view(view: Self::View) -> Self { + view.0 + } +} + +/// DMA receive buffer +/// +/// This is a contiguous buffer linked together by DMA descriptors of length +/// 4092. It can only be used for receiving data from a peripheral's FIFO. +/// See [ScopedDmaTxBuf] for transmitting data. +#[derive(Debug)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] +pub(crate) struct ScopedDmaRxBuf<'a> { + descriptors: DescriptorSet<'a>, + buffer: &'a mut [u8], + burst: BurstConfig, +} + +impl<'a> ScopedDmaRxBuf<'a> { + /// Creates a new [ScopedDmaRxBuf] from some descriptors and a buffer. + /// + /// There must be enough descriptors for the provided buffer. + /// Depending on alignment requirements, each descriptor can handle at most + /// 4092 bytes worth of buffer. + /// + /// Both the descriptors and buffer must be in DMA-capable memory. + /// Only DRAM is supported for descriptors. + pub fn new_with_config( + descriptors: &'a mut [DmaDescriptor], + buffer: &'a mut [u8], + config: impl Into, + ) -> Result { + let mut buf = Self { + descriptors: DescriptorSet::new(descriptors)?, + buffer, + burst: BurstConfig::default(), + }; + + buf.configure(config, buf.capacity())?; + + Ok(buf) + } + + fn configure( + &mut self, + burst: impl Into, + length: usize, + ) -> Result<(), DmaBufError> { + let burst = burst.into(); + self.set_length_fallible(length, burst)?; + + self.descriptors.link_with_buffer( + self.buffer, + burst.max_chunk_size_for(self.buffer, TransferDirection::In), + )?; + + self.burst = burst; + Ok(()) + } + + /// Configures the DMA to use burst transfers to access this buffer. + pub fn set_burst_config(&mut self, burst: BurstConfig) -> Result<(), DmaBufError> { + let len = self.len(); + self.configure(burst, len) + } + + /// Consume the buf, returning the descriptors and buffer. + pub fn split(self) -> (&'a mut [DmaDescriptor], &'a mut [u8]) { + (self.descriptors.into_inner(), self.buffer) + } + + /// Returns the size of the underlying buffer + pub fn capacity(&self) -> usize { + self.buffer.len() + } + + /// Returns the maximum number of bytes that this buf has been configured to + /// receive. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.descriptors + .linked_iter() + .map(|d| d.size()) + .sum::() + } + + fn set_length_fallible(&mut self, len: usize, burst: BurstConfig) -> Result<(), DmaBufError> { + if len > self.capacity() { + return Err(DmaBufError::BufferTooSmall); + } + burst.ensure_buffer_compatible(&self.buffer[..len], TransferDirection::In)?; + + self.descriptors.set_rx_length( + len, + burst.max_chunk_size_for(&self.buffer[..len], TransferDirection::In), + ) + } + + /// Reset the descriptors to only receive `len` amount of bytes into this + /// buf. + /// + /// The number of bytes in data must be less than or equal to the buffer + /// size. + pub fn set_length(&mut self, len: usize) { + unwrap!(self.set_length_fallible(len, self.burst)); + } + + /// Returns the entire underlying buffer as a slice than can be read. + pub fn as_slice(&self) -> &[u8] { + self.buffer + } + + /// Returns the entire underlying buffer as a slice than can be written. + pub fn as_mut_slice(&mut self) -> &mut [u8] { + self.buffer + } + + /// Return the number of bytes that was received by this buf. + pub fn number_of_received_bytes(&self) -> usize { + self.descriptors + .linked_iter() + .map(|d| d.len()) + .sum::() + } + + /// Reads the received data into the provided `buf`. + /// + /// If `buf.len()` is less than the amount of received data then only the + /// first `buf.len()` bytes of received data is written into `buf`. + /// + /// Returns the number of bytes in written to `buf`. + pub fn read_received_data(&self, mut buf: &mut [u8]) -> usize { + // Note that due to an ESP32 quirk, the last received descriptor may not get + // updated. + let capacity = buf.len(); + for chunk in self.received_data() { + if buf.is_empty() { + break; + } + let to_fill; + (to_fill, buf) = buf.split_at_mut(chunk.len()); + to_fill.copy_from_slice(chunk); + } + + capacity - buf.len() + } + + /// Returns the received data as an iterator of slices. + pub fn received_data(&self) -> impl Iterator { + self.descriptors.linked_iter().map(|desc| { + // SAFETY: We set up the descriptor to point to a subslice of the buffer, and + // here we are only recreating that slice with a perhaps shorter length. + // We are also not accessing `self.buffer` while this slice is alive, so we + // are not violating any aliasing rules. + unsafe { core::slice::from_raw_parts(desc.buffer.cast_const(), desc.len()) } + }) + } +} + +unsafe impl<'a> DmaRxBuffer for ScopedDmaRxBuf<'a> { + type View = BufView>; + type Final = ScopedDmaRxBuf<'a>; + + fn prepare(&mut self) -> Preparation { + for desc in self.descriptors.linked_iter_mut() { + desc.reset_for_rx(); + } + + cfg_if::cfg_if! { + if #[cfg(dma_can_access_psram)] { + // Optimization: avoid locking for PSRAM range. + let is_data_in_psram = !is_valid_ram_address(self.buffer.as_ptr() as usize); + if is_data_in_psram { + unsafe { + crate::soc::cache_invalidate_addr( + self.buffer.as_ptr() as u32, + self.buffer.len() as u32, + ) + }; + } + } + } + + Preparation { + start: self.descriptors.head(), + direction: TransferDirection::In, + #[cfg(dma_can_access_psram)] + accesses_psram: is_data_in_psram, + burst_transfer: self.burst, + check_owner: None, + auto_write_back: true, + } + } + + fn into_view(self) -> BufView { + BufView(self) + } + + fn from_view(view: Self::View) -> Self { + view.0 + } +} diff --git a/esp-hal/src/spi/master/dma.rs b/esp-hal/src/spi/master/dma.rs index 300867d355d..c0681a9bc0b 100644 --- a/esp-hal/src/spi/master/dma.rs +++ b/esp-hal/src/spi/master/dma.rs @@ -2,6 +2,7 @@ use core::{ cell::{Cell, UnsafeCell}, cmp::min, mem::{ManuallyDrop, MaybeUninit}, + ptr::NonNull, sync::atomic::{Ordering, fence}, }; @@ -9,8 +10,11 @@ use core::{ use embedded_hal::spi::{ErrorType, SpiBus}; use super::*; +#[cfg(dma_can_access_psram)] +use crate::{dma::ManualWritebackBuffer, soc::is_slice_in_psram}; use crate::{ dma::{ + CHUNK_SIZE, Channel, DmaChannelFor, DmaDescriptor, @@ -19,10 +23,17 @@ use crate::{ DmaRxBuffer, DmaTxBuf, DmaTxBuffer, + NoBuffer, PeripheralDmaChannel, + ScopedDmaRxBuf, + ScopedDmaTxBuf, + TransferDirection, asynch::DmaRxFuture, + prepare_for_rx, + prepare_for_tx, }, private::DropGuard, + soc::is_slice_in_dram, spi::DmaError, }; @@ -72,9 +83,11 @@ impl<'d> Spi<'d, Blocking> { /// /// The driver provides two separate approaches to transferring data: /// -/// - The slice-based API allows transferring data from/to slices of memory. The data will be copied -/// into an internal buffer before the transfer begins. These copy buffers must be set up by -/// passing them to [`with_buffers`](SpiDma::with_buffers) before the first transfer begins. +/// - The slice-based API allows transferring data from/to slices of memory. The data may be copied +/// into an internal buffer before the transfer begins. A pair of copy buffers can be set up by +/// passing them to [`with_buffers`](SpiDma::with_buffers) before the first transfer begins. For +/// more details on when copying is necessary, see the documentation of the +/// [`with_buffers`](SpiDma::with_buffers) method. /// - The buffer API allows transferring externally managed buffers. In this mode, you provide the /// buffers to be transferred. The buffer objects ensure that data is located in appropriate /// memory regions. The buffers and the driver object are moved into transfer objects for the @@ -84,6 +97,8 @@ impl<'d> Spi<'d, Blocking> { /// These approaches provide different trade-offs between memory usage / CPU overhead and ease of /// use. `embedded-hal` traits are implemented by the slice-based API's functions. /// +/// ## Examples +/// /// ```rust, no_run /// # {before_snippet} /// use esp_hal::{ @@ -95,7 +110,7 @@ impl<'d> Spi<'d, Blocking> { /// }, /// }; /// -/// // Optional: create and set up 32000 byte copy buffers. +/// // Optional: create and set up copy buffers. /// let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(32000); /// /// let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer)?; @@ -112,7 +127,6 @@ impl<'d> Spi<'d, Blocking> { /// # /// # {after_snippet} /// ``` -// TODO: update docs once the driver can transfer data without copying #[cfg_attr(feature = "defmt", derive(defmt::Format))] pub struct SpiDma<'d, Dm> where @@ -264,8 +278,8 @@ impl<'d> SpiDma<'d, Blocking> { } // The buffers must be set up when creating the driver. - unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer) }; - unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer) }; + unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer.into_scoped()) }; + unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer.into_scoped()) }; Self { spi, channel } } @@ -393,27 +407,38 @@ impl<'d> SpiDma<'d, Async> { /// Fill the given buffer with data from the bus. #[instability::unstable] pub async fn read_async(&mut self, words: &mut [u8]) -> Result<(), Error> { + if words.is_empty() { + return Ok(()); + } + self.wait_for_idle_async().await; self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 { + let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) { + DmaOperationKind::Copied => { + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct { + descriptors: &mut descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + }; + + if maybe_copy_buffer.chunk_size() == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = rx_buffer.capacity().min(MAX_DMA_SIZE); - - for chunk in words.chunks_mut(chunk_size) { - let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer()); + for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) { + let read_bytes = chunk.len(); + let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) }; + let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) }; - unsafe { spi.start_dma_transfer(chunk.len(), 0, rx_buffer, tx_buffer)? }; + self.transfer_buffers_dma_async(read_bytes, 0, rx_buffer, tx_buffer) + .await?; - spi.wait_for_idle_async().await; - - chunk.copy_from_slice(&rx_buffer.as_slice()[..chunk.len()]); - - spi.defuse(); + maybe_copy_buffer.finish(chunk); } Ok(()) @@ -422,26 +447,33 @@ impl<'d> SpiDma<'d, Async> { /// Transmit the given buffer to the bus. #[instability::unstable] pub async fn write_async(&mut self, words: &[u8]) -> Result<(), Error> { + if words.is_empty() { + return Ok(()); + } + self.wait_for_idle_async().await; self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if tx_buffer.capacity() == 0 { + let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) { + DmaOperationKind::Copied => { + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors), + }; + + if maybe_copy_buffer.chunk_size() == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let mut spi = DropGuard::new(self, |spi| spi.cancel_transfer()); - let chunk_size = tx_buffer.capacity().min(MAX_DMA_SIZE); + for chunk in words.chunks(maybe_copy_buffer.chunk_size()) { + let write_bytes = chunk.len(); + let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) }; + let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) }; - for chunk in words.chunks(chunk_size) { - tx_buffer.as_mut_slice()[..chunk.len()].copy_from_slice(chunk); - - unsafe { spi.start_dma_transfer(0, chunk.len(), rx_buffer, tx_buffer)? }; - - spi.wait_for_idle_async().await; + self.transfer_buffers_dma_async(0, write_bytes, rx_buffer, tx_buffer) + .await?; } - spi.defuse(); Ok(()) } @@ -450,18 +482,42 @@ impl<'d> SpiDma<'d, Async> { /// the bus into another buffer. #[instability::unstable] pub async fn transfer_async(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> { + if read.is_empty() && write.is_empty() { + return Ok(()); + } + self.wait_for_idle_async().await; self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 || tx_buffer.capacity() == 0 { + let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read) { + DmaOperationKind::Copied => { + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct { + descriptors: &mut rx_descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + }; + + let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write) { + DmaOperationKind::Copied => { + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors), + }; + + let chunk_size = min( + maybe_copy_rx_buffer.chunk_size(), + maybe_copy_tx_buffer.chunk_size(), + ); + + if chunk_size == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer()); - let chunk_size = min(rx_buffer.capacity(), tx_buffer.capacity()).min(MAX_DMA_SIZE); - let common_length = min(read.len(), write.len()); let (read_common, read_remainder) = read.split_at_mut(common_length); let (write_common, write_remainder) = write.split_at(common_length); @@ -470,18 +526,17 @@ impl<'d> SpiDma<'d, Async> { .chunks_mut(chunk_size) .zip(write_common.chunks(chunk_size)) { - tx_buffer.as_mut_slice()[..write_chunk.len()].copy_from_slice(write_chunk); + let read_bytes = read_chunk.len(); + let write_bytes = write_chunk.len(); + let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) }; + let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) }; - unsafe { - spi.start_dma_transfer(read_chunk.len(), write_chunk.len(), rx_buffer, tx_buffer)?; - } - spi.wait_for_idle_async().await; + self.transfer_buffers_dma_async(read_bytes, write_bytes, rx_buffer, tx_buffer) + .await?; - read_chunk.copy_from_slice(&rx_buffer.as_slice()[..read_chunk.len()]); + maybe_copy_rx_buffer.finish(read_chunk); } - spi.defuse(); - if !read_remainder.is_empty() { self.read_async(read_remainder).await } else if !write_remainder.is_empty() { @@ -495,34 +550,220 @@ impl<'d> SpiDma<'d, Async> { /// the bus into the same buffer. #[instability::unstable] pub async fn transfer_in_place_async(&mut self, words: &mut [u8]) -> Result<(), Error> { + if words.is_empty() { + return Ok(()); + } + self.wait_for_idle_async().await; self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 || tx_buffer.capacity() == 0 { + let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) = + match DmaOperationKind::for_write(words) { + DmaOperationKind::Copied => ( + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }), + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }), + ), + DmaOperationKind::InPlace => ( + MaybeCopyRxBuf::Direct { + descriptors: &mut rx_descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + MaybeCopyTxBuf::Direct(&mut tx_descriptors), + ), + }; + + let chunk_size = min( + maybe_copy_rx_buffer.chunk_size(), + maybe_copy_tx_buffer.chunk_size(), + ); + + if chunk_size == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = tx_buffer.capacity().min(MAX_DMA_SIZE); - - let mut spi = DropGuard::new(self, |spi| spi.cancel_transfer()); for chunk in words.chunks_mut(chunk_size) { - tx_buffer.as_mut_slice()[..chunk.len()].copy_from_slice(chunk); + let bytes = chunk.len(); + let ptr = NonNull::from(&mut *chunk); + let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) }; + let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) }; - unsafe { - spi.start_dma_transfer(chunk.len(), chunk.len(), rx_buffer, tx_buffer)?; - } - spi.wait_for_idle_async().await; - chunk.copy_from_slice(&rx_buffer.as_slice()[..chunk.len()]); + self.transfer_buffers_dma_async(bytes, bytes, rx_buffer, tx_buffer) + .await?; + + maybe_copy_rx_buffer.finish(chunk); } - spi.defuse(); + Ok(()) + } + async fn transfer_buffers_dma_async( + &mut self, + read_bytes: usize, + write_bytes: usize, + mut rx_buffer: impl DmaRxBuffer, + mut tx_buffer: impl DmaTxBuffer, + ) -> Result<(), Error> { + let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer()); + unsafe { + spi.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?; + } + spi.wait_for_idle_async().await; + spi.defuse(); Ok(()) } } +// +1 to make sure we have enough descriptors to satisfy strict alignment requirements +const LINK_DESCRIPTOR_COUNT: usize = MAX_DMA_SIZE.div_ceil(CHUNK_SIZE) + 2 + 1; + +enum MaybeCopyTxBuf<'a> { + Copy(&'a mut ScopedDmaTxBuf<'static>), + Direct(&'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT]), +} + +impl<'a> MaybeCopyTxBuf<'a> { + unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer { + match self { + MaybeCopyTxBuf::Copy(tx_buffer) => { + tx_buffer.as_mut_slice()[..data.len()].copy_from_slice(unsafe { data.as_ref() }); + NoBuffer(tx_buffer.prepare()) + } + MaybeCopyTxBuf::Direct(descriptors) => { + let (buffer, _) = unsafe { unwrap!(prepare_for_tx(&mut **descriptors, data, 1)) }; + buffer + } + } + } + + fn chunk_size(&self) -> usize { + match self { + MaybeCopyTxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE), + MaybeCopyTxBuf::Direct(_) => MAX_DMA_SIZE, + } + } +} + +enum MaybeCopyRxBuf<'a> { + Copy(&'a mut ScopedDmaRxBuf<'static>), + Direct { + descriptors: &'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT], + #[cfg(dma_can_access_psram)] + align_buffer: [Option; 2], + }, +} + +impl<'a> MaybeCopyRxBuf<'a> { + unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer { + match self { + MaybeCopyRxBuf::Copy(rx_buffer) => NoBuffer(rx_buffer.prepare()), + MaybeCopyRxBuf::Direct { + descriptors, + #[cfg(dma_can_access_psram)] + align_buffer, + } => { + let (buffer, _) = unsafe { + prepare_for_rx( + &mut **descriptors, + #[cfg(dma_can_access_psram)] + align_buffer, + data, + ) + }; + buffer + } + } + } + + fn chunk_size(&self) -> usize { + match self { + MaybeCopyRxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE), + MaybeCopyRxBuf::Direct { .. } => MAX_DMA_SIZE, + } + } + + fn finish(&mut self, chunk: &mut [u8]) { + match self { + MaybeCopyRxBuf::Copy(buffer) => { + chunk.copy_from_slice(&buffer.as_slice()[..chunk.len()]); + } + MaybeCopyRxBuf::Direct { + #[cfg(dma_can_access_psram)] + align_buffer, + .. + } => + { + #[cfg(dma_can_access_psram)] + for buffer in align_buffer.iter_mut() { + if let Some(buffer) = buffer.as_ref() { + buffer.write_back(); + } + *buffer = None; + } + } + } + } +} + +enum DmaOperationKind { + /// The entire slice must be copied into the internal buffer first + Copied, + + /// The slice can be transferred directly, with minimal copying done for alignment + InPlace, +} + +impl DmaOperationKind { + fn compute(buffer: &[u8], direction: TransferDirection) -> Self { + fn is_dma_compatible(buffer: &[u8], _direction: TransferDirection) -> bool { + // FIXME: lazy workaround for ESP32 TX DMA alignment requirements. + // `prepare_for_tx` and `prepare_for_rx` should be updated to handle ESP32. + #[cfg(esp32)] + if !((buffer.as_ptr() as usize).is_multiple_of(4) && buffer.len().is_multiple_of(4)) { + return false; + } + + if is_slice_in_dram(buffer) { + return true; + } + #[cfg(dma_can_access_psram)] + if is_slice_in_psram(buffer) { + #[cfg(esp32s2)] + if _direction == TransferDirection::In { + // For some reason, having tail bytes in internal RAM causes issues, so we + // force copying if the end of the PSRAM buffer is not aligned. + let tail_bytes = (buffer.as_ptr() as usize + buffer.len()).wrapping_neg() & 15; + if tail_bytes > 0 { + return false; + } + } + + return true; + } + + // TODO: C5+ DMA can read from flash + + false + } + + if is_dma_compatible(buffer, direction) { + Self::InPlace + } else { + Self::Copied + } + } + + fn for_read(buffer: &mut [u8]) -> Self { + Self::compute(buffer, TransferDirection::In) + } + + fn for_write(buffer: &[u8]) -> Self { + Self::compute(buffer, TransferDirection::Out) + } +} + impl<'d, Dm> SpiDma<'d, Dm> where Dm: DriverMode, @@ -688,15 +929,31 @@ where /// /// These buffers will be used to copy data when using the slice-based transfer functions. /// + /// Data is copied in two cases: + /// - When the buffer is not located in a memory region that can be accessed by the DMA. + #[cfg_attr(not(any(esp32c5, esp32c61)), doc = "The DMA cannot read flash memory.")] + /// - When the alignment of the buffer does not meet the DMA's requirements, the unaligned + /// parts of the buffer are copied. + #[cfg_attr( + esp32, + doc = "On ESP32, transferring from internal SRAM requires copying the entire buffer if it is + not 4-byte aligned. This is a limitation of the current implementation." + )] + #[cfg_attr( + esp32s2, + doc = "On ESP32-S2, receiving into to PSRAM requires the buffer's _end_ to be 16-byte + aligned, otherwise the driver requires copying the entire buffer." + )] + #[doc = ""] /// The maximum useful size for these buffers is 32736 bytes, any additional memory will /// be wasted. - // TODO: update docs once the driver can actually transfer data without copying - we'll need to - // explain when this function is necessary. + /// + /// For an example of how to create these buffers, see the [`SpiDma`] documentation. #[instability::unstable] pub fn with_buffers(self, dma_rx_buf: DmaRxBuf, dma_tx_buf: DmaTxBuf) -> SpiDma<'d, Dm> { unsafe { - (&mut *self.dma_driver().state.rx_buffer.get()).write(dma_rx_buf); - (&mut *self.dma_driver().state.tx_buffer.get()).write(dma_tx_buf); + (&mut *self.spi.dma_state().rx_buffer.get()).write(dma_rx_buf.into_scoped()); + (&mut *self.spi.dma_state().tx_buffer.get()).write(dma_tx_buf.into_scoped()); } self } @@ -945,27 +1202,50 @@ where self.driver().apply_config(config) } + fn transfer_buffers_dma( + &mut self, + read_bytes: usize, + write_bytes: usize, + mut rx_buffer: impl DmaRxBuffer, + mut tx_buffer: impl DmaTxBuffer, + ) -> Result<(), Error> { + unsafe { + self.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?; + } + self.wait_for_idle(); + Ok(()) + } + /// Reads data from the SPI bus using DMA. #[instability::unstable] pub fn read(&mut self, words: &mut [u8]) -> Result<(), Error> { self.wait_for_idle(); self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 { + let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) { + DmaOperationKind::Copied => { + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct { + descriptors: &mut descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + }; + + if maybe_copy_buffer.chunk_size() == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = rx_buffer.capacity().min(MAX_DMA_SIZE); + for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) { + let read_bytes = chunk.len(); + let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) }; + let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) }; - for chunk in words.chunks_mut(chunk_size) { - unsafe { - self.start_dma_transfer(chunk.len(), 0, rx_buffer, tx_buffer)?; - } + self.transfer_buffers_dma(read_bytes, 0, rx_buffer, tx_buffer)?; - self.wait_for_idle(); - chunk.copy_from_slice(&rx_buffer.as_slice()[..chunk.len()]); + maybe_copy_buffer.finish(chunk); } Ok(()) @@ -977,22 +1257,24 @@ where self.wait_for_idle(); self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if tx_buffer.capacity() == 0 { + let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) { + DmaOperationKind::Copied => { + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors), + }; + + if maybe_copy_buffer.chunk_size() == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = tx_buffer.capacity().min(MAX_DMA_SIZE); + for chunk in words.chunks(maybe_copy_buffer.chunk_size()) { + let write_bytes = chunk.len(); + let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) }; + let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) }; - for chunk in words.chunks(chunk_size) { - tx_buffer.as_mut_slice()[..chunk.len()].copy_from_slice(chunk); - - unsafe { - self.start_dma_transfer(0, chunk.len(), rx_buffer, tx_buffer)?; - } - - self.wait_for_idle(); + self.transfer_buffers_dma(0, write_bytes, rx_buffer, tx_buffer)?; } Ok(()) @@ -1004,14 +1286,35 @@ where self.wait_for_idle(); self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 || tx_buffer.capacity() == 0 { + let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read) { + DmaOperationKind::Copied => { + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct { + descriptors: &mut rx_descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + }; + + let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write) { + DmaOperationKind::Copied => { + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }) + } + DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors), + }; + + let chunk_size = min( + maybe_copy_rx_buffer.chunk_size(), + maybe_copy_tx_buffer.chunk_size(), + ); + + if chunk_size == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = min(rx_buffer.capacity(), tx_buffer.capacity()).min(MAX_DMA_SIZE); - let common_length = min(read.len(), write.len()); let (read_common, read_remainder) = read.split_at_mut(common_length); let (write_common, write_remainder) = write.split_at(common_length); @@ -1020,14 +1323,14 @@ where .chunks_mut(chunk_size) .zip(write_common.chunks(chunk_size)) { - tx_buffer.as_mut_slice()[..write_chunk.len()].copy_from_slice(write_chunk); + let read_bytes = read_chunk.len(); + let write_bytes = write_chunk.len(); + let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) }; + let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) }; - unsafe { - self.start_dma_transfer(read_chunk.len(), write_chunk.len(), rx_buffer, tx_buffer)?; - } - self.wait_for_idle(); + self.transfer_buffers_dma(read_bytes, write_bytes, rx_buffer, tx_buffer)?; - read_chunk.copy_from_slice(&rx_buffer.as_slice()[..read_chunk.len()]); + maybe_copy_rx_buffer.finish(read_chunk); } if !read_remainder.is_empty() { @@ -1045,22 +1348,42 @@ where self.wait_for_idle(); self.driver().setup_full_duplex()?; - let rx_buffer = unsafe { self.dma_driver().rx_buffer() }; - let tx_buffer = unsafe { self.dma_driver().tx_buffer() }; - if rx_buffer.capacity() == 0 || tx_buffer.capacity() == 0 { + let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT]; + let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) = + match DmaOperationKind::for_write(words) { + DmaOperationKind::Copied => ( + MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }), + MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }), + ), + DmaOperationKind::InPlace => ( + MaybeCopyRxBuf::Direct { + descriptors: &mut rx_descriptors, + #[cfg(dma_can_access_psram)] + align_buffer: [const { None }; 2], + }, + MaybeCopyTxBuf::Direct(&mut tx_descriptors), + ), + }; + + let chunk_size = min( + maybe_copy_rx_buffer.chunk_size(), + maybe_copy_tx_buffer.chunk_size(), + ); + + if chunk_size == 0 { return Err(Error::from(DmaError::BufferTooSmall)); } - let chunk_size = min(rx_buffer.capacity(), tx_buffer.capacity()).min(MAX_DMA_SIZE); - for chunk in words.chunks_mut(chunk_size) { - tx_buffer.as_mut_slice()[..chunk.len()].copy_from_slice(chunk); + let bytes = chunk.len(); + let ptr = NonNull::from(&mut *chunk); + let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) }; + let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) }; - unsafe { - self.start_dma_transfer(chunk.len(), chunk.len(), rx_buffer, tx_buffer)?; - } - self.wait_for_idle(); - chunk.copy_from_slice(&rx_buffer.as_slice()[..chunk.len()]); + self.transfer_buffers_dma(bytes, bytes, rx_buffer, tx_buffer)?; + + maybe_copy_rx_buffer.finish(chunk); } Ok(()) @@ -1221,11 +1544,11 @@ pub(super) struct DmaDriver { } impl DmaDriver { - unsafe fn rx_buffer(&self) -> &'static mut DmaRxBuf { + unsafe fn rx_buffer(&self) -> &'static mut ScopedDmaRxBuf<'static> { unsafe { self.state.rx_buffer() } } - unsafe fn tx_buffer(&self) -> &'static mut DmaTxBuf { + unsafe fn tx_buffer(&self) -> &'static mut ScopedDmaTxBuf<'static> { unsafe { self.state.tx_buffer() } } @@ -1386,8 +1709,8 @@ struct DmaState { tx_transfer_in_progress: Cell, rx_transfer_in_progress: Cell, - rx_buffer: UnsafeCell>, - tx_buffer: UnsafeCell>, + rx_buffer: UnsafeCell>>, + tx_buffer: UnsafeCell>>, } impl DmaState { @@ -1400,7 +1723,7 @@ impl DmaState { clippy::mut_from_ref, reason = "Safety requirements ensure this is okay" )] - unsafe fn rx_buffer(&self) -> &mut DmaRxBuf { + unsafe fn rx_buffer(&self) -> &mut ScopedDmaRxBuf<'static> { unsafe { (&mut *self.rx_buffer.get()).assume_init_mut() } } @@ -1413,7 +1736,7 @@ impl DmaState { clippy::mut_from_ref, reason = "Safety requirements ensure this is okay" )] - unsafe fn tx_buffer(&self) -> &mut DmaTxBuf { + unsafe fn tx_buffer(&self) -> &mut ScopedDmaTxBuf<'static> { unsafe { (&mut *self.tx_buffer.get()).assume_init_mut() } } } diff --git a/hil-test/src/bin/spi_full_duplex.rs b/hil-test/src/bin/spi_full_duplex.rs index 76fb6941743..74b9e08df9d 100644 --- a/hil-test/src/bin/spi_full_duplex.rs +++ b/hil-test/src/bin/spi_full_duplex.rs @@ -1,7 +1,7 @@ //! SPI Full Duplex test suite. //% CHIPS: esp32 esp32c2 esp32c3 esp32c5 esp32c6 esp32c61 esp32h2 esp32s2 esp32s3 -//% FEATURES(unstable): unstable +//% FEATURES(unstable): esp-alloc unstable //% FEATURES(stable): // FIXME: add async test cases that don't rely on PCNT @@ -29,9 +29,17 @@ cfg_if::cfg_if! { gpio::{Level, NoPin}, dma::{DmaDescriptor, DmaRxBuf, DmaTxBuf}, dma_buffers, + spi::master::SpiDma, }; + + #[cfg(all(spi_master_supports_dma, dma_can_access_psram))] + use allocator_api2::vec::Vec; + #[cfg(pcnt_driver_supported)] use esp_hal::pcnt::{channel::EdgeMode, unit::Unit, Pcnt}; + + #[cfg(all(spi_master_supports_dma, pcnt_driver_supported))] + use esp_hal::Async; } } @@ -68,8 +76,123 @@ struct Context { pcnt_unit: Unit<'static, 0>, } +#[cfg(all(pcnt_driver_supported, feature = "unstable"))] +macro_rules! set_up_pcnt { + ($ctx:expr, $input:ident) => {{ + $ctx.pcnt_unit + .channel0 + .set_edge_signal($ctx.$input.peripheral_input()); + $ctx.pcnt_unit + .channel0 + .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + $ctx.pcnt_unit + }}; +} + +/// Helper function to run a test case with various memory regions for the buffers, including +/// various alignments in external RAM if available. +#[cfg(feature = "unstable")] +fn run_test_in_all_memory_regions( + mut test_fn: impl FnMut(&mut [u8], &mut [u8]), +) { + defmt::info!("Testing buffer size {}", BUFFER_SIZE); + // TODO: add cases that mix internal and external RAM. + + // Test buffers in internal RAM + let mut tx_buf = [0; BUFFER_SIZE]; + let mut rx_buf = [0; BUFFER_SIZE]; + test_fn(&mut tx_buf, &mut rx_buf); + + #[cfg(dma_can_access_psram)] + { + // Test buffers in external RAM using various alignment offsets. + + const MAX_SHIFT: usize = 15; + let buf_len = BUFFER_SIZE + MAX_SHIFT; + + let mut external_rx_memory = Vec::with_capacity_in(buf_len, esp_alloc::ExternalMemory); + let mut external_tx_memory = Vec::with_capacity_in(buf_len, esp_alloc::ExternalMemory); + + external_rx_memory.resize(buf_len, 0); + external_tx_memory.resize(buf_len, 0); + + for shift in 0..=MAX_SHIFT { + external_rx_memory.fill(0); + + defmt::info!("Testing PSRAM with shift {}", shift); + let rx_buf = &mut external_rx_memory[shift..][..BUFFER_SIZE]; + let tx_buf = &mut external_tx_memory[shift..][..BUFFER_SIZE]; + + test_fn(tx_buf, rx_buf); + assert!(external_rx_memory[..shift].iter().all(|&b| b == 0)); + assert!( + external_rx_memory[shift + BUFFER_SIZE..] + .iter() + .all(|&b| b == 0) + ); + } + } +} + +/// Async version of [`run_test_in_all_memory_regions`]. +#[cfg(feature = "unstable")] +async fn run_async_test_in_all_memory_regions( + mut test_fn: impl AsyncFnMut(&mut [u8], &mut [u8]), +) { + defmt::info!("Testing buffer size {}", BUFFER_SIZE); + let mut tx_buf = [0; BUFFER_SIZE]; + let mut rx_buf = [0; BUFFER_SIZE]; + test_fn(&mut tx_buf, &mut rx_buf).await; + + #[cfg(dma_can_access_psram)] + { + const MAX_SHIFT: usize = 15; + let buf_len = BUFFER_SIZE + MAX_SHIFT; + + let mut external_rx_memory = Vec::with_capacity_in(buf_len, esp_alloc::ExternalMemory); + let mut external_tx_memory = Vec::with_capacity_in(buf_len, esp_alloc::ExternalMemory); + + external_rx_memory.resize(buf_len, 0); + external_tx_memory.resize(buf_len, 0); + + for shift in 0..=MAX_SHIFT { + external_rx_memory.fill(0); + + defmt::info!("Testing PSRAM with shift {}", shift); + let rx_buf = &mut external_rx_memory[shift..][..BUFFER_SIZE]; + let tx_buf = &mut external_tx_memory[shift..][..BUFFER_SIZE]; + + test_fn(tx_buf, rx_buf).await; + assert!(external_rx_memory[..shift].iter().all(|&b| b == 0)); + assert!( + external_rx_memory[shift + BUFFER_SIZE..] + .iter() + .all(|&b| b == 0) + ); + } + } +} + +#[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] +fn count_edges(buf: &[u8]) -> i16 { + let mut count = 0; + let mut prev_bit = 0; + for byte in buf { + for i in 0..8 { + let bit = (byte >> i) & 1; + if bit == 1 && prev_bit == 0 { + count += 1; + } + prev_bit = bit; + } + } + count +} + #[embedded_test::tests(default_timeout = 3, executor = hil_test::Executor::new())] mod tests { + use hil_test::assert_eq; + use super::*; #[init] @@ -78,6 +201,9 @@ mod tests { esp_hal::Config::default().with_cpu_clock(esp_hal::clock::CpuClock::max()), ); + #[cfg(all(dma_can_access_psram, feature = "unstable"))] + esp_alloc::psram_allocator!(peripherals.PSRAM, esp_hal::psram); + let (_, miso) = hil_test::common_test_pins!(peripherals); let sclk = hil_test::unconnected_pin!(peripherals); @@ -184,22 +310,15 @@ mod tests { let write = [0xde, 0xad, 0xbe, 0xef]; let mut read = [0x00; 2]; - cfg_if::cfg_if! { - if #[cfg(all(pcnt_driver_supported, feature = "unstable"))] { - let unit = ctx.pcnt_unit; - unit.channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - } - } + #[cfg(all(pcnt_driver_supported, feature = "unstable"))] + let sclk_counter = set_up_pcnt!(ctx, sclk_input); SpiBus::transfer(&mut ctx.spi, &mut read, &write).expect("Asymmetric transfer failed"); assert_eq!(read[0], write[0]); assert_eq!(read[1], write[1]); #[cfg(all(pcnt_driver_supported, feature = "unstable"))] - assert_eq!(unit.value(), 4 * 8); + assert_eq!(sclk_counter.value(), 4 * 8); } #[test] @@ -207,15 +326,8 @@ mod tests { let write = [0xde, 0xad]; let mut read = [0x00; 4]; - cfg_if::cfg_if! { - if #[cfg(all(pcnt_driver_supported, feature = "unstable"))] { - let unit = ctx.pcnt_unit; - unit.channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - } - } + #[cfg(all(pcnt_driver_supported, feature = "unstable"))] + let sclk_counter = set_up_pcnt!(ctx, sclk_input); SpiBus::transfer(&mut ctx.spi, &mut read, &write).expect("Asymmetric transfer failed"); assert_eq!(read[0], write[0]); @@ -224,7 +336,7 @@ mod tests { assert_eq!(read[3], 0x00); #[cfg(all(pcnt_driver_supported, feature = "unstable"))] - assert_eq!(unit.value(), 4 * 8); + assert_eq!(sclk_counter.value(), 4 * 8); } #[test] @@ -232,15 +344,8 @@ mod tests { let write = [0xde, 0xad, 0xbe, 0xef]; let mut read = [0x00; 2]; - cfg_if::cfg_if! { - if #[cfg(all(pcnt_driver_supported, feature = "unstable"))] { - let unit = ctx.pcnt_unit; - unit.channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - } - } + #[cfg(all(pcnt_driver_supported, feature = "unstable"))] + let sclk_counter = set_up_pcnt!(ctx, sclk_input); let mut spi = ctx.spi.into_async(); SpiBusAsync::transfer(&mut spi, &mut read, &write) @@ -250,7 +355,7 @@ mod tests { assert_eq!(read[1], write[1]); #[cfg(all(pcnt_driver_supported, feature = "unstable"))] - assert_eq!(unit.value(), 4 * 8); + assert_eq!(sclk_counter.value(), 4 * 8); } #[test] @@ -258,15 +363,8 @@ mod tests { let write = [0xde, 0xad]; let mut read = [0x00; 4]; - cfg_if::cfg_if! { - if #[cfg(all(pcnt_driver_supported, feature = "unstable"))] { - let unit = ctx.pcnt_unit; - unit.channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); - } - } + #[cfg(all(pcnt_driver_supported, feature = "unstable"))] + let sclk_counter = set_up_pcnt!(ctx, sclk_input); let mut spi = ctx.spi.into_async(); SpiBusAsync::transfer(&mut spi, &mut read, &write) @@ -278,7 +376,7 @@ mod tests { assert_eq!(read[3], 0x00); #[cfg(all(pcnt_driver_supported, feature = "unstable"))] - assert_eq!(unit.value(), 4 * 8); + assert_eq!(sclk_counter.value(), 4 * 8); } #[test] @@ -286,18 +384,13 @@ mod tests { fn test_asymmetric_write(mut ctx: Context) { let write = [0xde, 0xad, 0xbe, 0xef]; - let unit = ctx.pcnt_unit; - - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); SpiBus::write(&mut ctx.spi, &write[..]).expect("Asymmetric write failed"); // Flush because we're not reading, so the write may happen in the background ctx.spi.flush().expect("Flush failed"); - assert_eq!(unit.value(), 9); + assert_eq!(miso_pulse_counter.value(), 9); } #[test] @@ -305,19 +398,14 @@ mod tests { async fn test_async_asymmetric_write(ctx: Context) { let write = [0xde, 0xad, 0xbe, 0xef]; - let unit = ctx.pcnt_unit; - - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); let mut spi = ctx.spi.into_async(); SpiBusAsync::write(&mut spi, &write[..]) .await .expect("Asymmetric write failed"); - assert_eq!(unit.value(), 9); + assert_eq!(miso_pulse_counter.value(), 9); } #[test] @@ -325,12 +413,7 @@ mod tests { async fn async_write_after_sync_write_waits_for_flush(ctx: Context) { let write = [0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef]; - let unit = ctx.pcnt_unit; - - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); let mut spi = ctx.spi.into_async(); @@ -343,7 +426,7 @@ mod tests { .await .expect("Async write failed"); - assert_eq!(unit.value(), 34); + assert_eq!(miso_pulse_counter.value(), 34); } #[test] @@ -351,18 +434,13 @@ mod tests { fn test_asymmetric_write_transfer(mut ctx: Context) { let write = [0xde, 0xad, 0xbe, 0xef]; - let unit = ctx.pcnt_unit; - - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); SpiBus::transfer(&mut ctx.spi, &mut [], &write[..]).expect("Asymmetric transfer failed"); // Flush because we're not reading, so the write may happen in the background ctx.spi.flush().expect("Flush failed"); - assert_eq!(unit.value(), 9); + assert_eq!(miso_pulse_counter.value(), 9); } #[test] @@ -370,19 +448,14 @@ mod tests { async fn test_async_asymmetric_write_transfer(ctx: Context) { let write = [0xde, 0xad, 0xbe, 0xef]; - let unit = ctx.pcnt_unit; - - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); let mut spi = ctx.spi.into_async(); SpiBusAsync::transfer(&mut spi, &mut [], &write[..]) .await .expect("Asymmetric transfer failed"); - assert_eq!(unit.value(), 9); + assert_eq!(miso_pulse_counter.value(), 9); } #[test] @@ -454,13 +527,9 @@ mod tests { let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); - let unit = ctx.pcnt_unit; let mut spi = ctx.spi.with_dma(ctx.dma_channel); - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); // Fill the buffer where each byte has 3 pos edges. dma_tx_buf.as_mut_slice().fill(0b0110_1010); @@ -479,7 +548,7 @@ mod tests { .map_err(|e| e.0) .unwrap(); (spi, dma_tx_buf) = transfer.wait(); - assert_eq!(unit.value(), (i * 3 * TRANSFER_SIZE) as _); + assert_eq!(miso_pulse_counter.value(), (i * 3 * TRANSFER_SIZE) as _); } } @@ -492,13 +561,9 @@ mod tests { let mut dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let mut dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); - let unit = ctx.pcnt_unit; let mut spi = ctx.spi.with_dma(ctx.dma_channel); - unit.channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - unit.channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); // Fill the buffer where each byte has 3 pos edges. dma_tx_buf.as_mut_slice().fill(0b0110_1010); @@ -517,7 +582,7 @@ mod tests { .map_err(|e| e.0) .unwrap(); (spi, (dma_rx_buf, dma_tx_buf)) = transfer.wait(); - assert_eq!(unit.value(), (i * 3 * TRANSFER_SIZE) as _); + assert_eq!(miso_pulse_counter.value(), (i * 3 * TRANSFER_SIZE) as _); } } @@ -592,42 +657,54 @@ mod tests { #[test] #[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] fn test_dma_bus_read_write_pcnt(ctx: Context) { - const TRANSFER_SIZE: usize = 4; + // Test logic + fn test_write( + spi: &mut SpiDma<'_, Blocking>, + miso_pulse_counter: &Unit<'_, 0>, + tx_buf: &mut [u8], + rx_buf: &mut [u8], + ) { + // Fill the buffer with data where each byte has 3 pos edges. + tx_buf.fill(0b0110_1010); + + for _ in 1..4 { + miso_pulse_counter.clear(); + + // Preset as 5, expect 0 repeated receive + rx_buf.fill(5); + spi.read(rx_buf).unwrap(); + assert!(rx_buf.iter().all(|&b| b == 0)); + + spi.write(tx_buf).unwrap(); + assert_eq!(miso_pulse_counter.value(), count_edges(tx_buf)); + } + } + + // Set up SPI let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); - ctx.pcnt_unit - .channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - ctx.pcnt_unit - .channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); let mut spi = ctx .spi .with_dma(ctx.dma_channel) .with_buffers(dma_rx_buf, dma_tx_buf); - // Fill the buffer where each byte has 3 pos edges. - let tx_buf = [0b0110_1010; TRANSFER_SIZE]; - let mut rx_buf = [0; TRANSFER_SIZE]; - - for i in 1..4 { - // Preset as 5, expect 0 repeated receive - rx_buf.copy_from_slice(&[5; TRANSFER_SIZE]); - spi.read(&mut rx_buf).unwrap(); - assert_eq!(rx_buf, [0; TRANSFER_SIZE]); - - spi.write(&tx_buf).unwrap(); - assert_eq!(ctx.pcnt_unit.value(), (i * 3 * TRANSFER_SIZE) as _); - } + // Run test with various buffer sizes + run_test_in_all_memory_regions::<4>(|tx_buf, rx_buf| { + test_write(&mut spi, &miso_pulse_counter, tx_buf, rx_buf); + }); + run_test_in_all_memory_regions::<128>(|tx_buf, rx_buf| { + test_write(&mut spi, &miso_pulse_counter, tx_buf, rx_buf); + }); } #[test] #[cfg(all(spi_master_supports_dma, feature = "unstable"))] fn test_dma_bus_symmetric_transfer(ctx: Context) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); + let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(128); let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); @@ -636,62 +713,61 @@ mod tests { .with_dma(ctx.dma_channel) .with_buffers(dma_rx_buf, dma_tx_buf); - let tx_buf = [0xde, 0xad, 0xbe, 0xef]; - let mut rx_buf = [0; 4]; + fn test(spi: &mut SpiDma<'_, Blocking>, tx_buf: &mut [u8], rx_buf: &mut [u8]) { + // Fill TX buffer with data + let mut i = 0u8; + tx_buf.fill_with(|| { + let byte = i; + i = i.wrapping_add(1); + byte + }); - spi.transfer(&mut rx_buf, &tx_buf).unwrap(); + defmt::debug!("Symmetric transfer"); - assert_eq!(tx_buf, rx_buf); - } - - #[test] - #[cfg(all(spi_master_supports_dma, feature = "unstable"))] - fn test_dma_bus_asymmetric_transfer(ctx: Context) { - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + spi.transfer(rx_buf, tx_buf).unwrap(); + assert_eq!(tx_buf, rx_buf); - let mut spi = ctx - .spi - .with_dma(ctx.dma_channel) - .with_buffers(dma_rx_buf, dma_tx_buf); - - let tx_buf = [0xde, 0xad, 0xbe, 0xef]; - let mut rx_buf = [0; 4]; - - spi.transfer(&mut rx_buf, &tx_buf).unwrap(); - - assert_eq!(&tx_buf[0..1], &rx_buf[0..1]); - } + rx_buf.fill(0); - #[test] - #[cfg(all(spi_master_supports_dma, feature = "unstable"))] - fn test_dma_bus_symmetric_transfer_huge_buffer(ctx: Context) { - const DMA_BUFFER_SIZE: usize = 4096; + defmt::debug!("Asymmetric transfer - read more than write"); - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(40); - let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); - let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); + let asymmetric_rx_len = rx_buf.len(); + let asymmetric_tx_len = tx_buf.len() / 2; + let smaller_len = asymmetric_rx_len.min(asymmetric_tx_len); + spi.transfer( + &mut rx_buf[0..asymmetric_rx_len], + &tx_buf[0..asymmetric_tx_len], + ) + .unwrap(); + assert_eq!(tx_buf[0..smaller_len], rx_buf[0..smaller_len]); - let mut spi = ctx - .spi - .with_dma(ctx.dma_channel) - .with_buffers(dma_rx_buf, dma_tx_buf); + rx_buf.fill(0); - let tx_buf = core::array::from_fn(|i| i as _); - let mut rx_buf = [0; DMA_BUFFER_SIZE]; + defmt::debug!("Asymmetric transfer - write more than read"); - spi.transfer(&mut rx_buf, &tx_buf).unwrap(); + let asymmetric_rx_len = rx_buf.len() / 2; + let asymmetric_tx_len = tx_buf.len(); + let smaller_len = asymmetric_rx_len.min(asymmetric_tx_len); + spi.transfer( + &mut rx_buf[0..asymmetric_rx_len], + &tx_buf[0..asymmetric_tx_len], + ) + .unwrap(); + assert_eq!(tx_buf[0..smaller_len], rx_buf[0..smaller_len]); + } - assert_eq!(tx_buf, rx_buf); + run_test_in_all_memory_regions::<4>(|tx_buf, rx_buf| test(&mut spi, tx_buf, rx_buf)); + run_test_in_all_memory_regions::<48>(|tx_buf, rx_buf| test(&mut spi, tx_buf, rx_buf)); + run_test_in_all_memory_regions::<63>(|tx_buf, rx_buf| test(&mut spi, tx_buf, rx_buf)); + // This test case exists because it (63-68 bytes) timed out on S2 when originally added. + run_test_in_all_memory_regions::<64>(|tx_buf, rx_buf| test(&mut spi, tx_buf, rx_buf)); + run_test_in_all_memory_regions::<4096>(|tx_buf, rx_buf| test(&mut spi, tx_buf, rx_buf)); } #[test] #[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] async fn test_async_dma_read_dma_write_pcnt(ctx: Context) { - const DMA_BUFFER_SIZE: usize = 8; - const TRANSFER_SIZE: usize = 5; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(DMA_BUFFER_SIZE); + let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); let mut spi = ctx @@ -700,34 +776,42 @@ mod tests { .with_buffers(dma_rx_buf, dma_tx_buf) .into_async(); - ctx.pcnt_unit - .channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - ctx.pcnt_unit - .channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); - let mut receive = [0; TRANSFER_SIZE]; + async fn test_write( + spi: &mut SpiDma<'_, Async>, + miso_pulse_counter: &Unit<'_, 0>, + tx_buf: &mut [u8], + rx_buf: &mut [u8], + ) { + tx_buf.fill(0b0110_1010); - // Fill the buffer where each byte has 3 pos edges. - let transmit = [0b0110_1010; TRANSFER_SIZE]; + for _ in 1..4 { + miso_pulse_counter.clear(); - for i in 1..4 { - receive.copy_from_slice(&[5; TRANSFER_SIZE]); - SpiBusAsync::read(&mut spi, &mut receive).await.unwrap(); - assert_eq!(receive, [0; TRANSFER_SIZE]); + rx_buf.fill(5); + SpiBusAsync::read(spi, rx_buf).await.unwrap(); + assert!(rx_buf.iter().all(|&b| b == 0)); - SpiBusAsync::write(&mut spi, &transmit).await.unwrap(); - assert_eq!(ctx.pcnt_unit.value(), (i * 3 * TRANSFER_SIZE) as _); + SpiBusAsync::write(spi, tx_buf).await.unwrap(); + assert_eq!(miso_pulse_counter.value(), count_edges(tx_buf)); + } } + + run_async_test_in_all_memory_regions::<4>(async |tx_buf, rx_buf| { + test_write(&mut spi, &miso_pulse_counter, tx_buf, rx_buf).await; + }) + .await; + run_async_test_in_all_memory_regions::<128>(async |tx_buf, rx_buf| { + test_write(&mut spi, &miso_pulse_counter, tx_buf, rx_buf).await; + }) + .await; } #[test] #[cfg(all(pcnt_driver_supported, spi_master_supports_dma, feature = "unstable"))] async fn test_async_dma_read_dma_transfer_pcnt(ctx: Context) { - const DMA_BUFFER_SIZE: usize = 8; - const TRANSFER_SIZE: usize = 5; - let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(DMA_BUFFER_SIZE); + let (rx_buffer, rx_descriptors, tx_buffer, tx_descriptors) = dma_buffers!(4); let dma_rx_buf = DmaRxBuf::new(rx_descriptors, rx_buffer).unwrap(); let dma_tx_buf = DmaTxBuf::new(tx_descriptors, tx_buffer).unwrap(); let mut spi = ctx @@ -736,29 +820,37 @@ mod tests { .with_buffers(dma_rx_buf, dma_tx_buf) .into_async(); - ctx.pcnt_unit - .channel0 - .set_edge_signal(ctx.miso_input.peripheral_input()); - ctx.pcnt_unit - .channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let miso_pulse_counter = set_up_pcnt!(ctx, miso_input); - let mut receive = [0; TRANSFER_SIZE]; + async fn test_transfer( + spi: &mut SpiDma<'_, Async>, + miso_pulse_counter: &Unit<'_, 0>, + tx_buf: &mut [u8], + rx_buf: &mut [u8], + ) { + tx_buf.fill(0b0110_1010); - // Fill the buffer where each byte has 3 pos edges. - let transmit = [0b0110_1010; TRANSFER_SIZE]; + for _ in 1..4 { + miso_pulse_counter.clear(); - for i in 1..4 { - receive.copy_from_slice(&[5; TRANSFER_SIZE]); - SpiBusAsync::read(&mut spi, &mut receive).await.unwrap(); - assert_eq!(receive, [0; TRANSFER_SIZE]); + rx_buf.fill(5); + SpiBusAsync::read(spi, rx_buf).await.unwrap(); + assert!(rx_buf.iter().all(|&b| b == 0)); - SpiBusAsync::transfer(&mut spi, &mut receive, &transmit) - .await - .unwrap(); - assert_eq!(ctx.pcnt_unit.value(), (i * 3 * TRANSFER_SIZE) as _); - assert_eq!(receive, [0b0110_1010; TRANSFER_SIZE]); + SpiBusAsync::transfer(spi, rx_buf, tx_buf).await.unwrap(); + assert_eq!(miso_pulse_counter.value(), count_edges(tx_buf)); + assert_eq!(tx_buf, rx_buf); + } } + + run_async_test_in_all_memory_regions::<4>(async |tx_buf, rx_buf| { + test_transfer(&mut spi, &miso_pulse_counter, tx_buf, rx_buf).await; + }) + .await; + run_async_test_in_all_memory_regions::<128>(async |tx_buf, rx_buf| { + test_transfer(&mut spi, &miso_pulse_counter, tx_buf, rx_buf).await; + }) + .await; } #[test] @@ -962,12 +1054,7 @@ mod tests { fn half_duplex_operation_works_after_nonblocking_write(mut ctx: Context) { // SpiBus::write returns before the transfer is complete. This test verifies that // half_duplex functions flush before they reconfigure the peripheral. - ctx.pcnt_unit - .channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - ctx.pcnt_unit - .channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let sclk_counter = set_up_pcnt!(ctx, sclk_input); ctx.spi .apply_config(&Config::default().with_frequency(Rate::from_khz(80))) @@ -982,9 +1069,9 @@ mod tests { .half_duplex_write(DataMode::Dual, Command::None, Address::None, 0, &buffer) .unwrap(); - assert_eq!(ctx.pcnt_unit.value(), 480); + assert_eq!(sclk_counter.value(), 480); - ctx.pcnt_unit.clear(); + sclk_counter.clear(); // 320 clock cycles let mut buffer = [0x00; 40]; @@ -995,7 +1082,7 @@ mod tests { .half_duplex_read(DataMode::Dual, Command::None, Address::None, 0, &mut buffer) .unwrap(); - assert_eq!(ctx.pcnt_unit.value(), 480); + assert_eq!(sclk_counter.value(), 480); } #[test] @@ -1054,12 +1141,7 @@ mod tests { fn half_duplex_operation_resets_size_before_empty_write(mut ctx: Context) { // SpiBus::write returns before the transfer is complete. This test verifies that // half_duplex functions flush before they reconfigure the peripheral. - ctx.pcnt_unit - .channel0 - .set_edge_signal(ctx.sclk_input.peripheral_input()); - ctx.pcnt_unit - .channel0 - .set_input_mode(EdgeMode::Hold, EdgeMode::Increment); + let sclk_counter = set_up_pcnt!(ctx, sclk_input); ctx.spi .apply_config(&Config::default().with_frequency(Rate::from_khz(80))) @@ -1082,7 +1164,7 @@ mod tests { ) .unwrap(); - assert_eq!(ctx.pcnt_unit.value(), 40); + assert_eq!(sclk_counter.value(), 40); } #[test] @@ -1110,9 +1192,7 @@ mod tests { let tx_buf = [0xde, 0xad, 0xbe, 0xef]; let mut rx_buf = [0; 4]; - spi.transfer(&mut rx_buf, &tx_buf).unwrap(); - assert_eq!(tx_buf, rx_buf); }