//! # TODO //! * Get rid of leaking buffers, if possible. //! * Add a parameter for the IRAM allocator. //! * Customizable interrupt handler priority. //! * Reorganize code into multiple modules. //! * Make peripherals generic. //! * Make the API actually safe to use. //! Currently, multiple instances are prevented from being instantiated by the usage //! of non-generic peripheral types which we hold onto. //! * Add V-SYNC support. //! * Add support for acyclic buffers and acyclic outbound transmissions (`Dpi::send(false, ..)`). //! * Release an 0.1.0. #![no_std] #![feature(allocator_api)] use core::{ alloc::Layout, cell::RefCell, fmt::Debug, ops::{Deref, DerefMut}, sync::atomic::{self, AtomicBool}, }; use alloc::{alloc::Allocator, boxed::Box, sync::Arc, vec}; use embassy_sync::{blocking_mutex::raw::CriticalSectionRawMutex, mutex::Mutex}; use esp_hal::{ Blocking, dma::{ self, BurstConfig, DmaDescriptor, DmaTxBuffer, Mem2Mem, SimpleMem2Mem, SimpleMem2MemTransfer, }, handler, interrupt::{self, Priority}, lcd_cam::lcd::dpi::{Dpi, DpiTransfer}, peripherals::{DMA, DMA_CH0, Interrupt}, ram, spi::master::AnySpi, }; use ouroboros::self_referencing; extern crate alloc; const DMA_CHANNEL_OUTBOUND: usize = 2; const INTERRUPT_OUTBOUND: Interrupt = Interrupt::DMA_OUT_CH2; static RUNNING_DMA_BOUNCE: Mutex>> = Mutex::new(RefCell::new(None)); struct DmaPeripheralWithChannel<'a> { peripheral: AnySpi<'a>, channel: DMA_CH0<'a>, } mod transfer { use super::*; // This would be the ideal implementation, but it doesn't work, because // I'm not sure how I could make lifetime `'d` in `SimpleMem2MemTransfer` invariant. // // #[self_referencing] // struct ReceivingTransfer { // peripheral: DmaPeripheralWithChannel<'static>, // #[borrows(mut peripheral)] // #[covariant] // mem2mem: SimpleMem2Mem<'this, Blocking>, // #[borrows(mut mem2mem)] // #[covariant] // transfer: Option>, // } #[self_referencing] struct ReceivingTransferInner { // This peripheral simultaneously exists cloned in `mem2mem`. // Care must be taken that it is not accessed before `mem2mem` is dropped. // // Implementation note: // Ideally, this would be referenced by `mem2mem` via `#[borrows(mut peripheral)]`, // but then ouroboros complains about the invariant lifetime `'d` on `transfer`. peripheral: DmaPeripheralWithChannel<'static>, mem2mem: SimpleMem2Mem<'static, Blocking>, #[borrows(mut mem2mem)] #[covariant] transfer: Option>, } pub struct ReceivingTransfer(ReceivingTransferInner); impl ReceivingTransfer { pub fn new( peripheral: DmaPeripheralWithChannel<'static>, mem2mem_builder: impl FnOnce( DmaPeripheralWithChannel<'static>, ) -> SimpleMem2Mem<'static, Blocking>, transfer_builder: impl for<'a> FnOnce( &'a mut SimpleMem2Mem<'static, Blocking>, ) -> SimpleMem2MemTransfer<'a, 'static, Blocking>, ) -> Self { let inner = ReceivingTransferInnerBuilder { // Safety: // These peripherals are not used until `mem2mem` is dropped. // This is ensured by making it a private field. peripheral: unsafe { DmaPeripheralWithChannel { peripheral: peripheral.peripheral.clone_unchecked(), channel: peripheral.channel.clone_unchecked(), } }, mem2mem: (mem2mem_builder)(peripheral), transfer_builder: move |mem2mem| Some((transfer_builder)(mem2mem)), } .build(); Self(inner) } pub fn with_transfer_mut( &mut self, callback: impl FnOnce(&mut Option>) -> R, ) -> R { self.0.with_transfer_mut(callback) } pub fn into_peripheral(self) -> DmaPeripheralWithChannel<'static> { self.0.into_heads().peripheral } } } use transfer::*; pub struct Swapchain { pub framebuffers: [&'static mut [u8]; 2], } impl Swapchain { pub fn into_reader_writer(self) -> (SwapchainReader, SwapchainWriter) { assert_eq!( self.framebuffers[0].len(), self.framebuffers[1].len(), "framebuffers in a swapchain must have an equal length" ); let reader_index = Arc::new(AtomicBool::new(true)); ( SwapchainReader { framebuffers_rw: [ self.framebuffers[0] as *const [u8], self.framebuffers[1] as *const [u8], ], reader_index: reader_index.clone(), }, SwapchainWriter { framebuffers_wr: [ self.framebuffers[1] as *mut [u8], self.framebuffers[0] as *mut [u8], ], reader_index, }, ) } } // TODO: Don't need to store the framebuffer length twice. Use `*const u8` instead, and store length separately. pub struct SwapchainReader { /// These are in the opposite order to `SwapchainWriter`'s framebuffers. framebuffers_rw: [*const [u8]; 2], reader_index: Arc, } unsafe impl Send for SwapchainReader {} impl SwapchainReader { fn len(&self) -> usize { self.framebuffers_rw[0].len() } fn load_read_index(&self) -> usize { self.reader_index.load(atomic::Ordering::SeqCst) as usize } fn get_latest_framebuffer(&self) -> &[u8] { unsafe { &*self.framebuffers_rw[self.load_read_index()] } } } // TODO: Don't need to store the framebuffer length twice. Use `*mut u8` instead, and store length separately. pub struct SwapchainWriter { /// These are in the opposite order to `SwapchainReader`'s framebuffers. framebuffers_wr: [*mut [u8]; 2], reader_index: Arc, } unsafe impl Send for SwapchainWriter {} impl SwapchainWriter { pub fn len(&self) -> usize { self.framebuffers_wr[0].len() } pub fn write(&mut self) -> SwapchainWriteGuard<'_> { let framebuffer_ptr = self.framebuffers_wr[self.reader_index.load(atomic::Ordering::SeqCst) as usize]; SwapchainWriteGuard { framebuffer: unsafe { &mut *framebuffer_ptr }, reader_index: &self.reader_index, } } } pub struct SwapchainWriteGuard<'a> { framebuffer: &'a mut [u8], reader_index: &'a AtomicBool, } impl Drop for SwapchainWriteGuard<'_> { fn drop(&mut self) { self.reader_index.fetch_xor(true, atomic::Ordering::SeqCst); } } impl<'a> Deref for SwapchainWriteGuard<'a> { type Target = [u8]; fn deref(&self) -> &Self::Target { self.framebuffer } } impl<'a> DerefMut for SwapchainWriteGuard<'a> { fn deref_mut(&mut self) -> &mut Self::Target { self.framebuffer } } pub struct DmaBounce { // TODO: Make these generic. // These currently cannot be generic, because they lack a `reborrow` method. peripheral_src: Option>, // This can also be more generic, see `DmaEligible` in `Mem2Mem::new`. peripheral_dst: Option>, // TODO: Combine with peripheral_dst using an enum? transfer_dst: Option>, // TODO: Consider having a separate burst config for the two transfers. burst_config: BurstConfig, cyclic: bool, /// The size of each window. window_size: usize, /// The number of windows. windows_len: usize, swapchain_src: SwapchainReader, // Two buffers of size `window_size`, // one of which is being written to, while the other is being read from. bounce_buffer_dst: &'static mut [u8], bounce_buffer_src: &'static mut [u8], // A descriptor list that spans a buffer of size `window_size`. // The buffer pointers need to be updated before each transmission to point to the correct window in the source buffer `src_buffer`. src_descs: &'static mut [DmaDescriptor], // A descriptor list that spans a buffer of size `window_size`. // The buffer pointers need to be updated before each transmission to point to the correct bounce buffer. bounce_dst_descs: &'static mut [DmaDescriptor], // A cyclic descriptor list that spans the buffers `bounce_buffer_dst` and `bounce_buffer_src`. bounce_src_descs: &'static mut [DmaDescriptor], descriptors_per_window: usize, // The index of the next window about to be received into the destination bounce buffer. window_index_next: usize, frame_index_next: usize, receiving_transfer: Option, } unsafe impl Send for DmaBounce {} impl DmaBounce { /// * `allocator` - The allocator used to allocate the bounce buffers. /// * `channel` - The DMA channel used to transfer data from the source buffer to the bounce buffers. /// * `peripheral_src` - The peripheral to transfer data from the source buffer to the bounce buffers. /// * `peripheral_dst` - The peripheral to transfer data to, from the bounce buffers. /// * `buffer_src` - The source buffer, typically allocated in external memory. /// * `row_front_porch_bytes` - The number of arbitrary-valued bytes to be sent in front of each row to the destination peripheral. /// * `row_width_bytes` - The width of a row, in bytes. /// * `window_size_rows` - The size of a single bounce buffer, in rows. /// * `burst_config` - The burst config to use for memory transfers (both in and out). TODO: This could be split. /// * `cyclic` - Experimental! Whether to use a cyclic descriptor list for transfer from the bounce buffers to the destination peripheral. pub fn new( allocator: impl Allocator + Copy + 'static, channel: DMA_CH0<'static>, peripheral_src: AnySpi<'static>, peripheral_dst: Dpi<'static, Blocking>, swapchain_src: SwapchainReader, row_front_porch_bytes: usize, row_width_bytes: usize, window_size_rows: usize, burst_config: BurstConfig, cyclic: bool, ) -> Self { assert_eq!( cyclic, true, "acyclic outbound transmissions are not yet implemented" ); let window_size = row_width_bytes * window_size_rows; assert_eq!( swapchain_src.len() % window_size, 0, "the size of a source buffer must be a multiple of the window size ({window_size} bytes), but it is {len} bytes large", len = swapchain_src.len() ); // Conservative alignment. Maxiumum of the cartesian product of [tx, rx] × [internal, external]. let alignment = burst_config.min_compatible_alignment(); for &swapchain_ptr in &swapchain_src.framebuffers_rw { assert_eq!( unsafe { &*swapchain_ptr }.as_ptr() as usize % alignment, 0, "the source buffer must be sufficiently aligned to {alignment} bytes for the burst config", ); } assert_eq!( row_width_bytes % alignment, 0, "the size of a row in bytes must be sufficiently aligned to {alignment} bytes for the burst config", ); assert_eq!( row_front_porch_bytes % alignment, 0, "the size of a row's front porch in bytes must be sufficiently aligned to {alignment} bytes for the burst config", ); // We need to make the destination peripheral read the front porch data from somewhere, // and that somewhere is currently the bounce buffer. // Therefore the front porch must be in bounds. assert!( row_front_porch_bytes <= window_size, "front porch too large" ); let windows_len = swapchain_src.len() / window_size; // TODO: Figure out a way to avoid `leak`ing memory. // We probably want to store the `Box`es and then unsafely extend the lifetime at sites of usage. let bounce_buffer_dst = Box::leak(allocate_dma_buffer_in(window_size, burst_config, allocator)); let bounce_buffer_src = Box::leak(allocate_dma_buffer_in(window_size, burst_config, allocator)); let src_descs = Self::linear_descriptors_for_buffer(window_size, burst_config, |desc| { desc.reset_for_tx(desc.next.is_null()); // Length for TX buffers must be set in software. // In RX buffers, it is set by hardware. desc.set_length(desc.size()); }); let bounce_dst_descs = Self::linear_descriptors_for_buffer(window_size, burst_config, |_| {}); let (bounce_src_descs, descriptors_per_window) = Self::bounce_descriptors_for_buffer( windows_len, row_front_porch_bytes, row_width_bytes, window_size_rows, unsafe { ( &mut *(bounce_buffer_dst as *mut _), &mut *(bounce_buffer_src as *mut _), ) }, burst_config, cyclic, ); Self { peripheral_src: Some(DmaPeripheralWithChannel { channel, peripheral: peripheral_src, }), peripheral_dst: Some(peripheral_dst), transfer_dst: None, burst_config, cyclic, window_size, windows_len, swapchain_src, bounce_buffer_dst, bounce_buffer_src, src_descs, bounce_dst_descs, bounce_src_descs, descriptors_per_window, window_index_next: 0, frame_index_next: 0, receiving_transfer: None, } } fn linear_descriptors_for_buffer( buffer_len: usize, burst_config: BurstConfig, mut setup_desc: impl FnMut(&mut DmaDescriptor), ) -> &'static mut [DmaDescriptor] { let max_chunk_size = burst_config.max_compatible_chunk_size(); let descriptors_len = dma::descriptor_count(buffer_len, max_chunk_size, false); // TODO: This leaks memory. Ensure it's only called during setup. let descriptors = Box::leak(vec![DmaDescriptor::EMPTY; descriptors_len].into_boxed_slice()); // Link up the descriptors. let mut next = core::ptr::null_mut(); for desc in descriptors.iter_mut().rev() { desc.next = next; next = desc; } // Prepare each descriptor's buffer size. let mut descriptors_it = descriptors.iter_mut(); let mut remaining_len = buffer_len; while remaining_len > 0 { let chunk_size = core::cmp::min(max_chunk_size, remaining_len); let desc = descriptors_it.next().unwrap(); desc.set_size(chunk_size); (setup_desc)(desc); remaining_len -= chunk_size; } descriptors } fn prepare_descriptors_window( bounce_buffer: &mut [u8], descriptors_window: &mut [DmaDescriptor], row_front_porch_bytes: usize, row_width_bytes: usize, window_size_rows: usize, max_chunk_size: usize, descriptors_per_row: usize, descriptors_per_row_front_porch: usize, ) { for (row_index_in_window, descriptors_row) in descriptors_window .chunks_mut(descriptors_per_row) .enumerate() { // let row_index = row_index_in_window + window_index * window_size_rows; let (descriptors_row_front_porch, descriptors_row_stored) = descriptors_row.split_at_mut(descriptors_per_row_front_porch); // Prepare front porch descriptors. { let mut descriptors_it = descriptors_row_front_porch.iter_mut(); let mut remaining_front_porch = row_front_porch_bytes; while remaining_front_porch > 0 { let desc = descriptors_it.next().unwrap(); let chunk_size = core::cmp::min(max_chunk_size, remaining_front_porch); remaining_front_porch -= chunk_size; // Just make it point at a bounce buffer. // It is guaranteed to have enough bytes by `DmaBounce::new`. desc.buffer = bounce_buffer.as_mut_ptr(); desc.set_size(chunk_size); desc.set_length(chunk_size); desc.reset_for_tx(false); } assert!( descriptors_it.next().is_none(), "front porch descriptors must be used up" ); assert_eq!( descriptors_row_front_porch .iter() .map(|desc| desc.size()) .sum::(), row_front_porch_bytes ); } // Prepare window descriptors. { let mut remaining_bounce_buffer = &mut bounce_buffer[row_index_in_window * row_width_bytes..][..row_width_bytes]; // if remaining_bounce_buffer.len() > row_width_bytes { // remaining_bounce_buffer = &mut remaining_bounce_buffer[..row_width_bytes]; // } for desc in &mut *descriptors_row_stored { let chunk_size = core::cmp::min(max_chunk_size, remaining_bounce_buffer.len()); desc.buffer = remaining_bounce_buffer.as_mut_ptr(); remaining_bounce_buffer = &mut remaining_bounce_buffer[chunk_size..]; desc.set_size(chunk_size); desc.set_length(chunk_size); desc.reset_for_tx(false); } assert!( remaining_bounce_buffer.is_empty(), "bounce buffer must be used up" ); assert_eq!( descriptors_row_stored .iter() .map(|desc| desc.size()) .sum::(), row_width_bytes ); } } // Set EOF bit on the last descriptor of the window, to signal // that the bounce buffer is done being read from. if let Some(last_desc) = descriptors_window.last_mut() { last_desc.reset_for_tx(true); } assert_eq!( descriptors_window .iter() .map(|desc| desc.size()) .sum::(), window_size_rows * (row_front_porch_bytes + row_width_bytes) ); } fn bounce_descriptors_for_buffer( windows_len: usize, row_front_porch_bytes: usize, row_width_bytes: usize, window_size_rows: usize, bounce_buffers: (&'static mut [u8], &'static mut [u8]), burst_config: BurstConfig, cyclic: bool, ) -> (&'static mut [DmaDescriptor], usize) { assert_eq!( bounce_buffers.0.len(), bounce_buffers.1.len(), "bounce buffers must be equal in size" ); // If an odd number of windows were needed, two descriptor lists would be needed, assert_eq!(windows_len % 2, 0, "the number of windows must be even"); let buffer_len = bounce_buffers.0.len(); assert_eq!( buffer_len, row_width_bytes * window_size_rows, "the provided bounce buffers have an invalid size" ); // Implementation note: // A cyclic descriptor could consist of just a set of descriptors per window, // so two sets in total, because there are two bounce buffers. // However, we can also access the pointer of the EOF descriptor within the // EOF interrupt handler, which lets us compute which descriptor generated that // interrupt. // This is useful in the case when an interrupt is missed. Then the number of interrupts // handled doesn't correspond to the number of windows sent to the destination peripheral. // In that case, the number of windows sent can be computed from the address of the descriptor. let max_chunk_size = burst_config.max_compatible_chunk_size(); let descriptors_per_row_front_porch = dma::descriptor_count(row_front_porch_bytes, max_chunk_size, false); let descriptors_per_row_stored = dma::descriptor_count(row_width_bytes, max_chunk_size, false); let descriptors_per_row = descriptors_per_row_stored + descriptors_per_row_front_porch; let descriptors_per_window = window_size_rows * descriptors_per_row; let descriptors_per_frame = descriptors_per_window * windows_len; let descriptors_frame = Box::leak(vec![DmaDescriptor::EMPTY; descriptors_per_frame].into_boxed_slice()); // Link up the descriptors. let mut next = if cyclic { descriptors_frame.first_mut().unwrap() as *mut _ } else { core::ptr::null_mut() }; for desc in descriptors_frame.iter_mut().rev() { desc.next = next; next = desc; } // Prepare each descriptor's buffer size. let bounce_buffers = [bounce_buffers.0, bounce_buffers.1]; for (window_index, descriptors_window) in descriptors_frame .chunks_mut(descriptors_per_window) .enumerate() { let bounce_buffer_index = window_index % 2; let bounce_buffer = &mut *bounce_buffers[bounce_buffer_index]; Self::prepare_descriptors_window( bounce_buffer, descriptors_window, row_front_porch_bytes, row_width_bytes, window_size_rows, max_chunk_size, descriptors_per_row, descriptors_per_row_front_porch, ); } assert_eq!( descriptors_frame .iter() .map(|desc| desc.size()) .sum::(), windows_len * window_size_rows * (row_front_porch_bytes + row_width_bytes) ); (descriptors_frame, descriptors_per_window) } /// Safety: /// TX descriptors require read access to the buffer. /// RX descriptors require write access to the buffer. unsafe fn linear_descriptors_prepare( descriptors: &mut [DmaDescriptor], mut buffer: Option<&[u8]>, mut setup_desc: impl FnMut(&mut DmaDescriptor), ) { for descriptor in descriptors.iter_mut() { if let Some(inner_buffer) = buffer { descriptor.buffer = inner_buffer.as_ptr() as *mut u8; buffer = Some(&inner_buffer[descriptor.size()..]); } (setup_desc)(descriptor); } if let Some(buffer) = buffer { assert!( buffer.is_empty(), "a buffer of an incompatible length was assigned to a descriptor set" ); } } fn enable_interrupts() { // Enable interrupts for the peripheral, pt. 1. interrupt::enable( INTERRUPT_OUTBOUND, dma_outbound_interrupt_handler.priority(), ) .unwrap(); // Bind the interrupt handler. unsafe { interrupt::bind_interrupt(INTERRUPT_OUTBOUND, dma_outbound_interrupt_handler.handler()); } // Enable interrupts for the peripheral, pt. 2. DMA::regs() .ch(DMA_CHANNEL_OUTBOUND) .out_int() .ena() .modify(|_, w| w.out_eof().bit(true)); } /// Receive a window of bytes into the current dst bounce buffer. /// Finally, swaps the bounce buffers. /// /// # Safety: /// TODO unsafe fn receive_window_start(&mut self) -> ReceivingTransfer { // Descriptors are initialized by `DmaTxBuf::new`. let buffer_src_window = &self.swapchain_src.get_latest_framebuffer() [self.window_index_next * self.window_size..][..self.window_size]; unsafe { Self::linear_descriptors_prepare(self.src_descs, Some(buffer_src_window), |_desc| { // No need to call `DmaDescriptor::reset_for_tx`, because // 1. we don't rely on the ownership flag; // 2. the EOF flag is already set during the construction of this buffer. }); // TODO: Precompute a descriptor list for each buffer, then use `None` instead of `Some(&mut *self.bounce_buffer_dst)`. Self::linear_descriptors_prepare( self.bounce_dst_descs, Some(self.bounce_buffer_dst), |desc| { desc.reset_for_rx(); }, ); } let peripheral = self .peripheral_src .take() .expect("the source DMA peripheral should be available"); ReceivingTransfer::new( peripheral, |peripheral| { // Extend the lifetime to 'static because it is required by Mem2Mem. // // Safety: // Pointees are done being used by the driver before this scope ends, // this is because we `SimpleMem2MemTransfer::wait()` on the transfer to finish. let bounce_dst_descs: &'static mut [DmaDescriptor] = unsafe { &mut *(self.bounce_dst_descs as *mut _) }; let src_descs: &'static mut [DmaDescriptor] = unsafe { &mut *(self.src_descs as *mut _) }; Mem2Mem::new(peripheral.channel, peripheral.peripheral) .with_descriptors(bounce_dst_descs, src_descs, self.burst_config) .unwrap() }, |mem2mem| { mem2mem .start_transfer(self.bounce_buffer_dst, buffer_src_window) .unwrap() }, ) } fn receive_window_wait( &mut self, mut receiving_transfer: ReceivingTransfer, increase_window_counter: bool, ) { receiving_transfer.with_transfer_mut(|receiving_transfer| { // TODO: Async let receiving_transfer = receiving_transfer .take() .expect("no ongoing inner transfer to a bounce buffer present"); if !receiving_transfer.is_done() { #[cfg(feature = "log")] log::debug!("Inbound transfer not yet finished, waiting via spinlock..."); receiving_transfer.wait().unwrap(); #[cfg(feature = "log")] log::debug!("Inbound transfer not finished."); } }); if increase_window_counter { self.increase_window_counter(1); } let existing_peripheral_src = self .peripheral_src .replace(receiving_transfer.into_peripheral()); assert!( existing_peripheral_src.is_none(), "no idle source DMA peripheral should be present, it should be receiving data instead" ); } fn receive_window(&mut self, increase_window_counter: bool) { let receiving_transfer = unsafe { self.receive_window_start() }; self.receive_window_wait(receiving_transfer, increase_window_counter); } fn increase_window_counter(&mut self, windows: isize) { // TODO: Updating `self.frame_index_next` without calling this function is error prone. // Index into an array with `self.window_index_next % 2` instead. if windows.rem_euclid(2) == 1 { core::mem::swap(&mut self.bounce_buffer_dst, &mut self.bounce_buffer_src); } let window_index_next = self.window_index_next as isize + windows; self.frame_index_next = (self.frame_index_next as isize + window_index_next / self.windows_len as isize) as usize; self.window_index_next = window_index_next.rem_euclid(self.windows_len as isize) as usize; } pub async fn launch_interrupt_driven_task(mut self) -> RunningDmaBounceHandle { Self::enable_interrupts(); // Reset the existing transfer left over since the outbound transfer was paused. if let Some(receiving_transfer) = self.receiving_transfer.take() { self.receive_window_wait(receiving_transfer, false); } // Reset window index. if self.window_index_next != 0 { self.increase_window_counter((self.windows_len - self.window_index_next) as isize); } // Fully receive the first windows, so that the outbound transfer can read valid data. self.receive_window(true); #[cfg(feature = "log")] log::debug!("DmaBounce: Starting outbound transfer."); let dma_tx_buffer = self.get_dma_tx_buffer(); let transfer = self .peripheral_dst .take() .unwrap() .send(self.cyclic /* Send perpetually */, dma_tx_buffer) .unwrap_or_else(|(error, _, _)| { panic!("failed to begin the transmission of the first frame: {error:?}"); }); self.transfer_dst = Some(transfer); // Start receiving the second window. self.receiving_transfer = Some(unsafe { self.receive_window_start() }); RunningDmaBounceHandle::new(self).await } fn get_dma_tx_buffer(&mut self) -> DmaTxBounceBuf { DmaTxBounceBuf { preparation: dma::Preparation { start: self.bounce_src_descs.first_mut().unwrap(), direction: dma::TransferDirection::Out, accesses_psram: false, burst_transfer: self.burst_config, // We don't care about ownership. // Just yeet whatever the descriptors point to to the destination peripheral. check_owner: Some(false), auto_write_back: false, }, } } } pub struct DmaTxBounceBuf { preparation: dma::Preparation, } unsafe impl DmaTxBuffer for DmaTxBounceBuf { type View = Self; type Final = Self; fn prepare(&mut self) -> dma::Preparation { dma::Preparation { start: self.preparation.start, direction: self.preparation.direction, accesses_psram: self.preparation.accesses_psram, burst_transfer: self.preparation.burst_transfer, check_owner: self.preparation.check_owner, auto_write_back: self.preparation.auto_write_back, } } fn into_view(self) -> Self::View { self } fn from_view(view: Self::View) -> Self::Final { view } } #[must_use = "the `Drop` implementation uses a spinlock, which can result in a deadlock; use `RunningDmaBounceHandle::stop` instead of letting it be dropped"] pub struct RunningDmaBounceHandle { perform_drop: bool, // Prevent construction. _marker: (), } impl RunningDmaBounceHandle { async fn new(dma_bounce: DmaBounce) -> Self { let dma_state = RUNNING_DMA_BOUNCE.lock().await; *dma_state.borrow_mut() = Some(dma_bounce); Self { perform_drop: true, _marker: (), } } pub async fn stop(mut self) -> DmaBounce { #[cfg(feature = "log")] log::debug!("DmaBounce: Stopping outbound transfer due to `stop`."); let mut dma_bounce = RUNNING_DMA_BOUNCE .lock() .await .borrow_mut() .take() .expect("an instance of a running `DmaBounce` should be available"); let transfer_dst = dma_bounce .transfer_dst .take() .expect("an instance of a transfer to the destination peripheral should be available"); let (peripheral_dst, _dma_tx_buffer) = transfer_dst.stop(); let previous_peripheral_dst = dma_bounce.peripheral_dst.replace(peripheral_dst); self.perform_drop = false; assert!( previous_peripheral_dst.is_none(), "there should be no existing destination peripheral in `DmaBounce`" ); dma_bounce } } impl Drop for RunningDmaBounceHandle { fn drop(&mut self) { if !self.perform_drop { return; } #[cfg(feature = "log")] log::debug!("DmaBounce: Stopping outbound transfer due to `drop`."); let dma_bounce = loop { if let Ok(dma_bounce) = RUNNING_DMA_BOUNCE.try_lock() { break dma_bounce; } }; let mut dma_bounce = dma_bounce .borrow_mut() .take() .expect("an instance of a running `DmaBounce` should be available"); let transfer_dst = dma_bounce .transfer_dst .take() .expect("an instance of a transfer to the destination peripheral should be available"); let (peripheral_dst, _dma_tx_buffer) = transfer_dst.stop(); dma_bounce.peripheral_dst = Some(peripheral_dst); drop(dma_bounce); } } impl Debug for RunningDmaBounceHandle { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("RunningDmaBounceHandle") .finish_non_exhaustive() } } #[handler(priority = Priority::Priority3)] #[ram] // Improves performance. fn dma_outbound_interrupt_handler() { let interrupt = DMA::regs().ch(DMA_CHANNEL_OUTBOUND).out_int(); let bounce_buffer_sent = interrupt.st().read().out_eof().bit_is_set(); if !bounce_buffer_sent { // This should never happen. #[cfg(feature = "log")] log::warn!("DMA interrupt handler invoked without `OUT_EOF` bit having been set."); return; } // Clear the bit by writing 1 to the clear bits. interrupt.clr().write(|w| w.out_eof().bit(true)); let Ok(dma_state) = RUNNING_DMA_BOUNCE.try_lock() else { // If we can't acquire a lock guard, just give up. #[cfg(feature = "log")] log::trace!("Failed to lock `RUNNING_DMA_BOUNCE`."); return; }; let mut dma_state = dma_state.borrow_mut(); let Some(dma_state) = dma_state.as_mut() else { // The outbound transmission was stopped. #[cfg(feature = "log")] log::trace!("`RUNNING_DMA_BOUNCE` is empty."); return; }; // The descriptor of the buffer with an EOF flag that just finished being sent. let descriptor_ptr = DMA::regs() .ch(DMA_CHANNEL_OUTBOUND) .out_eof_des_addr() .read() .out_eof_des_addr() .bits() as *const DmaDescriptor; // This is the index of the window that just finished being transmitted to the destination peripheral. let window_sent_index = unsafe { descriptor_ptr.offset_from_unsigned(dma_state.bounce_src_descs.as_ptr()) } / dma_state.descriptors_per_window; // The next window to be sent is `(window_sent_index + 1) % dma_state.windows_len`. // That is not the window we want to buffer, because the transmissions would race. // We instead want to buffer the next window: let window_index_next = (window_sent_index + 2) % dma_state.windows_len; // Swap bounce buffers. if (dma_state.windows_len + window_index_next - dma_state.window_index_next) % 2 == 1 { core::mem::swap( &mut dma_state.bounce_buffer_dst, &mut dma_state.bounce_buffer_src, ); } dma_state.window_index_next = window_index_next; let receiving_transfer = dma_state .receiving_transfer .take() .expect("no ongoing transfer to a bounce buffer present"); dma_state.receive_window_wait( receiving_transfer, // `window_index_next` is already updated above. false, ); // If there is any ongoing transfer, cancel it and start a new one. dma_state.receiving_transfer = Some(unsafe { dma_state.receive_window_start() }); } pub fn allocate_dma_buffer_in( len: usize, burst_config: BurstConfig, alloc: A, ) -> Box<[u8], A> { // Conservative alignment. Maxiumum of the cartesian product of [tx, rx] × [internal, external]. let alignment = burst_config.min_compatible_alignment(); assert_eq!( len % alignment, 0, "the size of a DMA buffer must be a multiple of {alignment} bytes, but it is {len} bytes large" ); // ⚠️ Note: For chips that support DMA to/from PSRAM (ESP32-S3) DMA transfers to/from PSRAM // have extra alignment requirements. The address and size of the buffer pointed to by each // descriptor must be a multiple of the cache line (block) size. This is 32 bytes on ESP32-S3. // That is ensured by the `assert_eq` preceding this block. unsafe { let raw = alloc .allocate_zeroed(Layout::from_size_align(len, alignment).unwrap()) .expect("failed to allocate a DMA buffer"); Box::from_raw_in(raw.as_ptr(), alloc) } }