Feature/NVMulator

documentation/tapasco-NVMulator.md

@@ -0,0 +1,51 @@
+NVMulator (Non-Volatile Memory Emulator)
+===================
+An open-source easy-to-use hardware emulation module that can be seamlessly inserted between the PE of the FPGA processing elements on the FPGA and a conventional DRAM-based memory system. This feature has been proposed in [[Tamimi2023]](#paper) into TaPaSCo.
+
+Table of Contents
+-----------------
+  1. [Usage](#usage)
+  2. [Compatibility](#compatibility)
+
+Usage <a name="usage"/>
+-----
+To use the NVMulator, please follow the steps outlined below:
+
+### Composing a hardware design
+
+```
+tapasco compose [arraysum x 1] @ 100MHz -p AU280  --features 'NVMulator {enabled: true}'
+```
+
+
+### API example 
+To set the desired latency, please use ```nvMulator(READ_DELAY, WRITE_DELAY, NVM_MODE)``` function call in the API of the TaPaSCo as follow:
+
+```
+#define READ_DELAY  100  // in clock cycles
+#define WRITE_DELAY 200  // in clock cycles
+#define NVM_MODE    1    // enable NVM emulation mode
+
+int main() {
+    tapasco::Tapasco tapasco;
+
+    tapasco.nvMulator(READ_DELAY, WRITE_DELAY, NVM_MODE);
+
+    auto job = tapasco.launch(PE_ID, reg1, reg2, ...);
+    job();
+
+    return 0;
+}
+```
+
+
+Compatibility <a name="compatibility"/>
+-------------
+
+NVMulator only supports the Alveo U280 platform that exploits DDR memory. It is not compatible
+with PE-local memories and HBM. Compatibility with other TaPaSCo features is not guaranteed.
+
+References
+----------
+[Tamimi2023] Tamimi, S., Bernhardt, B., Florian, S., Petrov, I., and Koch, A. (2023). NVMulator: A Configurable Open-Source Non-Volatile Memory Emulator for FPGAs. In *Applied Reconfigurable Computing. Architectures, Tools, and Applications (ARC)*.<a name="paper"/>
+

runtime/libtapasco/src/device.rs

@@ -348,8 +348,21 @@ impl Device {
         let zynqmp_vfio_mode = true;
         let mut is_pcie = false;
         let mut svm_in_use = false;
+        let mut nv_in_use = false;
+        let mut nv_offset = 0;
         if name == "pcie" {
-
+
+            // check whether NVMulator is in use
+            for comp in &s.platform {
+                if comp.name == "PLATFORM_COMPONENT_NVMULATOR" {
+                    nv_in_use = true;
+                    nv_offset = comp.offset;          
+                    if nv_offset == 0 {
+                        trace!("ERROR: NVMulator offset is zero!");
+                    }
+                    trace!("Found the NVMulator module in {}", nv_offset);
+                }
+            }
             // check whether SVM is in use
             for comp in &s.platform {
                 if comp.name == "PLATFORM_COMPONENT_MMU" {
@@ -406,6 +419,7 @@ impl Device {
                             settings
                                 .get::<usize>("dma.write_buffers")
                                 .context(ConfigSnafu)?,
+                                nv_offset as usize,
                         )
                             .context(DMASnafu)?,
                     ),

runtime/libtapasco/src/dma.rs

@@ -84,6 +84,7 @@ type Result<T, E = Error> = std::result::Result<T, E>;
 pub trait DMAControl: Debug {
     fn copy_to(&self, data: &[u8], ptr: DeviceAddress) -> Result<()>;
     fn copy_from(&self, ptr: DeviceAddress, data: &mut [u8]) -> Result<()>;
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()>;
 }
 
 #[derive(Debug, Getters)]
@@ -144,6 +145,11 @@ impl DMAControl for DriverDMA {
         };
         Ok(())
     }
+
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()> {
+        trace!("configuring NVMulator");
+        Ok(())
+    }
 }
 
 #[derive(Debug, Getters)]
@@ -186,6 +192,11 @@ impl DMAControl for VfioDMA {
         // nothing to copy, 'data' is same buffer that PE operated on
         Ok(())
     }
+
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()> {
+        trace!("configuring NVMulator");
+        Ok(())
+    }
 }
 
 /// Use the CPU to transfer data
@@ -263,6 +274,11 @@ impl DMAControl for DirectDMA {
 
         Ok(())
     }
+
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()> {
+        trace!("configuring NVMulator");
+        Ok(())
+    }
 }
 
 /// DMA implementation for SVM support
@@ -316,4 +332,9 @@ impl DMAControl for SVMDMA {
         trace!("Migration to host memory complete.");
         Ok(())
     }
+
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()> {
+        trace!("configuring NVMulator");
+        Ok(())
+    }
 }

runtime/libtapasco/src/dma_user_space.rs

@@ -87,6 +87,7 @@ pub struct UserSpaceDMA {
     write_int_cntr: AtomicU64,
     read_cntr: AtomicU64,
     read_int_cntr: AtomicU64,
+    emulator_offset:usize,
 }
 
 impl UserSpaceDMA {
@@ -100,6 +101,7 @@ impl UserSpaceDMA {
         read_num_buf: usize,
         write_buf_size: usize,
         write_num_buf: usize,
+        emulator_offset: usize,
     ) -> Result<Self> {
         trace!(
             "Using setting: Read {} x {}B, Write {} x {}B",
@@ -181,6 +183,7 @@ impl UserSpaceDMA {
             write_int_cntr: AtomicU64::new(0),
             read_int_cntr: AtomicU64::new(0),
             read_cntr: AtomicU64::new(0),
+            emulator_offset: emulator_offset,
         })
     }
 
@@ -298,6 +301,32 @@ impl UserSpaceDMA {
         used_buffers.retain(|(x, _y, _z, _a)| *x >= read_int_cntr_used);
         Ok(())
     }
+
+    fn configure_nvMulator(
+        &self,
+        dma_engine_memory: &MutexGuard<Arc<MmapMut>>,
+        read_delay: u64,
+        write_delay: u64,
+        mode: u64,
+    ) {
+        let mut offset = (self.emulator_offset as usize + 0x30) as isize;
+        unsafe {
+            let ptr = dma_engine_memory.as_ptr().offset(offset);
+            write_volatile(ptr as *mut u64, read_delay);
+        };
+
+        offset = (self.emulator_offset as usize + 0x40) as isize;
+        unsafe {
+            let ptr = dma_engine_memory.as_ptr().offset(offset);
+            write_volatile(ptr as *mut u64, write_delay);
+        };
+
+        offset = (self.emulator_offset as usize + 0x20) as isize;
+        unsafe {
+            let ptr = dma_engine_memory.as_ptr().offset(offset);
+            write_volatile(ptr as *mut u64, mode);
+        };
+    }
 }
 
 impl DMAControl for UserSpaceDMA {
@@ -437,4 +466,21 @@ impl DMAControl for UserSpaceDMA {
 
         Ok(())
     }
+
+    fn nvMulator(&self, read_delay: u64, write_delay: u64, mode: u64) -> Result<()> {
+        trace!("configuring NVNulator with (read: {}, write: {}) in mode {}",
+        read_delay,
+        write_delay,
+        mode);
+
+        let dma_engine_memory = self.memory.lock()?;
+        self.configure_nvMulator(
+            &dma_engine_memory,
+            read_delay,
+            write_delay,
+            mode,
+        );
+
+        Ok(())
+    }
 }

runtime/libtapasco/src/ffi.rs

@@ -800,6 +800,22 @@ pub unsafe extern "C" fn tapasco_memory_copy_from(
     }
 }
 
+/// # Safety
+/// TODO
+#[no_mangle]
+pub unsafe extern "C" fn tapasco_nvMulator(
+    mem: *mut TapascoOffchipMemory,
+    read_delay: usize,
+    write_delay: usize,
+    mode: usize,
+) -> isize {
+
+    let tl = &mut *mem;
+    tl.dma().nvMulator(read_delay.try_into().unwrap(), write_delay.try_into().unwrap(), mode.try_into().unwrap()).context(DMASnafu);
+
+    return 10;
+}
+
 /// # Safety
 /// TODO
 #[no_mangle]

runtime/libtapasco/src/tapasco.hpp

@@ -332,6 +332,15 @@ class TapascoMemory {
     return this->copy_from(src, dst, len);
   }
 
+  int nvMulator(uint64_t read_delay, uint64_t write_delay, uint64_t mode) {
+    std::ostringstream stringStream;
+    if (tapasco_nvMulator(mem, read_delay, write_delay, mode) == -1) {
+      handle_error();
+      return -1;
+    }
+    return 0;
+  }
+
 private:
   TapascoOffchipMemory *mem;
 };
@@ -593,6 +602,17 @@ struct Tapasco {
     return this->default_memory_internal.copy_from(src, dst, len);
   }
 
+/**
+   * Configuring NVMulator. 
+   * @param read_delay delay for read channel (in clock cycles)
+   * @param write_delay delay for write channel (in clock cycles)
+   * @param mode enable NVM emulation mode
+   * @return TAPASCO_SUCCESS if copy was successful, an error code otherwise
+   **/
+  int nvMulator(size_t read_delay, size_t write_delay, size_t mode) {
+    return this->default_memory_internal.nvMulator(read_delay, write_delay, mode);
+  }
+
   /**
    * Returns the number of PEs of kernel k_id in the currently loaded
    *bitstream.