DUNE-DAQ
DUNE Trigger and Data Acquisition software
Loading...
Searching...
No Matches
dunedaq::flxlibs::CardWrapper Class Reference

#include <CardWrapper.hpp>

Collaboration diagram for dunedaq::flxlibs::CardWrapper:
[legend]

Public Member Functions

 CardWrapper (const appmodel::FelixInterface *cfg, std::vector< unsigned int >)
 CardWrapper Constructor.
 
 ~CardWrapper ()
 
 CardWrapper (const CardWrapper &)=delete
 CardWrapper is not copy-constructible.
 
CardWrapperoperator= (const CardWrapper &)=delete
 CardWrapper is not copy-assignable.
 
 CardWrapper (CardWrapper &&)=delete
 CardWrapper is not move-constructible.
 
CardWrapperoperator= (CardWrapper &&)=delete
 CardWrapper is not move-assignable.
 
void configure ()
 
void start ()
 
void stop ()
 
void set_running (bool should_run)
 
void graceful_stop ()
 
void set_block_addr_handler (std::function< void(uint64_t)> &handle)
 

Private Types

using UniqueFlxCard = std::unique_ptr<FlxCard>
 

Private Member Functions

void open_card ()
 
void close_card ()
 
int allocate_CMEM (uint8_t numa, u_long bsize, u_long *paddr, u_long *vaddr)
 
void init_DMA ()
 
void start_DMA ()
 
void stop_DMA ()
 
uint64_t bytes_available ()
 
void read_current_address ()
 
void process_DMA ()
 

Private Attributes

std::atomic< bool > m_run_marker
 
bool m_configured { false }
 
uint8_t m_card_id
 
uint8_t m_logical_unit
 
std::string m_card_id_str
 
uint8_t m_dma_id
 
size_t m_margin_blocks
 
size_t m_block_threshold
 
bool m_interrupt_mode
 
size_t m_poll_time
 
uint8_t m_numa_id
 
std::vector< unsigned int > m_links_enabled
 
std::string m_info_str
 
UniqueFlxCard m_flx_card
 
std::mutex m_card_mutex
 
std::size_t m_dma_memory_size
 
int m_cmem_handle
 
uint64_t m_virt_addr
 
uint64_t m_phys_addr
 
uint64_t m_current_addr
 
unsigned m_read_index
 
u_long m_destination
 
std::atomic< bool > m_run_lock
 
utilities::ReusableThread m_dma_processor
 
std::function< void(uint64_t)> m_handle_block_addr
 
bool m_block_addr_handler_available { false }
 

Static Private Attributes

static constexpr size_t m_max_links_per_card = 6
 
static constexpr size_t m_block_size = 4096
 
static constexpr size_t m_dma_wraparound = FLX_DMA_WRAPAROUND
 
static const std::string m_dma_processor_name = "flx-dma"
 

Detailed Description

Definition at line 29 of file CardWrapper.hpp.

Member Typedef Documentation

◆ UniqueFlxCard

using dunedaq::flxlibs::CardWrapper::UniqueFlxCard = std::unique_ptr<FlxCard>
private

Definition at line 94 of file CardWrapper.hpp.

Constructor & Destructor Documentation

◆ CardWrapper() [1/3]

dunedaq::flxlibs::CardWrapper::CardWrapper ( const appmodel::FelixInterface * cfg,
std::vector< unsigned int > enabled_links )

CardWrapper Constructor.

Parameters
nameInstance name for this CardWrapper instance

Definition at line 36 of file CardWrapper.cpp.

37 : m_run_marker{ false }
38 , m_card_id(cfg->get_card())
39 , m_logical_unit(cfg->get_slr())
40 , m_dma_id(cfg->get_dma_id())
41 , m_margin_blocks(cfg->get_dma_margin_blocks())
42 , m_block_threshold(cfg->get_dma_block_threshold())
43 , m_interrupt_mode(cfg->get_interrupt_mode())
44 , m_poll_time(cfg->get_poll_time())
45 , m_numa_id(cfg->get_numa_id())
46 , m_info_str("")
47 , m_run_lock{ false }
49 , m_handle_block_addr(nullptr)
50 , m_links_enabled(enabled_links)
51{
52 m_dma_memory_size = cfg->get_dma_memory_size_gb() * 1024 * 1024 * 1024UL;
53
54 std::ostringstream tnoss;
55 tnoss << m_dma_processor_name << "-" << std::to_string(m_card_id); // append physical card id
56 m_dma_processor.set_name(tnoss.str(), m_logical_unit); // set_name appends logical unit id
57
58 std::ostringstream cardoss;
59 cardoss << "[id:" << std::to_string(m_card_id) << " slr:" << std::to_string(m_logical_unit) << "]";
60 m_card_id_str = cardoss.str();
61
62 m_flx_card = std::make_unique<FlxCard>();
63 if (m_flx_card == nullptr) {
64 throw flxlibs::CardError(ERS_HERE, "Couldn't create FlxCard object.");
65 }
66}
#define ERS_HERE
utilities::ReusableThread m_dma_processor
std::function< void(uint64_t)> m_handle_block_addr
std::atomic< bool > m_run_lock
std::atomic< bool > m_run_marker
std::vector< unsigned int > m_links_enabled
static const std::string m_dma_processor_name
void set_name(const std::string &name, int tid)

◆ ~CardWrapper()

dunedaq::flxlibs::CardWrapper::~CardWrapper ( )

Definition at line 68 of file CardWrapper.cpp.

69{
70 TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << "CardWrapper destructor called. First stop check, then closing card.";
72 close_card();
73 TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << "CardWrapper destroyed.";
74}
#define TLVL_ENTER_EXIT_METHODS
#define TLOG_DEBUG(lvl,...)
Definition Logging.hpp:112

◆ CardWrapper() [2/3]

dunedaq::flxlibs::CardWrapper::CardWrapper ( const CardWrapper & )
delete

CardWrapper is not copy-constructible.

◆ CardWrapper() [3/3]

dunedaq::flxlibs::CardWrapper::CardWrapper ( CardWrapper && )
delete

CardWrapper is not move-constructible.

Member Function Documentation

◆ allocate_CMEM()

int dunedaq::flxlibs::CardWrapper::allocate_CMEM ( uint8_t numa,
u_long bsize,
u_long * paddr,
u_long * vaddr )
private

Definition at line 187 of file CardWrapper.cpp.

188{
189 TLOG_DEBUG(TLVL_WORK_STEPS) << "Allocating CMEM buffer " << m_card_id_str << " dma id:" << std::to_string(m_dma_id);
190 int handle;
191 unsigned ret = CMEM_Open(); // cmem_rcc.h
192 if (!ret) {
193 ret = CMEM_NumaSegmentAllocate(bsize, numa, const_cast<char*>(m_card_id_str.c_str()), &handle); // NUMA aware
194 // ret = CMEM_GFPBPASegmentAllocate(bsize, const_cast<char*>(m_card_id_str.c_str()), &handle); // non NUMA aware
195 }
196 if (!ret) {
197 ret = CMEM_SegmentPhysicalAddress(handle, paddr);
198 }
199 if (!ret) {
200 ret = CMEM_SegmentVirtualAddress(handle, vaddr);
201 }
202 if (ret) {
203 // rcc_error_print(stdout, ret);
204 m_card_mutex.lock();
205 m_flx_card->card_close();
206 m_card_mutex.unlock();
208 flxlibs::CardError(ERS_HERE,
209 "Not enough CMEM memory allocated or the application demands too much CMEM memory.\n"
210 "Fix the CMEM memory reservation in the driver or change the module's configuration."));
211 exit(EXIT_FAILURE);
212 }
213 return handle;
214}
void fatal(const Issue &issue)
Definition ers.hpp:88

◆ bytes_available()

uint64_t dunedaq::flxlibs::CardWrapper::bytes_available ( )
inlineprivate

◆ close_card()

void dunedaq::flxlibs::CardWrapper::close_card ( )
private

Definition at line 173 of file CardWrapper.cpp.

174{
175 TLOG_DEBUG(TLVL_WORK_STEPS) << "Closing FELIX card " << m_card_id_str;
176 try {
177 m_card_mutex.lock();
178 m_flx_card->card_close();
179 m_card_mutex.unlock();
180 } catch (FlxException& ex) {
181 ers::error(flxlibs::CardError(ERS_HERE, ex.what()));
182 exit(EXIT_FAILURE);
183 }
184}
void error(const Issue &issue)
Definition ers.hpp:81

◆ configure()

void dunedaq::flxlibs::CardWrapper::configure ( )

Definition at line 78 of file CardWrapper.cpp.

79{
80 if (m_configured) {
81 TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << "Card is already configured! Won't touch it.";
82 } else {
83 TLOG_DEBUG(TLVL_WORK_STEPS) << "Configuring CardWrapper of card " << m_card_id_str;
84 // Open card
85 open_card();
86 TLOG_DEBUG(TLVL_WORK_STEPS) << "Card[" << m_card_id_str << "] opened.";
87 // Allocate CMEM
89 TLOG_DEBUG(TLVL_WORK_STEPS) << "Card[" << m_card_id_str << "] CMEM memory allocated with "
90 << std::to_string(m_dma_memory_size) << " Bytes.";
91 // Stop currently running DMA
92 stop_DMA();
93 TLOG_DEBUG(TLVL_WORK_STEPS) << "Card[" << m_card_id_str << "] DMA interactions force stopped.";
94 // Init DMA between software and card
95 init_DMA();
96 TLOG_DEBUG(TLVL_WORK_STEPS) << "Card[" << m_card_id_str << "] DMA access initialized.";
97 // The rest was some CPU pinning.
98 TLOG_DEBUG(TLVL_WORK_STEPS) << m_card_id_str << "] is configured for datataking.";
99 m_configured = true;
100 }
101}
int allocate_CMEM(uint8_t numa, u_long bsize, u_long *paddr, u_long *vaddr)

◆ graceful_stop()

void dunedaq::flxlibs::CardWrapper::graceful_stop ( )

Definition at line 121 of file CardWrapper.cpp.

122{
123 TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << "Stopping CardWrapper of card " << m_card_id_str << "...";
124 if (m_run_marker.load()) {
125 set_running(false);
126 while (!m_dma_processor.get_readiness()) {
127 std::this_thread::sleep_for(std::chrono::milliseconds(10));
128 }
129 stop_DMA();
130 init_DMA();
131 TLOG() << "Stopped CardWrapper of card " << m_card_id_str << "!";
132 } else {
133 TLOG() << "CardWrapper of card " << m_card_id_str << " is already stopped!";
134 }
135}
void set_running(bool should_run)
#define TLOG(...)
Definition macro.hpp:22

◆ init_DMA()

void dunedaq::flxlibs::CardWrapper::init_DMA ( )
private

Definition at line 217 of file CardWrapper.cpp.

218{
219 TLOG_DEBUG(TLVL_WORK_STEPS) << "InitDMA issued...";
220 m_card_mutex.lock();
221 m_flx_card->dma_reset();
222 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard.dma_reset issued.";
223 m_flx_card->soft_reset();
224 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard.soft_reset issued.";
225 m_flx_card->irq_reset_counters();
226 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard.irq_reset_counters issued.";
227 // interrupted or polled DMA processing
228 if (m_interrupt_mode) {
229#if REGMAP_VERSION < 0x500
230 m_flx_card->irq_enable(IRQ_DATA_AVAILABLE);
231#else
233#endif
234 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard.irq_enable issued.";
235 } else {
236 m_flx_card->irq_disable();
237 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard.irq_disable issued.";
238 }
239 m_card_mutex.unlock();
242 m_read_index = 0;
243 TLOG_DEBUG(TLVL_WORK_STEPS) << "flxCard initDMA done card[" << m_card_id_str << "]";
244}
#define IRQ_DATA_AVAILABLE

◆ open_card()

void dunedaq::flxlibs::CardWrapper::open_card ( )
private

Definition at line 151 of file CardWrapper.cpp.

152{
153 TLOG_DEBUG(TLVL_WORK_STEPS) << "Opening FELIX card (with DMA lock mask)" << m_card_id_str;
154 try {
155 m_card_mutex.lock();
156 auto absolute_card_id = m_card_id + m_logical_unit;
157 u_int current_lock_mask = m_flx_card->get_lock_mask(absolute_card_id);
158 TLOG_DEBUG(TLVL_WORK_STEPS) << "Current lock mask for FELIX card " << m_card_id_str << " mask:" << int(current_lock_mask);
159 u_int to_lock_mask = u_int(m_dma_id+1);
160 if (current_lock_mask & to_lock_mask) { // LOCK_NONE=0, LOCK_DMA0=1, LOCK_DMA1=2 from FlxCard.h
161 ers::fatal(flxlibs::CardError(ERS_HERE, "FELIX card's DMA is locked by another process!"));
162 exit(EXIT_FAILURE);
163 }
164 m_flx_card->card_open(static_cast<int>(absolute_card_id), to_lock_mask); // FlxCard.h
165 m_card_mutex.unlock();
166 } catch (FlxException& ex) {
167 ers::error(flxlibs::CardError(ERS_HERE, ex.what()));
168 exit(EXIT_FAILURE);
169 }
170}

◆ operator=() [1/2]

CardWrapper & dunedaq::flxlibs::CardWrapper::operator= ( CardWrapper && )
delete

CardWrapper is not move-assignable.

◆ operator=() [2/2]

CardWrapper & dunedaq::flxlibs::CardWrapper::operator= ( const CardWrapper & )
delete

CardWrapper is not copy-assignable.

◆ process_DMA()

void dunedaq::flxlibs::CardWrapper::process_DMA ( )
private

Definition at line 281 of file CardWrapper.cpp.

282{
283 TLOG_DEBUG(TLVL_WORK_STEPS) << "CardWrapper starts processing blocks...";
284 while (m_run_marker.load()) {
285
286 // First fix us poll until read address makes sense
288 if (m_run_marker.load()) {
290 std::this_thread::sleep_for(std::chrono::microseconds(5000)); // fix 5ms initial poll
291 } else {
292 TLOG_DEBUG(TLVL_WORK_STEPS) << "Stop issued during poll! Returning...";
293 return;
294 }
295 }
296
297 // Loop or wait for interrupt while there are not enough data
299 if (m_run_marker.load()) {
300 if (m_interrupt_mode) {
301 m_card_mutex.lock();
302#if REGMAP_VERSION < 0x500
304#else
306#endif // REGMAP_VERSION
307 m_card_mutex.unlock();
308 } else { // poll mode
309 std::this_thread::sleep_for(std::chrono::microseconds(m_poll_time));
310 }
312 } else {
313 TLOG_DEBUG(TLVL_WORK_STEPS) << "Stop issued during waiting for data! Returning...";
314 return;
315 }
316 }
317
318 // Set write index and start DMA advancing
319 u_long write_index = (m_current_addr - m_phys_addr) / m_block_size;
320 uint64_t bytes = 0; // NOLINT
321 while (m_read_index != write_index) {
322 uint64_t from_address = m_virt_addr + (m_read_index * m_block_size); // NOLINT
323
324 // Handle block address
326 m_handle_block_addr(from_address);
327 }
328
329 // Advance
331 bytes += m_block_size;
332 }
333
334 // here check if we can move the read pointer in the circular buffer
338 }
339
340 // Finally, set new pointer
341 m_card_mutex.lock();
342 m_flx_card->dma_set_ptr(m_dma_id, m_destination);
343 m_card_mutex.unlock();
344 }
345 TLOG_DEBUG(TLVL_WORK_STEPS) << "CardWrapper processor thread finished.";
346}

◆ read_current_address()

void dunedaq::flxlibs::CardWrapper::read_current_address ( )
private

Definition at line 273 of file CardWrapper.cpp.

274{
275 m_card_mutex.lock();
276 m_current_addr = m_flx_card->m_bar0->DMA_DESC_STATUS[m_dma_id].current_address;
277 m_card_mutex.unlock();
278}

◆ set_block_addr_handler()

void dunedaq::flxlibs::CardWrapper::set_block_addr_handler ( std::function< void(uint64_t)> & handle)
inline

Definition at line 50 of file CardWrapper.hpp.

51 { // NOLINT
52 m_handle_block_addr = std::bind(handle, std::placeholders::_1);
54 }

◆ set_running()

void dunedaq::flxlibs::CardWrapper::set_running ( bool should_run)

Definition at line 144 of file CardWrapper.cpp.

145{
146 bool was_running = m_run_marker.exchange(should_run);
147 TLOG_DEBUG(TLVL_WORK_STEPS) << "Active state was toggled from " << was_running << " to " << should_run;
148}

◆ start()

void dunedaq::flxlibs::CardWrapper::start ( )

Definition at line 104 of file CardWrapper.cpp.

105{
106 TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << "Starting CardWrapper of card " << m_card_id_str << "...";
107 if (!m_run_marker.load()) {
109 TLOG() << "Block Address handler is not set! Is it intentional?";
110 }
111 start_DMA();
112 set_running(true);
114 TLOG() << "Started CardWrapper of card " << m_card_id_str << "...";
115 } else {
116 TLOG() << "CardWrapper of card " << m_card_id_str << " is already running!";
117 }
118}
bool set_work(Function &&f, Args &&... args)

◆ start_DMA()

void dunedaq::flxlibs::CardWrapper::start_DMA ( )
private

Definition at line 247 of file CardWrapper.cpp.

248{
249 TLOG_DEBUG(TLVL_WORK_STEPS) << "Issuing flxCard.dma_to_host for card " << m_card_id_str
250 << " dma id:" << std::to_string(m_dma_id);
251 m_card_mutex.lock();
253 m_card_mutex.unlock();
254}
static constexpr size_t m_dma_wraparound

◆ stop()

void dunedaq::flxlibs::CardWrapper::stop ( )

Definition at line 138 of file CardWrapper.cpp.

139{
141}

◆ stop_DMA()

void dunedaq::flxlibs::CardWrapper::stop_DMA ( )
private

Definition at line 257 of file CardWrapper.cpp.

258{
259 TLOG_DEBUG(TLVL_WORK_STEPS) << "Issuing flxCard.dma_stop for card " << m_card_id_str
260 << " dma id:" << std::to_string(m_dma_id);
261 m_card_mutex.lock();
262 m_flx_card->dma_stop(m_dma_id);
263 m_card_mutex.unlock();
264}

Member Data Documentation

◆ m_block_addr_handler_available

bool dunedaq::flxlibs::CardWrapper::m_block_addr_handler_available { false }
private

Definition at line 112 of file CardWrapper.hpp.

112{ false };

◆ m_block_size

size_t dunedaq::flxlibs::CardWrapper::m_block_size = 4096
staticconstexprprivate

Definition at line 62 of file CardWrapper.hpp.

◆ m_block_threshold

size_t dunedaq::flxlibs::CardWrapper::m_block_threshold
private

Definition at line 86 of file CardWrapper.hpp.

◆ m_card_id

uint8_t dunedaq::flxlibs::CardWrapper::m_card_id
private

Definition at line 81 of file CardWrapper.hpp.

◆ m_card_id_str

std::string dunedaq::flxlibs::CardWrapper::m_card_id_str
private

Definition at line 83 of file CardWrapper.hpp.

◆ m_card_mutex

std::mutex dunedaq::flxlibs::CardWrapper::m_card_mutex
private

Definition at line 96 of file CardWrapper.hpp.

◆ m_cmem_handle

int dunedaq::flxlibs::CardWrapper::m_cmem_handle
private

Definition at line 100 of file CardWrapper.hpp.

◆ m_configured

bool dunedaq::flxlibs::CardWrapper::m_configured { false }
private

Definition at line 80 of file CardWrapper.hpp.

80{ false };

◆ m_current_addr

uint64_t dunedaq::flxlibs::CardWrapper::m_current_addr
private

Definition at line 103 of file CardWrapper.hpp.

◆ m_destination

u_long dunedaq::flxlibs::CardWrapper::m_destination
private

Definition at line 105 of file CardWrapper.hpp.

◆ m_dma_id

uint8_t dunedaq::flxlibs::CardWrapper::m_dma_id
private

Definition at line 84 of file CardWrapper.hpp.

◆ m_dma_memory_size

std::size_t dunedaq::flxlibs::CardWrapper::m_dma_memory_size
private

Definition at line 99 of file CardWrapper.hpp.

◆ m_dma_processor

utilities::ReusableThread dunedaq::flxlibs::CardWrapper::m_dma_processor
private

Definition at line 110 of file CardWrapper.hpp.

◆ m_dma_processor_name

const std::string dunedaq::flxlibs::CardWrapper::m_dma_processor_name = "flx-dma"
inlinestaticprivate

Definition at line 108 of file CardWrapper.hpp.

◆ m_dma_wraparound

size_t dunedaq::flxlibs::CardWrapper::m_dma_wraparound = FLX_DMA_WRAPAROUND
staticconstexprprivate

Definition at line 63 of file CardWrapper.hpp.

◆ m_flx_card

UniqueFlxCard dunedaq::flxlibs::CardWrapper::m_flx_card
private

Definition at line 95 of file CardWrapper.hpp.

◆ m_handle_block_addr

std::function<void(uint64_t)> dunedaq::flxlibs::CardWrapper::m_handle_block_addr
private

Definition at line 111 of file CardWrapper.hpp.

◆ m_info_str

std::string dunedaq::flxlibs::CardWrapper::m_info_str
private

Definition at line 91 of file CardWrapper.hpp.

◆ m_interrupt_mode

bool dunedaq::flxlibs::CardWrapper::m_interrupt_mode
private

Definition at line 87 of file CardWrapper.hpp.

◆ m_links_enabled

std::vector<unsigned int> dunedaq::flxlibs::CardWrapper::m_links_enabled
private

Definition at line 90 of file CardWrapper.hpp.

◆ m_logical_unit

uint8_t dunedaq::flxlibs::CardWrapper::m_logical_unit
private

Definition at line 82 of file CardWrapper.hpp.

◆ m_margin_blocks

size_t dunedaq::flxlibs::CardWrapper::m_margin_blocks
private

Definition at line 85 of file CardWrapper.hpp.

◆ m_max_links_per_card

size_t dunedaq::flxlibs::CardWrapper::m_max_links_per_card = 6
staticconstexprprivate

Definition at line 59 of file CardWrapper.hpp.

◆ m_numa_id

uint8_t dunedaq::flxlibs::CardWrapper::m_numa_id
private

Definition at line 89 of file CardWrapper.hpp.

◆ m_phys_addr

uint64_t dunedaq::flxlibs::CardWrapper::m_phys_addr
private

Definition at line 102 of file CardWrapper.hpp.

◆ m_poll_time

size_t dunedaq::flxlibs::CardWrapper::m_poll_time
private

Definition at line 88 of file CardWrapper.hpp.

◆ m_read_index

unsigned dunedaq::flxlibs::CardWrapper::m_read_index
private

Definition at line 104 of file CardWrapper.hpp.

◆ m_run_lock

std::atomic<bool> dunedaq::flxlibs::CardWrapper::m_run_lock
private

Definition at line 109 of file CardWrapper.hpp.

◆ m_run_marker

std::atomic<bool> dunedaq::flxlibs::CardWrapper::m_run_marker
private

Definition at line 79 of file CardWrapper.hpp.

◆ m_virt_addr

uint64_t dunedaq::flxlibs::CardWrapper::m_virt_addr
private

Definition at line 101 of file CardWrapper.hpp.


The documentation for this class was generated from the following files: