Line data Source code
1 : /**
2 : * @file FelixReaderModule.cc FelixReaderModule DAQModule implementation
3 : *
4 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
5 : * Licensing/copyright details are in the COPYING file that you should have
6 : * received with this code.
7 : */
8 : #include "confmodel/Connection.hpp"
9 : #include "confmodel/QueueWithSourceId.hpp"
10 : #include "confmodel/DetDataSender.hpp"
11 : #include "confmodel/DetectorStream.hpp"
12 : #include "confmodel/GeoId.hpp"
13 :
14 : #include "appmodel/DataMoveCallbackConf.hpp"
15 : #include "appmodel/DataReaderModule.hpp"
16 : #include "appmodel/FelixDetectorToDaqConnection.hpp"
17 : #include "appmodel/FelixInterface.hpp"
18 : #include "appmodel/FelixDataSender.hpp"
19 :
20 : #include "CreateElink.hpp"
21 : #include "FelixReaderModule.hpp"
22 : #include "FelixIssues.hpp"
23 :
24 : #include "logging/Logging.hpp"
25 :
26 : #include "flxcard/FlxException.h"
27 :
28 : #include <chrono>
29 : #include <memory>
30 : #include <string>
31 : #include <thread>
32 : #include <utility>
33 : #include <vector>
34 :
35 : /**
36 : * @brief Name used by TRACE TLOG calls from this source file
37 : */
38 : #define TRACE_NAME "FelixReaderModule" // NOLINT
39 :
40 : /**
41 : * @brief TRACE debug levels used in this source file
42 : */
43 : enum
44 : {
45 : TLVL_ENTER_EXIT_METHODS = 5,
46 : TLVL_WORK_STEPS = 10,
47 : TLVL_BOOKKEEPING = 15
48 : };
49 :
50 : namespace dunedaq {
51 : namespace flxlibs {
52 :
53 0 : FelixReaderModule::FelixReaderModule(const std::string& name)
54 : : DAQModule(name)
55 0 : , m_configured(false)
56 0 : , m_card_id(0)
57 0 : , m_logical_unit(0)
58 0 : , m_links_enabled()
59 0 : , m_num_links(0)
60 0 : , m_block_size(0)
61 0 : , m_block_router(nullptr)
62 : //, block_ptr_sinks_{ }
63 :
64 : {
65 :
66 0 : register_command("conf", &FelixReaderModule::do_configure);
67 0 : register_command("start", &FelixReaderModule::do_start);
68 0 : register_command("stop_trigger_sources", &FelixReaderModule::do_stop);
69 0 : }
70 :
71 : void
72 0 : FelixReaderModule::init(const std::shared_ptr<appfwk::ConfigurationManager> mcfg)
73 : {
74 : //auto ini = args.get<appfwk::app::ModInit>();
75 :
76 0 : auto modconf = mcfg->get_dal<appmodel::DataReaderModule>(get_name());
77 0 : auto session = mcfg->get_session();
78 :
79 0 : if (modconf->get_connections().size() != 1) {
80 0 : throw InitializationError(ERS_HERE, "FLX Data Reader does not have a unique associated flx_if");
81 : }
82 :
83 0 : const auto det_conn = modconf->get_connections()[0]->cast<appmodel::FelixDetectorToDaqConnection>();
84 :
85 : // Create a source_id to local elink map
86 0 : std::map<uint, uint> src_id_to_elink_map;
87 0 : auto flx_if = det_conn->receiver()->cast<appmodel::FelixInterface>();
88 0 : auto det_senders = det_conn->get_felix_senders();
89 :
90 0 : if (!det_senders.empty()) { // Redundant test as schema forbids 0 senders
91 0 : for (const auto& data_sender : det_senders) {
92 : // Check if sender enabled
93 0 : if (data_sender->is_disabled(*session)) {
94 0 : continue;
95 : }
96 0 : for (const auto& stream : data_sender->get_streams()) {
97 0 : src_id_to_elink_map[stream->get_source_id()] = data_sender->get_link();
98 : }
99 :
100 0 : TLOG(TLVL_BOOKKEEPING) << "Registering link: " << (uint32_t)data_sender->get_link() << " / " << m_links_enabled.size();
101 0 : m_links_enabled.push_back(data_sender->get_link());
102 :
103 : }
104 : }
105 0 : m_num_links = m_links_enabled.size();
106 0 : if (flx_if != nullptr) {
107 0 : m_card_wrapper = std::make_unique<CardWrapper>(flx_if, m_links_enabled);
108 0 : m_card_id = flx_if->get_card();
109 0 : m_logical_unit = flx_if->get_slr();
110 0 : m_block_size = flx_if->get_dma_block_size() * m_1kb_block_size;
111 0 : m_chunk_trailer_size = flx_if->get_chunk_trailer_size();
112 : }
113 :
114 0 : for (auto cb :modconf->get_raw_data_callbacks()) {
115 0 : auto cb_with_id = cb->cast<appmodel::DataMoveCallbackConf>();
116 0 : if (cb_with_id == nullptr) continue;
117 0 : TLOG_DEBUG(TLVL_WORK_STEPS) << ": CardReader output callback is " << cb_with_id->UID();
118 0 : TLOG_DEBUG(TLVL_WORK_STEPS) << "Creating ElinkModel for target callback: " << cb_with_id->UID() << " DLH number: " << cb_with_id->get_source_id();
119 0 : auto elink = src_id_to_elink_map[cb_with_id->get_source_id()];
120 0 : auto link_ptr = m_elinks[elink] = createElinkModel(cb_with_id);
121 0 : if ( ! link_ptr ) {
122 0 : ers::fatal(InitializationError(ERS_HERE, "CreateElink failed to provide an appropriate model for queue!"));
123 : }
124 0 : register_node( cb_with_id->UID(), link_ptr);
125 0 : link_ptr->init(m_block_queue_capacity);
126 0 : }
127 :
128 : // Router function of block to appropriate ElinkHandlers
129 0 : m_block_router = [&](uint64_t block_addr) { // NOLINT
130 : // block_counter++;
131 0 : const auto* block = const_cast<felix::packetformat::block*>(
132 0 : felix::packetformat::block_from_bytes(reinterpret_cast<const char*>(block_addr)) // NOLINT
133 : );
134 0 : auto elink = block->elink;
135 0 : if (m_elinks.count(elink) != 0) {
136 0 : m_elinks[elink]->queue_in_block_address(block_addr);
137 : } else {
138 : // Really bad -> unexpeced ELINK ID in Block.
139 : // This check is needed in order to avoid dynamically add thousands
140 : // of ELink parser implementations on the fly, in case the data
141 : // corruption is extremely severe.
142 : //
143 : // Possible causes:
144 : // -> enabled links that don't connect to anything
145 : // -> unexpected format (fw/sw version missmatch)
146 : // -> data corruption from FE
147 : // -> data corruption from CR (really rare, last possible cause)
148 :
149 : // NO TLOG_DEBUG, but should count and periodically report corrupted DMA blocks.
150 : }
151 0 : };
152 :
153 : // Set function for the CardWrapper's block processor.
154 0 : m_card_wrapper->set_block_addr_handler(m_block_router);
155 0 : }
156 :
157 : void
158 0 : FelixReaderModule::do_configure(const CommandData_t& /*args*/)
159 : {
160 :
161 0 : bool is_32b_trailer = false;
162 :
163 0 : TLOG(TLVL_BOOKKEEPING) << "Number of felix links specified in configuration: " << m_links_enabled.size();
164 0 : TLOG(TLVL_BOOKKEEPING) << "Number of data link handlers: " << m_elinks.size();
165 :
166 : // Config checks
167 0 : if (m_num_links != m_elinks.size()) {
168 0 : ers::fatal(ElinkConfigurationInconsistency(ERS_HERE, m_links_enabled.size()));
169 : }
170 0 : if (m_block_size % m_1kb_block_size != 0) {
171 0 : ers::fatal(BlockSizeConfigurationInconsistency(ERS_HERE, m_block_size));
172 0 : } else if (m_block_size != m_1kb_block_size && m_chunk_trailer_size != m_32b_trailer_size) {
173 0 : ers::fatal(BlockSizeConfigurationInconsistency(ERS_HERE, m_block_size));
174 0 : } else if (m_chunk_trailer_size == m_32b_trailer_size) {
175 0 : is_32b_trailer = true;
176 : }
177 :
178 : // Configure components
179 0 : TLOG(TLVL_WORK_STEPS) << "Card ID: " << m_card_id;
180 0 : TLOG(TLVL_WORK_STEPS) << "Configuring components with Block size:" << m_block_size
181 0 : << " & trailer size: " << m_chunk_trailer_size;
182 0 : m_card_wrapper->configure();
183 : // get linkids defined by queues
184 0 : std::vector<int> linkids;
185 0 : for(auto& [id, elink] : m_elinks) {
186 0 : linkids.push_back(id);
187 : }
188 : // loop through all elinkmodels, change the linkids to link tags and configure
189 0 : for (unsigned i = 0; i < m_num_links; ++i) {
190 0 : auto elink = m_elinks.extract(linkids[i]);
191 0 : auto tag = m_links_enabled[i] * m_elink_multiplier;
192 0 : elink.key() = tag;
193 0 : m_elinks.insert(std::move(elink));
194 0 : m_elinks[tag]->set_ids(m_card_id, m_logical_unit, m_links_enabled[i], tag);
195 0 : m_elinks[tag]->conf(m_block_size, is_32b_trailer);
196 0 : }
197 0 : }
198 :
199 : void
200 0 : FelixReaderModule::do_start(const CommandData_t& /*args*/)
201 : {
202 0 : m_card_wrapper->start();
203 0 : for (auto& [tag, elink] : m_elinks) {
204 0 : elink->acquire_callback();
205 0 : elink->start();
206 : }
207 0 : }
208 :
209 : void
210 0 : FelixReaderModule::do_stop(const CommandData_t& /*args*/)
211 : {
212 0 : m_card_wrapper->stop();
213 0 : for (auto& [tag, elink] : m_elinks) {
214 0 : elink->stop();
215 : }
216 0 : }
217 :
218 :
219 : } // namespace flxlibs
220 : } // namespace dunedaq
221 :
222 0 : DEFINE_DUNE_DAQ_MODULE(dunedaq::flxlibs::FelixReaderModule)
|
