Line data Source code
1 : /**
2 : * @file HSIReadout.cpp HSIReadout class
3 : * implementation
4 : *
5 : * This is part of the DUNE DAQ Software Suite, copyright 2020.
6 : * Licensing/copyright details are in the COPYING file that you should have
7 : * received with this code.
8 : */
9 :
10 : #include "HSIReadout.hpp"
11 :
12 : #include "timing/HSIDesignInterface.hpp"
13 : #include "timing/TimingIssues.hpp"
14 :
15 : #include "hsilibs/opmon/hsi_readout_info.pb.cc"
16 :
17 : #include "confmodel/Connection.hpp"
18 : #include "confmodel/DaqModule.hpp"
19 : #include "logging/Logging.hpp" // NOTE: if ISSUES ARE DECLARED BEFORE include logging/Logging.hpp, TLOG_DEBUG<<issue wont work.
20 : #include "rcif/cmd/Nljs.hpp"
21 : #include <chrono>
22 : #include <cstdlib>
23 : #include <memory>
24 : #include <string>
25 : #include <thread>
26 : #include <vector>
27 :
28 : namespace dunedaq {
29 0 : ERS_DECLARE_ISSUE(hsilibs,
30 : InvalidHSIEventHeader,
31 : " Invalid hsi buffer event header: 0x" << std::hex << header,
32 : ((uint32_t)header)) // NOLINT(build/unsigned)
33 0 : ERS_DECLARE_ISSUE(hsilibs,
34 : InvalidHSIEventTimestamp,
35 : " Invalid hsi buffer event timestamp: 0x" << std::hex << timestamp,
36 : ((uint64_t)timestamp)) // NOLINT(build/unsigned)
37 0 : ERS_DECLARE_ISSUE(hsilibs,
38 : InvalidNumberReadoutHSIWords,
39 : " Invalid number of hsi words readout from buffer: 0x" << std::hex << n_words,
40 : ((uint16_t)n_words)) // NOLINT(build/unsigned)
41 : namespace hsilibs {
42 :
43 0 : HSIReadout::HSIReadout(const std::string& name)
44 : : HSIEventSender(name)
45 0 : , m_thread(std::bind(&HSIReadout::do_hsi_work, this, std::placeholders::_1))
46 0 : , m_readout_period(1000)
47 0 : , m_hsi_device(nullptr)
48 0 : , m_readout_counter(0)
49 0 : , m_last_readout_timestamp(0)
50 :
51 : {
52 0 : register_command("conf", &HSIReadout::do_configure);
53 0 : register_command("start", &HSIReadout::do_start);
54 0 : register_command("stop", &HSIReadout::do_stop);
55 0 : register_command("scrap", &HSIReadout::do_scrap);
56 0 : }
57 :
58 : void
59 0 : HSIReadout::init(std::shared_ptr<appfwk::ConfigurationManager> mcfg)
60 : {
61 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering init() method";
62 0 : HSIEventSender::init(mcfg);
63 0 : auto mdal = mcfg->get_dal<appmodel::HSIReadout>(get_name());
64 :
65 0 : for (auto con : mdal->get_outputs()) {
66 0 : if (con->get_data_type() == datatype_to_string<HSI_FRAME_STRUCT>()) {
67 :
68 0 : m_raw_hsi_data_sender = get_iom_sender<HSI_FRAME_STRUCT>(con->UID());
69 : }
70 : }
71 0 : m_params = mdal->get_configuration();
72 :
73 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting init() method";
74 0 : }
75 :
76 : void
77 0 : HSIReadout::do_configure(const CommandData_t& /*data*/)
78 : {
79 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_configure() method";
80 :
81 0 : m_readout_period = m_params->get_readout_period();
82 :
83 0 : configure_uhal(m_params->get_uhal_log_level(), m_params->get_connections_file()); // configure hw ipbus connection
84 :
85 0 : if (m_params->get_hsi_device_name().empty()) {
86 0 : throw UHALDeviceNameIssue(ERS_HERE, "Device name for HSIReadout should not be empty");
87 : }
88 0 : m_hsi_device_name = m_params->get_hsi_device_name();
89 :
90 0 : try {
91 0 : m_hsi_device = std::make_unique<uhal::HwInterface>(m_connection_manager->getDevice(m_hsi_device_name));
92 0 : } catch (const uhal::exception::ConnectionUIDDoesNotExist& exception) {
93 0 : std::stringstream message;
94 0 : message << "UHAL device name not " << m_hsi_device_name << " in connections file";
95 0 : throw UHALDeviceNameIssue(ERS_HERE, message.str(), exception);
96 0 : }
97 :
98 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_configure() method";
99 0 : }
100 :
101 : void
102 0 : HSIReadout::do_start(const CommandData_t& data)
103 : {
104 0 : TLOG() << get_name() << ": Entering do_start() method";
105 0 : auto start_params = data.get<rcif::cmd::StartParams>();
106 0 : m_run_number.store(start_params.run);
107 0 : m_thread.start_working_thread("read-hsi-events");
108 0 : TLOG() << get_name() << " successfully started";
109 0 : TLOG() << get_name() << ": Exiting do_start() method";
110 0 : }
111 :
112 : void
113 0 : HSIReadout::do_stop(const CommandData_t& /*data*/)
114 : {
115 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_stop() method";
116 0 : m_thread.stop_working_thread();
117 0 : TLOG() << get_name() << " successfully stopped";
118 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_stop() method";
119 0 : }
120 :
121 : void
122 0 : HSIReadout::do_scrap(const CommandData_t& /*data*/)
123 : {
124 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_scrap() method";
125 0 : scrap_uhal();
126 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_scrap() method";
127 0 : }
128 :
129 : void
130 0 : HSIReadout::do_hsi_work(std::atomic<bool>& running_flag)
131 : {
132 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_hsievent_work() method";
133 :
134 0 : m_readout_counter = 0;
135 0 : m_sent_counter = 0;
136 0 : m_failed_to_send_counter = 0;
137 :
138 0 : m_last_readout_timestamp = 0;
139 0 : m_last_sent_timestamp = 0;
140 :
141 0 : auto hsi_design = dynamic_cast<const timing::HSIDesignInterface*>(&m_hsi_device->getNode(""));
142 0 : auto hsi_node = hsi_design->get_hsi_node();
143 0 : auto ept_node = hsi_design->get_endpoint_node_plain(0);
144 :
145 0 : while (running_flag.load()) {
146 :
147 : // endpoint should be ready if already running
148 0 : auto hsi_endpoint_ready = ept_node->endpoint_ready();
149 0 : if (!hsi_endpoint_ready) {
150 0 : auto hsi_endpoint_state = ept_node->read_endpoint_state();
151 0 : ers::error(timing::EndpointNotReady(ERS_HERE, "HSI", hsi_endpoint_state));
152 : }
153 :
154 0 : auto hsi_emulation_mode = hsi_node.read_signal_source_mode();
155 :
156 0 : uhal::ValVector<uint32_t> hsi_words;
157 0 : try {
158 0 : uint16_t n_words_in_buffer; // NOLINT(build/unsigned)
159 :
160 0 : hsi_words = hsi_node.read_data_buffer(n_words_in_buffer, false, true);
161 0 : update_buffer_counts(n_words_in_buffer);
162 0 : TLOG_DEBUG(5) << get_name() << ": Number of words in HSI buffer: " << n_words_in_buffer;
163 0 : } catch (const uhal::exception::UdpTimeout& excpt) {
164 0 : ers::error(HSIReadoutNetworkIssue(ERS_HERE, excpt));
165 0 : std::this_thread::sleep_for(std::chrono::microseconds(m_readout_period));
166 0 : continue;
167 0 : }
168 :
169 0 : constexpr size_t n_words_per_hsi_buffer_event = timing::HSINode::hsi_buffer_event_words_number;
170 : // one or more complete events
171 0 : if (hsi_words.size() % n_words_per_hsi_buffer_event == 0 && hsi_words.size() > 0) {
172 0 : uint n_hsi_events = hsi_words.size() / n_words_per_hsi_buffer_event;
173 :
174 0 : TLOG_DEBUG(4) << get_name() << ": Have readout " << n_hsi_events << " HSIEvent(s) ";
175 :
176 0 : m_readout_counter.store(m_readout_counter.load() + n_hsi_events);
177 0 : for (uint i = 0; i < n_hsi_events; ++i) {
178 0 : std::array<uint32_t, n_words_per_hsi_buffer_event> raw_event;
179 :
180 0 : auto event_start = hsi_words.begin() + (i * n_words_per_hsi_buffer_event);
181 0 : std::copy_n(event_start, n_words_per_hsi_buffer_event, raw_event.begin());
182 :
183 0 : uint32_t header = raw_event.at(0); // NOLINT(build/unsigned)
184 0 : uint32_t ts_low = raw_event.at(1); // NOLINT(build/unsigned)
185 0 : uint32_t ts_high = raw_event.at(2); // NOLINT(build/unsigned)
186 0 : uint32_t data = raw_event.at(3); // NOLINT(build/unsigned)
187 0 : uint32_t trigger = raw_event.at(4); // NOLINT(build/unsigned)
188 :
189 : // put together the timestamp
190 0 : uint64_t ts = ts_low | (static_cast<uint64_t>(ts_high) << 32); // NOLINT(build/unsigned)
191 :
192 : // bits 31-16 contain the HSI device ID
193 0 : uint32_t hsi_device_id = header >> 16; // NOLINT(build/unsigned)
194 : // bits 15-0 contain the sequence counter
195 0 : uint32_t counter = header & 0x0000ffff; // NOLINT(build/unsigned)
196 :
197 0 : if ((header >> 16) != 0xaa00) {
198 0 : ers::error(InvalidHSIEventHeader(ERS_HERE, header));
199 0 : continue;
200 : }
201 :
202 0 : if (!ts) {
203 0 : ers::warning(InvalidHSIEventTimestamp(ERS_HERE, ts));
204 0 : continue;
205 : }
206 :
207 0 : if (counter > 0 && counter % 60000 == 0) {
208 0 : TLOG_DEBUG(3) << "Sequence counter from firmware: " << counter;
209 : }
210 :
211 0 : TLOG_DEBUG(3) << get_name() << ": read out data: " << std::showbase << std::hex << header << ", " << ts << ", "
212 0 : << data << ", " << std::bitset<32>(trigger) << ", "
213 0 : << "ts: " << ts << "\n";
214 :
215 : // In lieu of propper HSI channel to signal mapping, fake signal map when HSI firmware+hardware is in emulation
216 : // mode.
217 : // TODO DAQ/HSI team 24/03/22 Put in place HSI channel to signal mapping.
218 :
219 0 : if (hsi_emulation_mode) {
220 0 : TLOG_DEBUG(3) << " HSI hardware is in emulation mode, faking (overwriting) signal map from firmware+hardware "
221 0 : "to have (only) bit 7 high.";
222 0 : trigger = 1UL << 7;
223 : }
224 :
225 0 : dfmessages::HSIEvent event = dfmessages::HSIEvent(hsi_device_id, trigger, ts, counter, m_run_number);
226 :
227 0 : m_last_readout_timestamp.store(ts);
228 :
229 0 : send_hsi_event(event);
230 :
231 : // Send raw HSI data to a DLH
232 0 : std::array<uint32_t, 7> hsi_struct;
233 0 : hsi_struct[0] = (0x1 << 6) | 0x1; // DAQHeader, frame version: 1, det id: 1
234 0 : hsi_struct[1] = ts_low;
235 0 : hsi_struct[2] = ts_high;
236 0 : hsi_struct[3] = data;
237 0 : hsi_struct[4] = 0x0;
238 0 : hsi_struct[5] = trigger;
239 0 : hsi_struct[6] = counter;
240 :
241 0 : TLOG_DEBUG(3) << get_name() << ": Formed HSI_FRAME_STRUCT " << std::hex << "0x" << hsi_struct[0] << ", 0x"
242 0 : << hsi_struct[1] << ", 0x" << hsi_struct[2] << ", 0x" << hsi_struct[3] << ", 0x" << hsi_struct[4]
243 0 : << ", 0x" << hsi_struct[5] << ", 0x" << hsi_struct[6] << "\n";
244 :
245 0 : send_raw_hsi_data(hsi_struct, m_raw_hsi_data_sender.get());
246 : }
247 : }
248 : // empty buffer is ok
249 0 : else if (hsi_words.size() == 0) {
250 0 : TLOG_DEBUG(20) << "Empty HSI buffter";
251 : }
252 : // anything else is unexpected
253 : else {
254 0 : ers::error(InvalidNumberReadoutHSIWords(ERS_HERE, hsi_words.size()));
255 : }
256 0 : std::this_thread::sleep_for(std::chrono::microseconds(m_readout_period));
257 0 : }
258 0 : std::ostringstream oss_summ;
259 0 : oss_summ << ": Exiting the read_hsievents() method, read out " << m_readout_counter.load()
260 0 : << " HSIEvent messages and successfully sent " << m_sent_counter.load() << " copies. ";
261 0 : ers::info(hsilibs::ProgressUpdate(ERS_HERE, get_name(), oss_summ.str()));
262 0 : TLOG_DEBUG(2) << get_name() << ": Exiting do_work() method";
263 0 : } // NOLINT
264 :
265 : void
266 0 : HSIReadout::update_buffer_counts(uint16_t new_count) // NOLINT(build/unsigned)
267 : {
268 0 : std::unique_lock mon_data_lock(m_buffer_counts_mutex);
269 0 : if (m_buffer_counts.size() > 1000)
270 0 : m_buffer_counts.pop_front();
271 0 : m_buffer_counts.push_back(new_count);
272 0 : }
273 :
274 : double
275 0 : HSIReadout::read_average_buffer_counts()
276 : {
277 0 : std::unique_lock mon_data_lock(m_buffer_counts_mutex);
278 :
279 0 : double total_counts;
280 0 : uint32_t number_of_counts; // NOLINT(build/unsigned)
281 :
282 0 : total_counts = 0;
283 0 : number_of_counts = m_buffer_counts.size();
284 :
285 0 : if (number_of_counts) {
286 0 : for (uint i = 0; i < number_of_counts; ++i) { // NOLINT(build/unsigned)
287 0 : total_counts = total_counts + m_buffer_counts.at(i);
288 : }
289 0 : return total_counts / number_of_counts;
290 : } else {
291 : return 0;
292 : }
293 0 : }
294 :
295 : void
296 0 : HSIReadout::generate_opmon_data()
297 : {
298 :
299 0 : opmon::HSIReadoutInfo info;
300 :
301 0 : info.set_readout_hsi_events_counter(m_readout_counter.load());
302 0 : info.set_sent_hsi_events_counter(m_sent_counter.load());
303 0 : info.set_failed_to_send_hsi_events_counter(m_failed_to_send_counter.load());
304 0 : info.set_last_readout_timestamp(m_last_readout_timestamp.load());
305 0 : info.set_last_sent_timestamp(m_last_sent_timestamp.load());
306 0 : info.set_average_buffer_occupancy(read_average_buffer_counts());
307 :
308 0 : publish(std::move(info));
309 0 : }
310 :
311 : } // namespace hsilibs
312 : } // namespace dunedaq
313 :
314 0 : DEFINE_DUNE_DAQ_MODULE(dunedaq::hsilibs::HSIReadout)
315 :
316 : // Local Variables:
317 : // c-basic-offset: 2
318 : // End:
|
