Line data Source code
1 : /**
2 : * @file HSIReadout.cpp HSIReadout class
3 : * implementation
4 : *
5 : * This is part of the DUNE DAQ Software Suite, copyright 2020.
6 : * Licensing/copyright details are in the COPYING file that you should have
7 : * received with this code.
8 : */
9 :
10 : #include "HSIReadout.hpp"
11 :
12 : #include "timing/TimingIssues.hpp"
13 : #include "timing/HSIDesignInterface.hpp"
14 :
15 : #include "hsilibs/opmon/hsi_readout_info.pb.cc"
16 :
17 : #include "logging/Logging.hpp" // NOTE: if ISSUES ARE DECLARED BEFORE include logging/Logging.hpp, TLOG_DEBUG<<issue wont work.
18 : #include "rcif/cmd/Nljs.hpp"
19 : #include "confmodel/DaqModule.hpp"
20 : #include "confmodel/Connection.hpp"
21 : #include <chrono>
22 : #include <cstdlib>
23 : #include <memory>
24 : #include <string>
25 : #include <thread>
26 : #include <vector>
27 :
28 : namespace dunedaq {
29 0 : ERS_DECLARE_ISSUE(hsilibs, InvalidHSIEventHeader, " Invalid hsi buffer event header: 0x" << std::hex << header, ((uint32_t)header)) // NOLINT(build/unsigned)
30 0 : ERS_DECLARE_ISSUE(hsilibs, InvalidHSIEventTimestamp, " Invalid hsi buffer event timestamp: 0x" << std::hex << timestamp, ((uint64_t)timestamp)) // NOLINT(build/unsigned)
31 0 : ERS_DECLARE_ISSUE(hsilibs, InvalidNumberReadoutHSIWords, " Invalid number of hsi words readout from buffer: 0x" << std::hex << n_words, ((uint16_t)n_words)) // NOLINT(build/unsigned)
32 : namespace hsilibs {
33 :
34 0 : HSIReadout::HSIReadout(const std::string& name)
35 : : HSIEventSender(name)
36 0 : , m_thread(std::bind(&HSIReadout::do_hsi_work, this, std::placeholders::_1))
37 0 : , m_readout_period(1000)
38 0 : , m_hsi_device(nullptr)
39 0 : , m_readout_counter(0)
40 0 : , m_last_readout_timestamp(0)
41 :
42 : {
43 0 : register_command("conf", &HSIReadout::do_configure);
44 0 : register_command("start", &HSIReadout::do_start);
45 0 : register_command("stop", &HSIReadout::do_stop);
46 0 : register_command("scrap", &HSIReadout::do_scrap);
47 0 : }
48 :
49 : void
50 0 : HSIReadout::init(std::shared_ptr<appfwk::ConfigurationManager> mcfg)
51 : {
52 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering init() method";
53 0 : HSIEventSender::init(mcfg);
54 0 : auto mdal = mcfg->get_dal<appmodel::HSIReadout>(get_name());
55 :
56 0 : for (auto con : mdal->get_outputs()) {
57 0 : if (con->get_data_type() == datatype_to_string<HSI_FRAME_STRUCT>()) {
58 :
59 0 : m_raw_hsi_data_sender = get_iom_sender<HSI_FRAME_STRUCT>(con->UID());
60 : }
61 : }
62 0 : m_params = mdal->get_configuration();
63 :
64 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting init() method";
65 0 : }
66 :
67 : void
68 0 : HSIReadout::do_configure(const CommandData_t& /*data*/)
69 : {
70 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_configure() method";
71 :
72 0 : m_readout_period = m_params->get_readout_period();
73 :
74 0 : configure_uhal(m_params->get_uhal_log_level(), m_params->get_connections_file()); // configure hw ipbus connection
75 :
76 0 : if (m_params->get_hsi_device_name().empty())
77 : {
78 0 : throw UHALDeviceNameIssue(ERS_HERE, "Device name for HSIReadout should not be empty");
79 : }
80 0 : m_hsi_device_name = m_params->get_hsi_device_name();
81 :
82 0 : try {
83 0 : m_hsi_device = std::make_unique<uhal::HwInterface>(m_connection_manager->getDevice(m_hsi_device_name));
84 0 : } catch (const uhal::exception::ConnectionUIDDoesNotExist& exception) {
85 0 : std::stringstream message;
86 0 : message << "UHAL device name not " << m_hsi_device_name << " in connections file";
87 0 : throw UHALDeviceNameIssue(ERS_HERE, message.str(), exception);
88 0 : }
89 :
90 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_configure() method";
91 0 : }
92 :
93 : void
94 0 : HSIReadout::do_start(const CommandData_t& data)
95 : {
96 0 : TLOG() << get_name() << ": Entering do_start() method";
97 0 : auto start_params = data.get<rcif::cmd::StartParams>();
98 0 : m_run_number.store(start_params.run);
99 0 : m_thread.start_working_thread("read-hsi-events");
100 0 : TLOG() << get_name() << " successfully started";
101 0 : TLOG() << get_name() << ": Exiting do_start() method";
102 0 : }
103 :
104 : void
105 0 : HSIReadout::do_stop(const CommandData_t& /*data*/)
106 : {
107 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_stop() method";
108 0 : m_thread.stop_working_thread();
109 0 : TLOG() << get_name() << " successfully stopped";
110 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_stop() method";
111 0 : }
112 :
113 : void
114 0 : HSIReadout::do_scrap(const CommandData_t& /*data*/)
115 : {
116 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_scrap() method";
117 0 : scrap_uhal();
118 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_scrap() method";
119 0 : }
120 :
121 : void
122 0 : HSIReadout::do_hsi_work(std::atomic<bool>& running_flag)
123 : {
124 0 : TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_hsievent_work() method";
125 :
126 0 : m_readout_counter = 0;
127 0 : m_sent_counter = 0;
128 0 : m_failed_to_send_counter = 0;
129 :
130 0 : m_last_readout_timestamp = 0;
131 0 : m_last_sent_timestamp = 0;
132 :
133 0 : auto hsi_design = dynamic_cast<const timing::HSIDesignInterface*> (&m_hsi_device->getNode(""));
134 0 : auto hsi_node = hsi_design->get_hsi_node();
135 0 : auto ept_node = hsi_design->get_endpoint_node_plain(0);
136 :
137 0 : while (running_flag.load()) {
138 :
139 : // endpoint should be ready if already running
140 0 : auto hsi_endpoint_ready = ept_node->endpoint_ready();
141 0 : if (!hsi_endpoint_ready)
142 : {
143 0 : auto hsi_endpoint_state = ept_node->read_endpoint_state();
144 0 : ers::error(timing::EndpointNotReady(ERS_HERE, "HSI", hsi_endpoint_state));
145 : }
146 :
147 0 : auto hsi_emulation_mode = hsi_node.read_signal_source_mode();
148 :
149 0 : uhal::ValVector<uint32_t> hsi_words;
150 0 : try
151 : {
152 0 : uint16_t n_words_in_buffer; // NOLINT(build/unsigned)
153 :
154 0 : hsi_words = hsi_node.read_data_buffer(n_words_in_buffer, false, true);
155 0 : update_buffer_counts(n_words_in_buffer);
156 0 : TLOG_DEBUG(5) << get_name() << ": Number of words in HSI buffer: " << n_words_in_buffer;
157 : }
158 0 : catch (const uhal::exception::UdpTimeout& excpt)
159 : {
160 0 : ers::error(HSIReadoutNetworkIssue(ERS_HERE, excpt));
161 0 : std::this_thread::sleep_for(std::chrono::microseconds(m_readout_period));
162 0 : continue;
163 0 : }
164 :
165 0 : constexpr size_t n_words_per_hsi_buffer_event = timing::HSINode::hsi_buffer_event_words_number;
166 : // one or more complete events
167 0 : if (hsi_words.size() % n_words_per_hsi_buffer_event == 0 && hsi_words.size() > 0)
168 : {
169 0 : uint n_hsi_events = hsi_words.size() / n_words_per_hsi_buffer_event;
170 :
171 0 : TLOG_DEBUG(4) << get_name() << ": Have readout " << n_hsi_events << " HSIEvent(s) ";
172 :
173 0 : m_readout_counter.store(m_readout_counter.load() + n_hsi_events);
174 0 : for (uint i = 0; i < n_hsi_events; ++i)
175 : {
176 0 : std::array<uint32_t, n_words_per_hsi_buffer_event> raw_event;
177 :
178 0 : auto event_start = hsi_words.begin() + (i * n_words_per_hsi_buffer_event);
179 0 : std::copy_n(event_start, n_words_per_hsi_buffer_event, raw_event.begin());
180 :
181 0 : uint32_t header = raw_event.at(0); // NOLINT(build/unsigned)
182 0 : uint32_t ts_low = raw_event.at(1); // NOLINT(build/unsigned)
183 0 : uint32_t ts_high = raw_event.at(2); // NOLINT(build/unsigned)
184 0 : uint32_t data = raw_event.at(3); // NOLINT(build/unsigned)
185 0 : uint32_t trigger = raw_event.at(4); // NOLINT(build/unsigned)
186 :
187 : // put together the timestamp
188 0 : uint64_t ts = ts_low | (static_cast<uint64_t>(ts_high) << 32); // NOLINT(build/unsigned)
189 :
190 : // bits 31-16 contain the HSI device ID
191 0 : uint32_t hsi_device_id = header >> 16; // NOLINT(build/unsigned)
192 : // bits 15-0 contain the sequence counter
193 0 : uint32_t counter = header & 0x0000ffff; // NOLINT(build/unsigned)
194 :
195 0 : if ((header >> 16) != 0xaa00) {
196 0 : ers::error(InvalidHSIEventHeader(ERS_HERE,header));
197 0 : continue;
198 : }
199 :
200 0 : if (!ts)
201 : {
202 0 : ers::warning(InvalidHSIEventTimestamp(ERS_HERE,ts));
203 0 : continue;
204 : }
205 :
206 0 : if (counter > 0 && counter % 60000 == 0)
207 : {
208 0 : TLOG_DEBUG(3) << "Sequence counter from firmware: " << counter;
209 : }
210 :
211 0 : TLOG_DEBUG(3) << get_name() << ": read out data: " << std::showbase << std::hex << header << ", " << ts
212 0 : << ", " << data << ", " << std::bitset<32>(trigger) << ", "
213 0 : << "ts: " << ts << "\n";
214 :
215 : // In lieu of propper HSI channel to signal mapping, fake signal map when HSI firmware+hardware is in emulation mode.
216 : // TODO DAQ/HSI team 24/03/22 Put in place HSI channel to signal mapping.
217 :
218 0 : if (hsi_emulation_mode)
219 : {
220 0 : TLOG_DEBUG(3) << " HSI hardware is in emulation mode, faking (overwriting) signal map from firmware+hardware to have (only) bit 7 high.";
221 0 : trigger = 1UL << 7;
222 : }
223 :
224 0 : dfmessages::HSIEvent event = dfmessages::HSIEvent(hsi_device_id, trigger, ts, counter, m_run_number);
225 :
226 0 : m_last_readout_timestamp.store(ts);
227 :
228 0 : send_hsi_event(event);
229 :
230 : // Send raw HSI data to a DLH
231 0 : std::array<uint32_t, 7> hsi_struct;
232 0 : hsi_struct[0] = (0x1 << 6) | 0x1; // DAQHeader, frame version: 1, det id: 1
233 0 : hsi_struct[1] = ts_low;
234 0 : hsi_struct[2] = ts_high;
235 0 : hsi_struct[3] = data;
236 0 : hsi_struct[4] = 0x0;
237 0 : hsi_struct[5] = trigger;
238 0 : hsi_struct[6] = counter;
239 :
240 0 : TLOG_DEBUG(3) << get_name() << ": Formed HSI_FRAME_STRUCT "
241 0 : << std::hex
242 0 : << "0x" << hsi_struct[0]
243 0 : << ", 0x" << hsi_struct[1]
244 0 : << ", 0x" << hsi_struct[2]
245 0 : << ", 0x" << hsi_struct[3]
246 0 : << ", 0x" << hsi_struct[4]
247 0 : << ", 0x" << hsi_struct[5]
248 0 : << ", 0x" << hsi_struct[6]
249 0 : << "\n";
250 :
251 0 : send_raw_hsi_data(hsi_struct, m_raw_hsi_data_sender.get());
252 : }
253 : }
254 : // empty buffer is ok
255 0 : else if (hsi_words.size() == 0)
256 : {
257 0 : TLOG_DEBUG(20) << "Empty HSI buffter";
258 : }
259 : // anything else is unexpected
260 : else
261 : {
262 0 : ers::error(InvalidNumberReadoutHSIWords(ERS_HERE, hsi_words.size()));
263 : }
264 0 : std::this_thread::sleep_for(std::chrono::microseconds(m_readout_period));
265 0 : }
266 0 : std::ostringstream oss_summ;
267 0 : oss_summ << ": Exiting the read_hsievents() method, read out " << m_readout_counter.load()
268 0 : << " HSIEvent messages and successfully sent " << m_sent_counter.load() << " copies. ";
269 0 : ers::info(hsilibs::ProgressUpdate(ERS_HERE, get_name(), oss_summ.str()));
270 0 : TLOG_DEBUG(2) << get_name() << ": Exiting do_work() method";
271 0 : }
272 :
273 : void
274 0 : HSIReadout::update_buffer_counts(uint16_t new_count) // NOLINT(build/unsigned)
275 : {
276 0 : std::unique_lock mon_data_lock(m_buffer_counts_mutex);
277 0 : if (m_buffer_counts.size() > 1000)
278 0 : m_buffer_counts.pop_front();
279 0 : m_buffer_counts.push_back(new_count);
280 0 : }
281 :
282 : double
283 0 : HSIReadout::read_average_buffer_counts()
284 : {
285 0 : std::unique_lock mon_data_lock(m_buffer_counts_mutex);
286 :
287 0 : double total_counts;
288 0 : uint32_t number_of_counts; // NOLINT(build/unsigned)
289 :
290 0 : total_counts = 0;
291 0 : number_of_counts = m_buffer_counts.size();
292 :
293 0 : if (number_of_counts) {
294 0 : for (uint i = 0; i < number_of_counts; ++i) { // NOLINT(build/unsigned)
295 0 : total_counts = total_counts + m_buffer_counts.at(i);
296 : }
297 0 : return total_counts / number_of_counts;
298 : } else {
299 : return 0;
300 : }
301 0 : }
302 :
303 : void
304 0 : HSIReadout::generate_opmon_data()
305 : {
306 :
307 0 : opmon::HSIReadoutInfo info;
308 :
309 0 : info.set_readout_hsi_events_counter(m_readout_counter.load());
310 0 : info.set_sent_hsi_events_counter(m_sent_counter.load());
311 0 : info.set_failed_to_send_hsi_events_counter(m_failed_to_send_counter.load());
312 0 : info.set_last_readout_timestamp(m_last_readout_timestamp.load());
313 0 : info.set_last_sent_timestamp(m_last_sent_timestamp.load());
314 0 : info.set_average_buffer_occupancy(read_average_buffer_counts());
315 :
316 0 : publish(std::move(info));
317 :
318 0 : }
319 :
320 : } // namespace hsilibs
321 : } // namespace dunedaq
322 :
323 0 : DEFINE_DUNE_DAQ_MODULE(dunedaq::hsilibs::HSIReadout)
324 :
325 : // Local Variables:
326 : // c-basic-offset: 2
327 : // End:
|
