30#define TRACE_NAME "TimingHardwareManagerBase"
41 , m_hw_cmd_connection(
"timing_cmds")
42 , m_hw_command_receiver(nullptr)
43 , m_gather_interval(1e6)
44 , m_gather_interval_debug(10e6)
45 , m_monitored_device_name_master(
"")
46 , m_monitored_device_name_endpoint(
"")
47 , m_monitored_device_name_hsi(
"")
48 , m_received_hw_commands_counter{ 0 }
49 , m_accepted_hw_commands_counter{ 0 }
50 , m_rejected_hw_commands_counter{ 0 }
51 , m_failed_hw_commands_counter{ 0 }
52 , m_endpoint_scan_threads_clean_up_thread(nullptr)
63 m_params = mod_config->get_configuration();
66 for (
auto con : mod_config->get_inputs())
98 TLOG_DEBUG(3) << fanout->get_device() <<
": device, slot: " << fanout->get_fanout_slot() << std::endl;
117 auto time_of_scrap = std::chrono::high_resolution_clock::now();
120 auto now = std::chrono::high_resolution_clock::now();
121 auto ms_since_scrap = std::chrono::duration_cast<std::chrono::milliseconds>(
now - time_of_scrap);
122 TLOG_DEBUG(0) <<
"Have been waiting for " << ms_since_scrap.count() <<
" ms for " <<
m_command_threads.size() <<
" command threads to finish...";
123 std::this_thread::sleep_for(std::chrono::microseconds(250000));
142 if (!device_name.compare(
"")) {
144 message <<
"UHAL device name is an empty string";
145 throw UHALDeviceNameIssue(
ERS_HERE, message.str());
149 return dynamic_cast<const timing::TimingNode*
>(&hw_device_entry->second->getNode(
""));
151 TLOG_DEBUG(0) << get_name() <<
": hw device interface for: " << device_name
152 <<
" does not exist. I will try to create it.";
158 }
catch (
const uhal::exception::ConnectionUIDDoesNotExist&
exception) {
160 message <<
"UHAL device name not " << device_name <<
" in connections file";
164 TLOG_DEBUG(0) << get_name() <<
": hw device interface for: " << device_name <<
" successfully created.";
173 auto device_name = gatherer.get_device_name();
175 while (gatherer.run_gathering()) {
181 gatherer.collect_info_from_device(*design);
182 }
catch (
const std::exception& excpt) {
186 auto prev_gather_time = std::chrono::steady_clock::now();
187 auto next_gather_time = prev_gather_time + std::chrono::microseconds(gatherer.get_gather_interval());
190 auto slice_period = std::chrono::microseconds(10000);
191 auto next_slice_gather_time = prev_gather_time + slice_period;
193 bool break_flag =
false;
194 while (next_gather_time > next_slice_gather_time + slice_period) {
195 if (!gatherer.run_gathering()) {
196 TLOG_DEBUG(0) <<
"while waiting to gather data, negative run gatherer flag detected.";
200 std::this_thread::sleep_until(next_slice_gather_time);
201 next_slice_gather_time = next_slice_gather_time + slice_period;
203 if (break_flag ==
false) {
204 std::this_thread::sleep_until(next_gather_time);
212 std::string gatherer_name = device_name +
"_level_" + std::to_string(op_mon_level);
214 std::unique_ptr<InfoGatherer> gatherer = std::make_unique<InfoGatherer>(
220 TLOG_DEBUG(0) <<
"Registering info gatherer: " << gatherer_name;
221 m_info_gatherers.emplace(std::make_pair(gatherer_name, std::move(gatherer)));
223 TLOG() <<
"Skipping registration of " << gatherer_name <<
". Already exists.";
231 if (!device_name.compare(
"")) {
232 TLOG_DEBUG(0) << get_name() <<
" Starting all info gatherers";
234 it->second.get()->start_gathering_thread();
237 bool gatherer_found=
false;
239 TLOG_DEBUG(0) << get_name() <<
" Starting info gatherer: " << it->first;
240 it->second.get()->start_gathering_thread();
243 if (!gatherer_found)
ers::warning(AttemptedToControlNonExantInfoGatherer(
ERS_HERE,
"start", device_name));
251 if (!device_name.compare(
"")) {
252 TLOG_DEBUG(0) << get_name() <<
" Stopping all info gatherers";
254 it->second.get()->stop_gathering_thread();
257 bool gatherer_found=
false;
259 TLOG_DEBUG(0) << get_name() <<
" Stopping info gatherer: " << it->first;
260 it->second.get()->stop_gathering_thread();
263 if (!gatherer_found)
ers::warning(AttemptedToControlNonExantInfoGatherer(
ERS_HERE,
"stop", device_name));
267std::vector<std::string>
270 std::vector<std::string> running_gatherers;
273 TLOG_DEBUG(0) << get_name() <<
" Checking run state of info gatherer: " << it->first <<
", and the state is " << it->second.get()->run_gathering();
274 if (it->second.get()->run_gathering())
276 running_gatherers.push_back(it->first);
279 return running_gatherers;
287 std::ostringstream starting_stream;
288 starting_stream <<
": Executing process_hardware_command() callback.";
289 TLOG_DEBUG(0) << get_name() << starting_stream.str();
294 <<
", it is of type: " << timing_hw_cmd.
id <<
", targeting device: " << timing_hw_cmd.
device <<
", with payload: " << timing_hw_cmd.
payload.dump();
296 std::string hw_cmd_name = timing_hw_cmd.
id;
301 TLOG_DEBUG(0) <<
"Found hw cmd: " << hw_cmd_name;
303 std::invoke(cmd->second, timing_hw_cmd);
304 }
catch (
const std::exception&
exception) {
313 std::ostringstream exiting_stream;
316 TLOG_DEBUG(0) << get_name() << exiting_stream.str();
330 for (
auto& gatherer: running_hw_gatherers)
337 if (cmd_payload.
soft) {
339 design->soft_reset_io();
342 <<
" io reset, with supplied clk file: " << cmd_payload.
clock_config;
348 <<
" io reset, with supplied clk source: " << cmd_payload.
clock_source;
353 for (
auto& gatherer: running_hw_gatherers)
365 TLOG() << std::endl << design->get_status();
376 <<
" set timestamp, with supplied ts source: " << cmd_payload.
timestamp_source;
385 TLOG_DEBUG(0) << get_name() <<
": " << hw_cmd.
device <<
" master_endpoint_scan";
387 std::stringstream command_thread_uid;
388 auto t = std::time(
nullptr);
389 auto tm = *std::localtime(&t);
398 TLOG_DEBUG(1) <<
"Queuing: " << command_thread_uid.str();
400 auto thread_key = command_thread_uid.str();
412 for (
auto& endpoint_location : cmd_payload.
endpoints)
414 auto endpoint_address = endpoint_location.address;
415 auto fanout_slot = endpoint_location.fanout_slot;
416 auto sfp_slot = endpoint_location.sfp_slot;
420 TLOG_DEBUG(1) << get_name() <<
": " << hw_cmd.
device <<
" master_endpoint_scan starting: ept adr: " << endpoint_address <<
", ept sfp: " << sfp_slot <<
", fanout slot: " << fanout_slot;
429 if (fanout_slot >= 0)
436 catch(
const UHALDeviceClassIssue& e)
443 if (fanout_slot != 0)
450 catch(
const UHALDeviceClassIssue& e)
463 auto scan_result = master_design->get_master_node_plain()->scan_endpoint(endpoint_address,
true);
464 if (scan_result.alive)
466 auto current_rtt = scan_result.round_trip_time;
467 ers::info(EndpointRTTMeasurement(
ERS_HERE,fanout_slot,sfp_slot,endpoint_address,current_rtt));
471 if (previous_rtt != current_rtt)
475 ers::warning(ChangedEndpointRTTMeasurement(
ERS_HERE,fanout_slot,sfp_slot,endpoint_address,current_rtt,previous_rtt));
491 catch(std::exception& e)
494 master_design->get_master_node_plain()->switch_endpoint_sfp(endpoint_address,
false);
501 TLOG_DEBUG(0) <<
"Entering clean_endpoint_scan_threads()";
502 bool break_flag =
false;
507 if (
thread.second->joinable())
516 auto prev_clean_time = std::chrono::steady_clock::now();
517 auto next_clean_time = prev_clean_time + std::chrono::milliseconds(30);
520 auto flag_check_period = std::chrono::milliseconds(1);
521 auto next_flag_check_time = prev_clean_time + flag_check_period;
523 while (next_clean_time > next_flag_check_time + flag_check_period) {
525 TLOG_DEBUG(2) <<
"while waiting to clean up endpoint scan threads, negative run gatherer flag detected.";
529 std::this_thread::sleep_until(next_flag_check_time);
530 next_flag_check_time = next_flag_check_time + flag_check_period;
532 if (break_flag ==
false) {
533 std::this_thread::sleep_until(next_clean_time);
536 TLOG_DEBUG(0) <<
"Exiting clean_endpoint_scan_threads()";
543 TLOG_DEBUG(0) << get_name() <<
": " << hw_cmd.
device <<
" set endpoint delay";
559 <<
", parsed data: " << cmd_payload.
fl_cmd_id
590 design->get_endpoint_node_plain(cmd_payload.
endpoint_id)->disable();
613 design->get_hsi_node().reset_hsi();
625 design->configure_hsi(
635 design->get_hsi_node().start_hsi();
644 design->get_hsi_node().stop_hsi();
650 TLOG_DEBUG(0) << get_name() <<
": " << hw_cmd.
device <<
" hsi print status";
653 TLOG() << std::endl << design->get_hsi_node().get_status();
#define DUNE_DAQ_SERIALIZABLE(Type, typestring)
static std::shared_ptr< IOManager > get()
Base class for timing endpoint design nodes.
Base class for timing nodes.
std::unique_ptr< uhal::ConnectionManager > m_connection_manager
void configure_uhal(const dunedaq::timinglibs::dal::TimingHardwareInterfaceConf *mdal)
std::atomic< bool > m_run_endpoint_scan_cleanup_thread
const timing::TimingNode * get_timing_device_plain(const std::string &device_name)
void hsi_stop(const timingcmd::TimingHwCmd &hw_cmd)
std::atomic< uint64_t > m_accepted_hw_commands_counter
void io_reset(const timingcmd::TimingHwCmd &hw_cmd)
void print_status(const timingcmd::TimingHwCmd &hw_cmd)
void register_info_gatherer(uint gather_interval, const std::string &device_name, int op_mon_level)
std::map< uint, int > m_monitored_endpoints_round_trip_times
TIMING_DEV get_timing_device(const std::string &device_name)
std::map< std::string, std::unique_ptr< uhal::HwInterface > > m_hw_device_map
virtual void perform_endpoint_scan(const timingcmd::TimingHwCmd &hw_cmd)
std::atomic< uint64_t > m_received_hw_commands_counter
std::map< uint, std::string > m_monitored_device_names_fanout
virtual void clean_endpoint_scan_threads()
void endpoint_reset(const timingcmd::TimingHwCmd &hw_cmd)
void init(std::shared_ptr< appfwk::ConfigurationManager > mcfg) override
std::shared_ptr< source_t > m_hw_command_receiver
uint m_gather_interval_debug
virtual void do_scrap(const nlohmann::json &)
void hsi_reset(const timingcmd::TimingHwCmd &hw_cmd)
void set_endpoint_delay(const timingcmd::TimingHwCmd &hw_cmd)
virtual void process_hardware_command(timingcmd::TimingHwCmd &timing_hw_cmd)
std::unique_ptr< dunedaq::utilities::ReusableThread > m_endpoint_scan_threads_clean_up_thread
std::atomic< uint64_t > m_rejected_hw_commands_counter
std::atomic< uint64_t > m_failed_hw_commands_counter
void gather_monitor_data(InfoGatherer &gatherer)
void hsi_configure(const timingcmd::TimingHwCmd &hw_cmd)
std::string m_monitored_device_name_endpoint
void set_timestamp(const timingcmd::TimingHwCmd &hw_cmd)
virtual void start_hw_mon_gathering(const std::string &device_name="")
std::string m_monitored_device_name_master
virtual void stop_hw_mon_gathering(const std::string &device_name="")
void hsi_print_status(const timingcmd::TimingHwCmd &hw_cmd)
void send_fl_cmd(const timingcmd::TimingHwCmd &hw_cmd)
std::mutex m_hw_device_map_mutex
std::string m_monitored_device_name_hsi
std::mutex master_sfp_mutex
void hsi_start(const timingcmd::TimingHwCmd &hw_cmd)
std::map< std::string, std::unique_ptr< std::thread > > m_command_threads
std::map< timingcmd::TimingHwCmdId, std::function< void(const timingcmd::TimingHwCmd &)> > m_timing_hw_cmd_map_
std::mutex m_command_threads_map_mutex
std::string m_hw_cmd_connection
void endpoint_enable(const timingcmd::TimingHwCmd &hw_cmd)
TimingHardwareManagerBase(const std::string &name)
TimingHardwareManagerBase Constructor.
void endpoint_disable(const timingcmd::TimingHwCmd &hw_cmd)
const timinglibs::dal::TimingHardwareManagerConf * m_params
virtual std::vector< std::string > check_hw_mon_gatherer_is_running(const std::string &device_name)
virtual void conf(const nlohmann::json &data)
void master_endpoint_scan(const timingcmd::TimingHwCmd &hw_cmd)
std::map< std::string, std::unique_ptr< InfoGatherer > > m_info_gatherers
uint32_t get_gather_interval() const
Get "gather_interval" attribute value. Hardware device data gather interval [us].
const std::string & get_monitored_device_name_hsi() const
Get "monitored_device_name_hsi" attribute value. Name of hsi device to be monitored.
const std::string & get_monitored_device_name_endpoint() const
Get "monitored_device_name_endpoint" attribute value. Name of timing endpoint device to be monitored.
uint32_t get_gather_interval_debug() const
Get "gather_interval_debug" attribute value. Hardware device data gather debug interval [us].
const std::string & get_monitored_device_name_master() const
Get "monitored_device_name_master" attribute value. Name of timing master device to be monitored.
const std::vector< const dunedaq::timinglibs::dal::TimingFanoutDevice * > & get_monitored_device_names_fanout() const
Get "monitored_device_names_fanout" relationship value. Timing fanout devices to be monitored.
Base class for any user define issue.
caught dunedaq::conffwk::Exception exception
#define TLOG_DEBUG(lvl,...)
void from_json(const data_t &j, EndpointLocation &obj)
Unknown serialization Cannot deserialize std::string datatype_to_string()
Unknown serialization Cannot deserialize message
void warning(const Issue &issue)
void info(const Issue &issue)
void error(const Issue &issue)
UintData timestamp_source
TimingHwCmdPayload payload
TimingEndpointLocations endpoints
UintData number_of_commands_to_send