Line data Source code
1 : /**
2 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
3 : * Licensing/copyright details are in the COPYING file that you should have
4 : * received with this code.
5 : *
6 : */
7 :
8 : #include "hdf5libs/HDF5RawDataFile.hpp"
9 :
10 : #include "logging/Logging.hpp"
11 :
12 : #include <algorithm>
13 : #include <filesystem>
14 : #include <map>
15 : #include <memory>
16 : #include <set>
17 : #include <sstream>
18 : #include <string>
19 : #include <utility>
20 : #include <vector>
21 :
22 : namespace dunedaq {
23 : namespace hdf5libs {
24 :
25 : constexpr uint32_t MAX_FILELAYOUT_VERSION = 4294967295; // NOLINT(build/unsigned)
26 :
27 : /**
28 : * @brief Constructor for writing a new file
29 : */
30 29 : HDF5RawDataFile::HDF5RawDataFile(std::string file_name,
31 : daqdataformats::run_number_t run_number,
32 : size_t file_index,
33 : std::string application_name,
34 : HDF5FileLayoutParameters fl_params,
35 : HDF5SourceIDHandler::source_id_geo_id_map_t srcid_geoid_map,
36 : unsigned compression_level,
37 : std::string inprogress_filename_suffix,
38 29 : unsigned open_flags)
39 29 : : m_bare_file_name(file_name)
40 29 : , m_compression_level(compression_level)
41 58 : , m_open_flags(open_flags)
42 : {
43 :
44 : // check and make sure that the file isn't ReadOnly
45 29 : if (m_open_flags == HighFive::File::ReadOnly) {
46 0 : throw IncompatibleOpenFlags(ERS_HERE, file_name, m_open_flags);
47 : }
48 :
49 29 : auto filename_to_open = m_bare_file_name + inprogress_filename_suffix;
50 :
51 : // do the file open
52 29 : try {
53 29 : m_file_ptr.reset(new HighFive::File(filename_to_open, m_open_flags));
54 0 : } catch (std::exception const& excpt) {
55 0 : throw FileOpenFailed(ERS_HERE, filename_to_open, excpt.what());
56 0 : }
57 :
58 29 : m_recorded_size = 0;
59 29 : m_uncompressed_raw_data_size = 0;
60 29 : m_total_file_size = 0;
61 :
62 29 : size_t file_creation_timestamp =
63 29 : std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
64 :
65 29 : TLOG_DEBUG(TLVL_BASIC) << "Created HDF5 file (" << file_name << ") at time " << file_creation_timestamp << " .";
66 :
67 : // write some file attributes
68 29 : write_attribute("run_number", run_number);
69 29 : write_attribute("file_index", file_index);
70 29 : write_attribute("creation_timestamp", file_creation_timestamp);
71 29 : write_attribute("application_name", application_name);
72 :
73 : // set the file layout contents
74 29 : m_file_layout_ptr.reset(new HDF5FileLayout(fl_params));
75 29 : write_file_layout();
76 :
77 : // write the SourceID-related attributes
78 : //HDF5SourceIDHandler::populate_source_id_geo_id_map(srcid_geoid_map, m_file_level_source_id_geo_id_map);
79 29 : m_file_level_source_id_geo_id_map = srcid_geoid_map;
80 29 : HDF5SourceIDHandler::store_file_level_geo_id_info(*m_file_ptr, m_file_level_source_id_geo_id_map);
81 :
82 : // write the record type
83 29 : m_record_type = fl_params.record_name_prefix;
84 29 : write_attribute("record_type", m_record_type);
85 :
86 : // write the compression level
87 29 : write_attribute("compression_level", m_compression_level);
88 29 : }
89 :
90 :
91 43 : HDF5RawDataFile::~HDF5RawDataFile()
92 : {
93 43 : if (m_file_ptr.get() != nullptr && m_open_flags != HighFive::File::ReadOnly) {
94 29 : if (! m_file_ptr->hasAttribute("recorded_size")) {
95 29 : write_attribute("recorded_size", m_recorded_size);
96 : }
97 :
98 29 : if (! m_file_ptr->hasAttribute("uncompressed_raw_data_size")) {
99 29 : write_attribute("uncompressed_raw_data_size", m_uncompressed_raw_data_size);
100 : }
101 :
102 29 : if (! m_file_ptr->hasAttribute("total_file_size")) {
103 29 : write_attribute("total_file_size", m_total_file_size);
104 : }
105 :
106 29 : if (! m_file_ptr->hasAttribute("closing_timestamp")) {
107 29 : size_t file_closing_timestamp =
108 29 : std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
109 29 : write_attribute("closing_timestamp", file_closing_timestamp);
110 : }
111 :
112 29 : m_file_ptr->flush();
113 :
114 : // rename file to the bare name, if needed
115 29 : if (m_file_ptr->getName() != m_bare_file_name) {
116 29 : std::filesystem::rename(m_file_ptr->getName(), m_bare_file_name);
117 : }
118 : }
119 :
120 : // explicit destruction; not really needed, but nice to be clear...
121 43 : m_file_ptr.reset();
122 43 : m_file_layout_ptr.reset();
123 43 : }
124 :
125 : /**
126 : * @brief Fetch the list of all file-level Attribute names.
127 : */
128 : std::vector<std::string>
129 0 : HDF5RawDataFile::HDF5RawDataFile::get_attribute_names()
130 : {
131 0 : return m_file_ptr->listAttributeNames();
132 : }
133 :
134 : /**
135 : * @brief Write a TriggerRecord to the file.
136 : */
137 : void
138 75 : HDF5RawDataFile::write(const daqdataformats::TriggerRecord& tr)
139 : {
140 : // the source_id_path map that we will build up as we write the TR header
141 : // and fragments (and then write the map into the HDF5 TR_record Group)
142 75 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
143 :
144 : // the map of fragment types to SourceIDS
145 75 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
146 :
147 : // the map of subdetectors to SourceIDS
148 75 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
149 :
150 : // write the record header into the HDF5 file/group
151 75 : HighFive::Group record_level_group = write(tr.get_header_ref(), source_id_path_map);
152 :
153 : // store the SourceID of the record header in the HDF5 file/group
154 : // (since there should only be one entry in the map at this point, we'll take advantage of that...)
155 150 : for (auto const& source_id_path : source_id_path_map) {
156 75 : HDF5SourceIDHandler::store_record_header_source_id(record_level_group, source_id_path.first);
157 : }
158 :
159 : // write all of the fragments into the HDF5 file/group
160 1415 : for (auto const& frag_ptr : tr.get_fragments_ref()) {
161 1340 : write(*frag_ptr, source_id_path_map);
162 1340 : HDF5SourceIDHandler::add_fragment_type_source_id_to_map(
163 1340 : fragment_type_source_id_map, frag_ptr->get_fragment_type(), frag_ptr->get_element_id());
164 1340 : HDF5SourceIDHandler::add_subdetector_source_id_to_map(
165 : subdetector_source_id_map,
166 1340 : static_cast<detdataformats::DetID::Subdetector>(frag_ptr->get_detector_id()),
167 1340 : frag_ptr->get_element_id());
168 : }
169 :
170 : // store all of the record-level maps in the HDF5 file/group
171 75 : HDF5SourceIDHandler::store_record_level_path_info(record_level_group, source_id_path_map);
172 75 : HDF5SourceIDHandler::store_record_level_fragment_type_map(record_level_group, fragment_type_source_id_map);
173 75 : HDF5SourceIDHandler::store_record_level_subdetector_map(record_level_group, subdetector_source_id_map);
174 75 : }
175 :
176 : /**
177 : * @brief Write a TimeSlice to the file.
178 : */
179 : void
180 30 : HDF5RawDataFile::write(const daqdataformats::TimeSlice& ts)
181 : {
182 30 : std::string tsh_path = m_file_layout_ptr->get_path_string(ts.get_header());
183 30 : if (m_file_ptr->exist(tsh_path)) {throw TimeSliceAlreadyExists(ERS_HERE, tsh_path);}
184 :
185 : // the source_id_path map that we will build up as we write the TR header
186 : // and fragments (and then write the map into the HDF5 TR_record Group)
187 30 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
188 :
189 : // the map of fragment types to SourceIDS
190 30 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
191 :
192 : // the map of subdetectors to SourceIDS
193 30 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
194 :
195 : // write the record header into the HDF5 file/group
196 30 : HighFive::Group record_level_group = write(ts.get_header(), source_id_path_map);
197 :
198 : // store the SourceID of the record header in the HDF5 file/group
199 : // (since there should only be one entry in the map at this point, we'll take advantage of that...)
200 60 : for (auto const& source_id_path : source_id_path_map) {
201 30 : HDF5SourceIDHandler::store_record_header_source_id(record_level_group, source_id_path.first);
202 : }
203 :
204 : // write all of the fragments into the HDF5 file/group
205 270 : for (auto const& frag_ptr : ts.get_fragments_ref()) {
206 240 : write(*frag_ptr, source_id_path_map);
207 240 : HDF5SourceIDHandler::add_fragment_type_source_id_to_map(
208 240 : fragment_type_source_id_map, frag_ptr->get_fragment_type(), frag_ptr->get_element_id());
209 240 : HDF5SourceIDHandler::add_subdetector_source_id_to_map(
210 : subdetector_source_id_map,
211 240 : static_cast<detdataformats::DetID::Subdetector>(frag_ptr->get_detector_id()),
212 240 : frag_ptr->get_element_id());
213 : }
214 :
215 : // store all of the record-level maps in the HDF5 file/group
216 30 : HDF5SourceIDHandler::store_record_level_path_info(record_level_group, source_id_path_map);
217 30 : HDF5SourceIDHandler::store_record_level_fragment_type_map(record_level_group, fragment_type_source_id_map);
218 30 : HDF5SourceIDHandler::store_record_level_subdetector_map(record_level_group, subdetector_source_id_map);
219 30 : }
220 :
221 : /**
222 : * @brief Write a TriggerRecordHeader to the file.
223 : */
224 : HighFive::Group
225 75 : HDF5RawDataFile::write(const daqdataformats::TriggerRecordHeader& trh,
226 : HDF5SourceIDHandler::source_id_path_map_t& path_map)
227 : {
228 75 : std::tuple<size_t, std::string, HighFive::Group> write_results =
229 75 : do_write(m_file_layout_ptr->get_path_elements(trh),
230 75 : static_cast<const char*>(trh.get_storage_location()),
231 : trh.get_total_size_bytes(),
232 75 : m_compression_level);
233 75 : m_recorded_size += std::get<0>(write_results);
234 75 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, trh.get_header().element_id, std::get<1>(write_results));
235 150 : return std::get<2>(write_results);
236 75 : }
237 :
238 : /**
239 : * @brief Write a TimeSliceHeader to the file.
240 : */
241 : HighFive::Group
242 30 : HDF5RawDataFile::write(const daqdataformats::TimeSliceHeader& tsh, HDF5SourceIDHandler::source_id_path_map_t& path_map)
243 : {
244 30 : std::tuple<size_t, std::string, HighFive::Group> write_results =
245 30 : do_write(m_file_layout_ptr->get_path_elements(tsh),
246 : (const char*)(&tsh),
247 : sizeof(daqdataformats::TimeSliceHeader),
248 30 : m_compression_level);
249 30 : m_recorded_size += std::get<0>(write_results);
250 30 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, tsh.element_id, std::get<1>(write_results));
251 60 : return std::get<2>(write_results);
252 30 : }
253 :
254 : /**
255 : * @brief Write a Fragment to the file.
256 : */
257 : void
258 1580 : HDF5RawDataFile::write(const daqdataformats::Fragment& frag, HDF5SourceIDHandler::source_id_path_map_t& path_map)
259 : {
260 1580 : std::tuple<size_t, std::string, HighFive::Group> write_results =
261 1580 : do_write(m_file_layout_ptr->get_path_elements(frag.get_header()),
262 1580 : static_cast<const char*>(frag.get_storage_location()),
263 : frag.get_size(),
264 1580 : m_compression_level);
265 1580 : m_recorded_size += std::get<0>(write_results);
266 :
267 1580 : daqdataformats::SourceID source_id = frag.get_element_id();
268 1580 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, source_id, std::get<1>(write_results));
269 1580 : }
270 :
271 : /**
272 : * @brief write the file layout
273 : */
274 : void
275 29 : HDF5RawDataFile::write_file_layout()
276 : {
277 29 : auto fl_json = m_file_layout_ptr->get_file_layout_params().to_json();
278 29 : write_attribute("filelayout_params", fl_json.dump());
279 29 : write_attribute("filelayout_version", m_file_layout_ptr->get_version());
280 29 : }
281 :
282 : /**
283 : * @brief write bytes to a dataset in the file, at the appropriate path
284 : */
285 : std::tuple<size_t, std::string, HighFive::Group>
286 1685 : HDF5RawDataFile::do_write(std::vector<std::string> const& group_and_dataset_path_elements,
287 : const char* raw_data_ptr,
288 : size_t raw_data_size_bytes,
289 : unsigned compression_level)
290 : {
291 1685 : const std::string dataset_name = group_and_dataset_path_elements.back();
292 :
293 : // create top level group if needed
294 1685 : std::string const& top_level_group_name = group_and_dataset_path_elements.at(0);
295 1685 : if (!m_file_ptr->exist(top_level_group_name))
296 105 : m_file_ptr->createGroup(top_level_group_name);
297 :
298 : // setup sub_group to work with
299 1685 : HighFive::Group sub_group = m_file_ptr->getGroup(top_level_group_name);
300 1685 : if (!sub_group.isValid()) {
301 0 : throw InvalidHDF5Group(ERS_HERE, top_level_group_name);
302 : }
303 1685 : HighFive::Group top_level_group = sub_group;
304 :
305 : // Create the remaining subgroups
306 3370 : for (size_t idx = 1; idx < group_and_dataset_path_elements.size() - 1; ++idx) {
307 : // group_dataset.size()-1 because the last element is the dataset
308 1685 : std::string const& child_group_name = group_and_dataset_path_elements[idx];
309 1685 : if (child_group_name.empty()) {
310 0 : throw InvalidHDF5Group(ERS_HERE, child_group_name);
311 : }
312 1685 : if (!sub_group.exist(child_group_name)) {
313 105 : sub_group.createGroup(child_group_name);
314 : }
315 1685 : HighFive::Group child_group = sub_group.getGroup(child_group_name);
316 1685 : if (!child_group.isValid()) {
317 0 : throw InvalidHDF5Group(ERS_HERE, child_group_name);
318 : }
319 1685 : sub_group = child_group;
320 1685 : }
321 :
322 : // Create dataset
323 1685 : HighFive::DataSpace data_space = HighFive::DataSpace({ raw_data_size_bytes, 1 });
324 1685 : HighFive::DataSetCreateProps data_set_create_props;
325 1685 : HighFive::DataSetAccessProps data_set_access_props;
326 :
327 1685 : if (compression_level > 0) {
328 415 : std::vector<hsize_t> chunk_size = {raw_data_size_bytes, 1};
329 415 : data_set_create_props.add(HighFive::Chunking(chunk_size));
330 415 : data_set_create_props.add(HighFive::Deflate(compression_level));
331 415 : }
332 :
333 1685 : m_uncompressed_raw_data_size += raw_data_size_bytes;
334 :
335 1685 : auto data_set = sub_group.createDataSet<char>(dataset_name, data_space, data_set_create_props, data_set_access_props);
336 :
337 1685 : if (data_set.isValid()) {
338 1685 : data_set.write_raw(raw_data_ptr);
339 1685 : m_total_file_size = m_file_ptr->getFileSize();
340 1685 : m_file_ptr->flush();
341 3370 : return std::make_tuple(data_set.getStorageSize(), data_set.getPath(), top_level_group);
342 : } else {
343 0 : throw InvalidHDF5Dataset(ERS_HERE, dataset_name, m_file_ptr->getName());
344 : }
345 1685 : }
346 :
347 : /**
348 : * @brief Constructor for reading (and optionally writing) a file
349 : */
350 14 : HDF5RawDataFile::HDF5RawDataFile(const std::string& file_name, bool allow_writing)
351 14 : : m_open_flags(HighFive::File::ReadOnly)
352 : {
353 14 : if (allow_writing) {m_open_flags = HighFive::File::ReadWrite;}
354 14 : m_bare_file_name = file_name;
355 14 : size_t pos = m_bare_file_name.rfind(s_inprogress_suffix);
356 14 : if (pos != std::string::npos) {
357 0 : m_bare_file_name.erase(pos);
358 : }
359 :
360 : // do the file open
361 14 : try {
362 14 : m_file_ptr = std::make_unique<HighFive::File>(file_name, m_open_flags);
363 0 : } catch (std::exception const& excpt) {
364 0 : throw FileOpenFailed(ERS_HERE, file_name, excpt.what());
365 0 : }
366 :
367 14 : if (m_file_ptr->hasAttribute("recorded_size"))
368 14 : m_recorded_size = get_attribute<size_t>("recorded_size");
369 : else
370 0 : m_recorded_size = 0;
371 :
372 14 : if (m_file_ptr->hasAttribute("uncompressed_raw_data_size"))
373 14 : m_uncompressed_raw_data_size = get_attribute<size_t>("uncompressed_raw_data_size");
374 : else
375 0 : m_uncompressed_raw_data_size = 0;
376 :
377 14 : if (m_file_ptr->hasAttribute("total_file_size"))
378 14 : m_total_file_size = get_attribute<size_t>("total_file_size");
379 : else
380 0 : m_total_file_size = 0;
381 :
382 14 : if (m_file_ptr->hasAttribute("compression_level"))
383 14 : m_compression_level = get_attribute<unsigned>("compression_level");
384 : else
385 0 : m_compression_level = 0;
386 :
387 14 : read_file_layout();
388 :
389 14 : if (m_file_ptr->hasAttribute("record_type"))
390 14 : m_record_type = get_attribute<std::string>("record_type");
391 : else
392 0 : m_record_type = m_file_layout_ptr->get_record_name_prefix();
393 :
394 14 : check_file_layout();
395 :
396 : // HDF5SourceIDHandler operations need to come *after* read_file_layout()
397 : // because they count on the filelayout_version, which is set in read_file_layout().
398 14 : HDF5SourceIDHandler sid_handler(get_version());
399 14 : sid_handler.fetch_file_level_geo_id_info(*m_file_ptr, m_file_level_source_id_geo_id_map);
400 14 : }
401 :
402 : void
403 14 : HDF5RawDataFile::read_file_layout()
404 : {
405 14 : HDF5FileLayoutParameters fl_params;
406 14 : uint32_t version = 0; // NOLINT(build/unsigned)
407 :
408 14 : std::string fl_str;
409 14 : try {
410 14 : fl_str = get_attribute<std::string>("filelayout_params");
411 14 : nlohmann::json fl_json = nlohmann::json::parse(fl_str);
412 14 : fl_params = HDF5FileLayoutParameters(fl_json);
413 :
414 14 : version = get_attribute<uint32_t>("filelayout_version"); // NOLINT(build/unsigned)
415 :
416 14 : } catch (InvalidHDF5Attribute const&) {
417 0 : ers::info(MissingFileLayout(ERS_HERE, version));
418 0 : }
419 :
420 : // now reset the HDF5Filelayout object
421 14 : m_file_layout_ptr.reset(new HDF5FileLayout(fl_params, version));
422 14 : }
423 :
424 : void
425 14 : HDF5RawDataFile::check_file_layout()
426 : {
427 14 : if (get_version() < 2)
428 0 : return;
429 :
430 14 : std::string record_type = get_attribute<std::string>("record_type");
431 14 : if (record_type.compare(m_file_layout_ptr->get_record_name_prefix()) != 0)
432 0 : throw BadRecordType(ERS_HERE, record_type, m_file_layout_ptr->get_record_name_prefix());
433 14 : }
434 :
435 : void
436 18 : HDF5RawDataFile::check_record_type(std::string rt_name)
437 : {
438 18 : if (get_version() < 2)
439 : return;
440 :
441 18 : if (m_file_layout_ptr->get_record_name_prefix().compare(rt_name) != 0)
442 0 : throw WrongRecordTypeRequested(ERS_HERE, rt_name, m_file_layout_ptr->get_record_name_prefix());
443 : }
444 :
445 : // HDF5 Utility function to recursively traverse a file
446 : void
447 176 : HDF5RawDataFile::explore_subgroup(const HighFive::Group& parent_group,
448 : std::string relative_path,
449 : std::vector<std::string>& path_list)
450 : {
451 176 : if (relative_path.size() > 0 && relative_path.compare(relative_path.size() - 1, 1, "/") == 0)
452 16 : relative_path.pop_back();
453 :
454 176 : std::vector<std::string> childNames = parent_group.listObjectNames();
455 :
456 1256 : for (auto& child_name : childNames) {
457 1080 : std::string full_path = relative_path + "/" + child_name;
458 1080 : HighFive::ObjectType child_type = parent_group.getObjectType(child_name);
459 :
460 1080 : if (child_type == HighFive::ObjectType::Dataset) {
461 920 : path_list.push_back(full_path);
462 160 : } else if (child_type == HighFive::ObjectType::Group) {
463 160 : HighFive::Group child_group = parent_group.getGroup(child_name);
464 : // start the recusion
465 160 : std::string new_path = relative_path + "/" + child_name;
466 160 : explore_subgroup(child_group, new_path, path_list);
467 160 : }
468 1080 : }
469 176 : }
470 :
471 : void
472 72 : HDF5RawDataFile::add_record_level_info_to_caches_if_needed(record_id_t rid)
473 : {
474 : // we should probably check that all relevant caches have an entry for the
475 : // specified record ID, but we will just check one, in the interest of
476 : // performance, and trust the "else" part of this routine to fill in *all*
477 : // of the appropriate caches
478 72 : if (m_source_id_path_cache.count(rid) != 0) {
479 32 : return;
480 : }
481 :
482 : // create the handler to do the work
483 40 : HDF5SourceIDHandler sid_handler(get_version());
484 :
485 : // determine the HDF5 Group that corresponds to the specified record
486 40 : std::string record_level_group_name = m_file_layout_ptr->get_record_number_string(rid.first, rid.second);
487 40 : HighFive::Group record_group = m_file_ptr->getGroup(record_level_group_name);
488 40 : if (!record_group.isValid()) {
489 0 : throw InvalidHDF5Group(ERS_HERE, record_level_group_name);
490 : }
491 :
492 : // start with a copy of the file-level source-id-to-geo-id map and give the
493 : // handler an opportunity to add any record-level additions
494 40 : HDF5SourceIDHandler::source_id_geo_id_map_t local_source_id_geo_id_map = m_file_level_source_id_geo_id_map;
495 40 : sid_handler.fetch_record_level_geo_id_info(record_group, local_source_id_geo_id_map);
496 :
497 : // fetch the record-level source-id-to-path map
498 40 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
499 40 : sid_handler.fetch_source_id_path_info(record_group, source_id_path_map);
500 :
501 : // fetch the record-level fragment-type-to-source-id map
502 40 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
503 40 : sid_handler.fetch_fragment_type_source_id_info(record_group, fragment_type_source_id_map);
504 :
505 : // fetch the record-level subdetector-to-source-id map
506 40 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
507 40 : sid_handler.fetch_subdetector_source_id_info(record_group, subdetector_source_id_map);
508 :
509 : // loop through the source-id-to-path map to create various lists of SourceIDs in the record
510 40 : daqdataformats::SourceID rh_sid = sid_handler.fetch_record_header_source_id(record_group);
511 40 : std::set<daqdataformats::SourceID> full_source_id_set;
512 40 : std::set<daqdataformats::SourceID> fragment_source_id_set;
513 40 : HDF5SourceIDHandler::subsystem_source_id_map_t subsystem_source_id_map;
514 500 : for (auto const& source_id_path : source_id_path_map) {
515 460 : full_source_id_set.insert(source_id_path.first);
516 460 : if (source_id_path.first != rh_sid) {
517 420 : fragment_source_id_set.insert(source_id_path.first);
518 : }
519 460 : HDF5SourceIDHandler::add_subsystem_source_id_to_map(
520 460 : subsystem_source_id_map, source_id_path.first.subsystem, source_id_path.first);
521 : }
522 :
523 : // note that even if the "fetch" methods above fail to add anything to the specified
524 : // maps, the maps will still be valid (though, possibly empty), and once we add them
525 : // to the caches here, we will be assured that lookups from the caches will not fail.
526 40 : m_source_id_cache[rid] = full_source_id_set;
527 40 : m_record_header_source_id_cache[rid] = rh_sid;
528 40 : m_fragment_source_id_cache[rid] = fragment_source_id_set;
529 40 : m_source_id_geo_id_cache[rid] = local_source_id_geo_id_map;
530 40 : m_source_id_path_cache[rid] = source_id_path_map;
531 40 : m_subsystem_source_id_cache[rid] = subsystem_source_id_map;
532 40 : m_fragment_type_source_id_cache[rid] = fragment_type_source_id_map;
533 40 : m_subdetector_source_id_cache[rid] = subdetector_source_id_map;
534 40 : }
535 :
536 : /**
537 : * @brief Return a vector of dataset names
538 : */
539 : std::vector<std::string>
540 16 : HDF5RawDataFile::get_dataset_paths(std::string top_level_group_name)
541 : {
542 16 : if (top_level_group_name.empty())
543 16 : top_level_group_name = m_file_ptr->getPath();
544 :
545 : // Vector containing the path list to the HDF5 datasets
546 16 : std::vector<std::string> path_list;
547 :
548 16 : HighFive::Group parent_group = m_file_ptr->getGroup(top_level_group_name);
549 16 : if (!parent_group.isValid())
550 0 : throw InvalidHDF5Group(ERS_HERE, top_level_group_name);
551 :
552 16 : explore_subgroup(parent_group, top_level_group_name, path_list);
553 :
554 16 : return path_list;
555 16 : }
556 :
557 : /**
558 : * @brief Return all of the record numbers in the file.
559 : */
560 : HDF5RawDataFile::record_id_set // NOLINT(build/unsigned)
561 242 : HDF5RawDataFile::get_all_record_ids()
562 : {
563 242 : if (!m_all_record_ids_in_file.empty())
564 232 : return m_all_record_ids_in_file;
565 :
566 : // records are at the top level
567 :
568 10 : HighFive::Group parent_group = m_file_ptr->getGroup(m_file_ptr->getPath());
569 :
570 10 : std::vector<std::string> childNames = parent_group.listObjectNames();
571 10 : const std::string record_prefix = m_file_layout_ptr->get_record_name_prefix();
572 10 : const size_t record_prefix_size = record_prefix.size();
573 :
574 60 : for (auto const& name : childNames) {
575 50 : auto loc = name.find(record_prefix);
576 :
577 50 : if (loc == std::string::npos)
578 0 : continue;
579 :
580 50 : auto rec_num_string = name.substr(loc + record_prefix_size);
581 :
582 50 : loc = rec_num_string.find(".");
583 50 : if (loc == std::string::npos) {
584 20 : m_all_record_ids_in_file.insert(std::make_pair(std::stoll(rec_num_string), 0));
585 : } else {
586 30 : auto seq_num_string = rec_num_string.substr(loc + 1);
587 30 : rec_num_string.resize(loc); // remove anything from '.' onwards
588 30 : m_all_record_ids_in_file.insert(std::make_pair(std::stoll(rec_num_string), std::stoi(seq_num_string)));
589 30 : }
590 :
591 50 : } // end loop over childNames
592 :
593 10 : return m_all_record_ids_in_file;
594 10 : }
595 :
596 : std::set<uint64_t>
597 4 : HDF5RawDataFile::get_all_record_numbers() // NOLINT(build/unsigned)
598 : {
599 8 : ers::warning(DeprecatedUsage(ERS_HERE,
600 : "get_all_record_numbers()",
601 8 : "Use get_all_record_ids(), which returns a record_number,sequence_number pair."));
602 :
603 4 : std::set<uint64_t> record_numbers; // NOLINT(build/unsigned)
604 24 : for (auto const& rid : get_all_record_ids())
605 24 : record_numbers.insert(rid.first);
606 :
607 4 : return record_numbers;
608 0 : }
609 :
610 : HDF5RawDataFile::record_id_set
611 6 : HDF5RawDataFile::get_all_trigger_record_ids()
612 : {
613 6 : check_record_type("TriggerRecord");
614 6 : return get_all_record_ids();
615 : }
616 :
617 : std::set<daqdataformats::trigger_number_t>
618 0 : HDF5RawDataFile::get_all_trigger_record_numbers()
619 : {
620 0 : ers::warning(
621 0 : DeprecatedUsage(ERS_HERE,
622 : "get_all_trigger_record_numbers()",
623 0 : "Use get_all_trigger_record_ids(), which returns a record_number,sequence_number pair."));
624 :
625 0 : return get_all_record_numbers();
626 : }
627 :
628 : HDF5RawDataFile::record_id_set
629 0 : HDF5RawDataFile::get_all_timeslice_ids()
630 : {
631 0 : check_record_type("TimeSlice");
632 0 : return get_all_record_ids();
633 : }
634 :
635 : std::set<daqdataformats::timeslice_number_t>
636 4 : HDF5RawDataFile::get_all_timeslice_numbers()
637 : {
638 4 : check_record_type("TimeSlice");
639 4 : return get_all_record_numbers();
640 : }
641 :
642 : /**
643 : * @brief Return a vector of dataset names that correspond to record headers
644 : */
645 : std::vector<std::string>
646 8 : HDF5RawDataFile::get_record_header_dataset_paths()
647 : {
648 :
649 8 : std::vector<std::string> rec_paths;
650 :
651 8 : if (get_version() >= 2) {
652 48 : for (auto const& rec_id : get_all_record_ids())
653 48 : rec_paths.push_back(get_record_header_dataset_path(rec_id));
654 : } else {
655 0 : for (auto const& path : get_dataset_paths()) {
656 0 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) != std::string::npos) {
657 0 : rec_paths.push_back(path);
658 : }
659 0 : }
660 : }
661 :
662 8 : return rec_paths;
663 0 : }
664 :
665 : std::vector<std::string>
666 4 : HDF5RawDataFile::get_trigger_record_header_dataset_paths()
667 : {
668 4 : check_record_type("TriggerRecord");
669 4 : return get_record_header_dataset_paths();
670 : }
671 :
672 : std::vector<std::string>
673 4 : HDF5RawDataFile::get_timeslice_header_dataset_paths()
674 : {
675 4 : check_record_type("TimeSlice");
676 4 : return get_record_header_dataset_paths();
677 : }
678 :
679 : std::string
680 40 : HDF5RawDataFile::get_record_header_dataset_path(const record_id_t& rid)
681 : {
682 40 : auto rec_id = get_all_record_ids().find(rid);
683 40 : if (rec_id == get_all_record_ids().end())
684 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
685 :
686 40 : if (get_version() <= 2) {
687 0 : return (m_file_ptr->getPath() + m_file_layout_ptr->get_record_header_path(rid.first, rid.second));
688 : } else {
689 40 : daqdataformats::SourceID source_id = get_record_header_source_id(rid);
690 40 : return m_source_id_path_cache[rid][source_id];
691 : }
692 : }
693 :
694 : std::string
695 0 : HDF5RawDataFile::get_record_header_dataset_path(const uint64_t rec_num, // NOLINT (build/unsigned)
696 : const daqdataformats::sequence_number_t seq_num)
697 : {
698 0 : return get_record_header_dataset_path(std::make_pair(rec_num, seq_num));
699 : }
700 :
701 : std::string
702 0 : HDF5RawDataFile::get_trigger_record_header_dataset_path(const record_id_t& rid)
703 : {
704 0 : check_record_type("TriggerRecord");
705 0 : return get_record_header_dataset_path(rid);
706 : }
707 :
708 : std::string
709 0 : HDF5RawDataFile::get_trigger_record_header_dataset_path(const daqdataformats::trigger_number_t trig_num,
710 : const daqdataformats::sequence_number_t seq_num)
711 : {
712 0 : check_record_type("TriggerRecord");
713 0 : return get_record_header_dataset_path(trig_num, seq_num);
714 : }
715 :
716 : std::string
717 0 : HDF5RawDataFile::get_timeslice_header_dataset_path(const record_id_t& rid)
718 : {
719 0 : check_record_type("TimeSlice");
720 0 : return get_record_header_dataset_path(rid.first, 0);
721 : }
722 :
723 : std::string
724 0 : HDF5RawDataFile::get_timeslice_header_dataset_path(const daqdataformats::timeslice_number_t ts_num)
725 : {
726 0 : check_record_type("TimeSlice");
727 0 : return get_record_header_dataset_path(ts_num);
728 : }
729 :
730 : /**
731 : * @brief Return a vector of dataset names that correspond to Fragemnts
732 : * Note: this gets all datsets, and then removes those that look like TriggerRecordHeader ones
733 : * one could instead loop through all system types and ask for appropriate datsets in those
734 : * however, probably that's more time consuming
735 : */
736 : std::vector<std::string>
737 8 : HDF5RawDataFile::get_all_fragment_dataset_paths()
738 : {
739 8 : std::vector<std::string> frag_paths;
740 :
741 468 : for (auto const& path : get_dataset_paths()) {
742 460 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) == std::string::npos)
743 420 : frag_paths.push_back(path);
744 8 : }
745 :
746 8 : return frag_paths;
747 0 : }
748 :
749 : // get all fragment dataset paths for given record ID
750 : std::vector<std::string>
751 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid)
752 : {
753 0 : auto rec_id = get_all_record_ids().find(rid);
754 0 : if (rec_id == get_all_record_ids().end())
755 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
756 :
757 0 : std::vector<std::string> frag_paths;
758 0 : if (get_version() <= 2) {
759 0 : std::string record_group_path =
760 0 : m_file_ptr->getPath() + m_file_layout_ptr->get_record_number_string(rid.first, rid.second);
761 :
762 0 : for (auto const& path : get_dataset_paths(record_group_path)) {
763 0 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) == std::string::npos)
764 0 : frag_paths.push_back(path);
765 0 : }
766 0 : } else {
767 0 : std::set<daqdataformats::SourceID> source_id_list = get_fragment_source_ids(rid);
768 0 : for (auto const& source_id : source_id_list) {
769 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
770 : }
771 0 : }
772 0 : return frag_paths;
773 0 : }
774 :
775 : // get all fragment dataset paths for given record ID
776 : std::vector<std::string>
777 0 : HDF5RawDataFile::get_fragment_dataset_paths(const uint64_t rec_num, // NOLINT (build/unsigned)
778 : const daqdataformats::sequence_number_t seq_num)
779 : {
780 0 : return get_fragment_dataset_paths(std::make_pair(rec_num, seq_num));
781 : }
782 :
783 : // get all fragment dataset paths for a Subsystem
784 : std::vector<std::string>
785 0 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID::Subsystem subsystem)
786 : {
787 0 : std::vector<std::string> frag_paths;
788 0 : for (auto const& rid : get_all_record_ids()) {
789 0 : if (get_version() <= 2) {
790 0 : auto datasets = get_dataset_paths(m_file_ptr->getPath() +
791 0 : m_file_layout_ptr->get_fragment_type_path(rid.first, rid.second, subsystem));
792 0 : frag_paths.insert(frag_paths.end(), datasets.begin(), datasets.end());
793 0 : } else {
794 0 : std::set<daqdataformats::SourceID> source_id_list = get_source_ids_for_subsystem(rid, subsystem);
795 0 : for (auto const& source_id : source_id_list) {
796 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
797 : }
798 0 : }
799 0 : }
800 0 : return frag_paths;
801 0 : }
802 :
803 : std::vector<std::string>
804 0 : HDF5RawDataFile::get_fragment_dataset_paths(const std::string& subsystem_name)
805 : {
806 0 : daqdataformats::SourceID::Subsystem subsystem = daqdataformats::SourceID::string_to_subsystem(subsystem_name);
807 0 : return get_fragment_dataset_paths(subsystem);
808 : }
809 :
810 : std::vector<std::string>
811 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid, const daqdataformats::SourceID::Subsystem subsystem)
812 : {
813 0 : auto rec_id = get_all_record_ids().find(rid);
814 0 : if (rec_id == get_all_record_ids().end())
815 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
816 :
817 0 : if (get_version() <= 2) {
818 0 : return get_dataset_paths(m_file_ptr->getPath() +
819 0 : m_file_layout_ptr->get_fragment_type_path(rid.first, rid.second, subsystem));
820 : } else {
821 0 : std::vector<std::string> frag_paths;
822 0 : std::set<daqdataformats::SourceID> source_id_list = get_source_ids_for_subsystem(rid, subsystem);
823 0 : for (auto const& source_id : source_id_list) {
824 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
825 : }
826 0 : return frag_paths;
827 0 : }
828 : }
829 :
830 : std::vector<std::string>
831 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid, const std::string& subsystem_name)
832 : {
833 0 : daqdataformats::SourceID::Subsystem subsystem = daqdataformats::SourceID::string_to_subsystem(subsystem_name);
834 0 : return get_fragment_dataset_paths(rid, subsystem);
835 : }
836 :
837 : #if 0
838 : // get all fragment dataset paths for a SourceID
839 : std::vector<std::string>
840 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID& source_id)
841 : {
842 : std::vector<std::string> frag_paths;
843 :
844 : for (auto const& rid : get_all_record_ids())
845 : frag_paths.push_back(m_file_ptr->getPath() +
846 : m_file_layout_ptr->get_fragment_path(rid.first, rid.second, source_id));
847 :
848 : return frag_paths;
849 : }
850 :
851 : std::vector<std::string>
852 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID::Subsystem type,
853 : const uint32_t id) // NOLINT(build/unsigned)
854 : {
855 : return get_fragment_dataset_paths(daqdataformats::SourceID(type, source_id));
856 : }
857 : std::vector<std::string>
858 : HDF5RawDataFile::get_fragment_dataset_paths(const std::string& typestring,
859 : const uint32_t id) // NOLINT(build/unsigned)
860 : {
861 : return get_fragment_dataset_paths(
862 : daqdataformats::SourceID(daqdataformats::SourceID::string_to_subsystem(typestring), source_id));
863 : }
864 :
865 : std::set<daqdataformats::SourceID>
866 : HDF5RawDataFile::get_source_ids(std::vector<std::string> const& frag_dataset_paths)
867 : {
868 : std::set<daqdataformats::SourceID> source_ids;
869 : std::vector<std::string> path_elements;
870 : std::string s;
871 : for (auto const& frag_dataset : frag_dataset_paths) {
872 : path_elements.clear();
873 : std::istringstream iss(frag_dataset);
874 : while (std::getline(iss, s, '/')) {
875 : if (s.size() > 0)
876 : path_elements.push_back(s);
877 : }
878 : source_ids.insert(m_file_layout_ptr->get_source_id_from_path_elements(path_elements));
879 : }
880 :
881 : return source_ids;
882 : }
883 : #endif
884 :
885 : HDF5SourceIDHandler::source_id_geo_id_map_t
886 0 : HDF5RawDataFile::get_srcid_geoid_map() const {
887 :
888 0 : return m_file_level_source_id_geo_id_map;
889 : }
890 :
891 : std::set<uint64_t> // NOLINT(build/unsigned)
892 0 : HDF5RawDataFile::get_all_geo_ids() const
893 : {
894 0 : std::set<uint64_t> set_of_geo_ids;
895 : // 13-Sep-2022, KAB
896 : // It would be safer, but slower, to fetch all of the geo_ids from the
897 : // individual records, and we'll go with faster, for now. If/when we
898 : // change the way that we determine the file-level and record-level
899 : // source_id-to-geo_id maps, we may need to change this code.
900 0 : for (auto const& map_entry : m_file_level_source_id_geo_id_map) {
901 0 : for (auto const& geo_id : map_entry.second) {
902 0 : set_of_geo_ids.insert(geo_id);
903 : }
904 : }
905 0 : return set_of_geo_ids;
906 0 : }
907 :
908 : std::set<uint64_t> // NOLINT(build/unsigned)
909 0 : HDF5RawDataFile::get_geo_ids(const record_id_t& rid)
910 : {
911 0 : auto rec_id = get_all_record_ids().find(rid);
912 0 : if (rec_id == get_all_record_ids().end())
913 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
914 :
915 0 : add_record_level_info_to_caches_if_needed(rid);
916 :
917 0 : std::set<uint64_t> set_of_geo_ids;
918 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
919 0 : for (auto const& geo_id : map_entry.second) {
920 0 : set_of_geo_ids.insert(geo_id);
921 : }
922 : }
923 0 : return set_of_geo_ids;
924 0 : }
925 :
926 : std::set<uint64_t> // NOLINT(build/unsigned)
927 0 : HDF5RawDataFile::get_geo_ids_for_subdetector(const record_id_t& rid,
928 : const detdataformats::DetID::Subdetector subdet)
929 : {
930 0 : auto rec_id = get_all_record_ids().find(rid);
931 0 : if (rec_id == get_all_record_ids().end())
932 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
933 :
934 0 : add_record_level_info_to_caches_if_needed(rid);
935 :
936 0 : std::set<uint64_t> set_of_geo_ids;
937 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
938 0 : for (auto const& geo_id : map_entry.second) {
939 : // FIXME: replace with a proper coder/decoder
940 :
941 0 : uint16_t det_id = 0xffff & geo_id;
942 0 : if (det_id == static_cast<uint16_t>(subdet)) {
943 0 : set_of_geo_ids.insert(geo_id);
944 : }
945 : }
946 : }
947 0 : return set_of_geo_ids;
948 0 : }
949 :
950 :
951 : // get all SourceIDs for given record ID
952 : std::set<daqdataformats::SourceID>
953 0 : HDF5RawDataFile::get_source_ids(const record_id_t& rid)
954 : {
955 0 : auto rec_id = get_all_record_ids().find(rid);
956 0 : if (rec_id == get_all_record_ids().end())
957 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
958 :
959 0 : add_record_level_info_to_caches_if_needed(rid);
960 :
961 0 : return m_source_id_cache[rid];
962 : }
963 :
964 : daqdataformats::SourceID
965 40 : HDF5RawDataFile::get_record_header_source_id(const record_id_t& rid)
966 : {
967 40 : auto rec_id = get_all_record_ids().find(rid);
968 40 : if (rec_id == get_all_record_ids().end())
969 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
970 :
971 40 : add_record_level_info_to_caches_if_needed(rid);
972 :
973 40 : return m_record_header_source_id_cache[rid];
974 : }
975 :
976 : std::set<daqdataformats::SourceID>
977 0 : HDF5RawDataFile::get_fragment_source_ids(const record_id_t& rid)
978 : {
979 0 : auto rec_id = get_all_record_ids().find(rid);
980 0 : if (rec_id == get_all_record_ids().end())
981 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
982 :
983 0 : add_record_level_info_to_caches_if_needed(rid);
984 :
985 0 : return m_fragment_source_id_cache[rid];
986 : }
987 :
988 : std::set<daqdataformats::SourceID>
989 0 : HDF5RawDataFile::get_source_ids_for_subsystem(const record_id_t& rid,
990 : const daqdataformats::SourceID::Subsystem subsystem)
991 : {
992 0 : auto rec_id = get_all_record_ids().find(rid);
993 0 : if (rec_id == get_all_record_ids().end())
994 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
995 :
996 0 : add_record_level_info_to_caches_if_needed(rid);
997 :
998 0 : return m_subsystem_source_id_cache[rid][subsystem];
999 : }
1000 :
1001 : std::set<daqdataformats::SourceID>
1002 0 : HDF5RawDataFile::get_source_ids_for_fragment_type(const record_id_t& rid, const daqdataformats::FragmentType frag_type)
1003 : {
1004 0 : auto rec_id = get_all_record_ids().find(rid);
1005 0 : if (rec_id == get_all_record_ids().end())
1006 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1007 :
1008 0 : add_record_level_info_to_caches_if_needed(rid);
1009 :
1010 0 : return m_fragment_type_source_id_cache[rid][frag_type];
1011 : }
1012 :
1013 : std::set<daqdataformats::SourceID>
1014 0 : HDF5RawDataFile::get_source_ids_for_fragtype_and_subdetector(const record_id_t& rid,
1015 : const std::string& frag_type_name,
1016 : const std::string& subdet_name)
1017 : {
1018 0 : daqdataformats::FragmentType frag_type = daqdataformats::string_to_fragment_type(frag_type_name);
1019 0 : if (frag_type == daqdataformats::FragmentType::kUnknown)
1020 0 : throw InvalidFragmentTypeString(ERS_HERE, frag_type_name);
1021 0 : detdataformats::DetID::Subdetector subdet = detdataformats::DetID::string_to_subdetector(subdet_name);
1022 0 : if (subdet == detdataformats::DetID::Subdetector::kUnknown)
1023 0 : throw InvalidSubdetectorString(ERS_HERE, subdet_name);
1024 :
1025 0 : auto rec_id = get_all_record_ids().find(rid);
1026 0 : if (rec_id == get_all_record_ids().end())
1027 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1028 :
1029 0 : add_record_level_info_to_caches_if_needed(rid);
1030 :
1031 0 : std::set<daqdataformats::SourceID> fragtype_match_sids = m_fragment_type_source_id_cache[rid][frag_type];
1032 0 : std::set<daqdataformats::SourceID> detid_match_sids = m_subdetector_source_id_cache[rid][subdet];
1033 0 : std::set<daqdataformats::SourceID> combined_set_sids;
1034 0 : for (auto ftsid : fragtype_match_sids) {
1035 0 : if (detid_match_sids.contains(ftsid)) {
1036 0 : combined_set_sids.insert(ftsid);
1037 : }
1038 : }
1039 0 : return combined_set_sids;
1040 0 : }
1041 :
1042 : std::set<daqdataformats::SourceID>
1043 0 : HDF5RawDataFile::get_source_ids_for_subdetector(const record_id_t& rid, const detdataformats::DetID::Subdetector subdet)
1044 : {
1045 0 : auto rec_id = get_all_record_ids().find(rid);
1046 0 : if (rec_id == get_all_record_ids().end())
1047 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1048 :
1049 0 : add_record_level_info_to_caches_if_needed(rid);
1050 :
1051 0 : return m_subdetector_source_id_cache[rid][subdet];
1052 : }
1053 :
1054 : std::unique_ptr<char[]>
1055 48 : HDF5RawDataFile::get_dataset_raw_data(const std::string& dataset_path)
1056 : {
1057 48 : HighFive::Group parent_group = m_file_ptr->getGroup("/");
1058 48 : HighFive::DataSet data_set = parent_group.getDataSet(dataset_path);
1059 :
1060 48 : if (!data_set.isValid())
1061 0 : throw InvalidHDF5Dataset(ERS_HERE, dataset_path, get_file_name());
1062 :
1063 48 : size_t data_size = data_set.getSpace().getElementCount() * sizeof(char);
1064 :
1065 48 : auto membuffer = std::make_unique<char[]>(data_size);
1066 :
1067 48 : data_set.read(membuffer.get());
1068 48 : return membuffer;
1069 48 : }
1070 :
1071 : std::unique_ptr<daqdataformats::Fragment>
1072 32 : HDF5RawDataFile::get_frag_ptr(const std::string& dataset_name)
1073 : {
1074 32 : auto membuffer = get_dataset_raw_data(dataset_name);
1075 32 : auto frag_ptr = std::make_unique<daqdataformats::Fragment>(
1076 32 : membuffer.release(), dunedaq::daqdataformats::Fragment::BufferAdoptionMode::kTakeOverBuffer);
1077 32 : return frag_ptr;
1078 32 : }
1079 :
1080 : std::unique_ptr<daqdataformats::Fragment>
1081 24 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid, const daqdataformats::SourceID& source_id)
1082 : {
1083 24 : if (get_version() < 2)
1084 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1085 :
1086 24 : auto rec_id = get_all_record_ids().find(rid);
1087 24 : if (rec_id == get_all_record_ids().end())
1088 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1089 :
1090 24 : add_record_level_info_to_caches_if_needed(rid);
1091 :
1092 24 : return get_frag_ptr(m_source_id_path_cache[rid][source_id]);
1093 : }
1094 :
1095 : std::unique_ptr<daqdataformats::Fragment>
1096 8 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1097 : const daqdataformats::sequence_number_t seq_num,
1098 : const daqdataformats::SourceID& source_id)
1099 : {
1100 8 : record_id_t rid = std::make_pair(rec_num, seq_num);
1101 8 : return get_frag_ptr(rid, source_id);
1102 : }
1103 :
1104 : std::unique_ptr<daqdataformats::Fragment>
1105 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1106 : const daqdataformats::SourceID::Subsystem type,
1107 : const uint32_t id) // NOLINT(build/unsigned)
1108 : {
1109 0 : daqdataformats::SourceID source_id(type, id);
1110 0 : return get_frag_ptr(rid, source_id);
1111 : }
1112 :
1113 : std::unique_ptr<daqdataformats::Fragment>
1114 0 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1115 : const daqdataformats::sequence_number_t seq_num,
1116 : const daqdataformats::SourceID::Subsystem type,
1117 : const uint32_t id) // NOLINT(build/unsigned)
1118 : {
1119 0 : record_id_t rid = std::make_pair(rec_num, seq_num);
1120 0 : daqdataformats::SourceID source_id(type, id);
1121 0 : return get_frag_ptr(rid, source_id);
1122 : }
1123 :
1124 : std::unique_ptr<daqdataformats::Fragment>
1125 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1126 : const std::string& typestring,
1127 : const uint32_t id) // NOLINT(build/unsigned)
1128 : {
1129 0 : daqdataformats::SourceID source_id(daqdataformats::SourceID::string_to_subsystem(typestring), id);
1130 0 : return get_frag_ptr(rid, source_id);
1131 : }
1132 :
1133 : std::unique_ptr<daqdataformats::Fragment>
1134 16 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1135 : const daqdataformats::sequence_number_t seq_num,
1136 : const std::string& typestring,
1137 : const uint32_t id) // NOLINT(build/unsigned)
1138 : {
1139 16 : record_id_t rid = std::make_pair(rec_num, seq_num);
1140 16 : daqdataformats::SourceID source_id(daqdataformats::SourceID::string_to_subsystem(typestring), id);
1141 16 : return get_frag_ptr(rid, source_id);
1142 : }
1143 :
1144 : std::unique_ptr<daqdataformats::Fragment>
1145 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1146 : const uint64_t geo_id) // NOLINT(build/unsigned)
1147 : {
1148 0 : daqdataformats::SourceID sid = get_source_id_for_geo_id(rid, geo_id);
1149 0 : return get_frag_ptr(rid, sid);
1150 : }
1151 :
1152 : std::unique_ptr<daqdataformats::Fragment>
1153 0 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1154 : const daqdataformats::sequence_number_t seq_num,
1155 : const uint64_t geo_id) // NOLINT(build/unsigned)
1156 : {
1157 0 : record_id_t rid = std::make_pair(rec_num, seq_num);
1158 0 : return get_frag_ptr(rid, geo_id);
1159 : }
1160 :
1161 : std::unique_ptr<daqdataformats::TriggerRecordHeader>
1162 8 : HDF5RawDataFile::get_trh_ptr(const std::string& dataset_name)
1163 : {
1164 8 : auto membuffer = get_dataset_raw_data(dataset_name);
1165 8 : auto trh_ptr = std::make_unique<daqdataformats::TriggerRecordHeader>(membuffer.release(), true);
1166 8 : return trh_ptr;
1167 8 : }
1168 :
1169 : std::unique_ptr<daqdataformats::TriggerRecordHeader>
1170 4 : HDF5RawDataFile::get_trh_ptr(const record_id_t& rid)
1171 : {
1172 4 : if (get_version() < 2)
1173 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1174 :
1175 4 : auto rec_id = get_all_record_ids().find(rid);
1176 4 : if (rec_id == get_all_record_ids().end())
1177 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1178 :
1179 4 : add_record_level_info_to_caches_if_needed(rid);
1180 :
1181 4 : daqdataformats::SourceID rh_source_id = m_record_header_source_id_cache[rid];
1182 4 : return get_trh_ptr(m_source_id_path_cache[rid][rh_source_id]);
1183 : }
1184 :
1185 : std::unique_ptr<daqdataformats::TimeSliceHeader>
1186 8 : HDF5RawDataFile::get_tsh_ptr(const std::string& dataset_name)
1187 : {
1188 8 : auto membuffer = get_dataset_raw_data(dataset_name);
1189 8 : auto tsh_ptr = std::make_unique<daqdataformats::TimeSliceHeader>(
1190 8 : *(reinterpret_cast<daqdataformats::TimeSliceHeader*>(membuffer.release()))); // NOLINT
1191 8 : return tsh_ptr;
1192 8 : }
1193 :
1194 : std::unique_ptr<daqdataformats::TimeSliceHeader>
1195 4 : HDF5RawDataFile::get_tsh_ptr(const record_id_t& rid)
1196 : {
1197 4 : if (get_version() < 2)
1198 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1199 :
1200 4 : auto rec_id = get_all_record_ids().find(rid);
1201 4 : if (rec_id == get_all_record_ids().end())
1202 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1203 :
1204 4 : add_record_level_info_to_caches_if_needed(rid);
1205 :
1206 4 : daqdataformats::SourceID rh_source_id = m_record_header_source_id_cache[rid];
1207 4 : return get_tsh_ptr(m_source_id_path_cache[rid][rh_source_id]);
1208 : }
1209 :
1210 : daqdataformats::TriggerRecord
1211 0 : HDF5RawDataFile::get_trigger_record(const record_id_t& rid)
1212 : {
1213 0 : daqdataformats::TriggerRecord trigger_record(*get_trh_ptr(rid));
1214 0 : for (auto const& frag_path : get_fragment_dataset_paths(rid)) {
1215 0 : trigger_record.add_fragment(get_frag_ptr(frag_path));
1216 0 : }
1217 :
1218 0 : return trigger_record;
1219 0 : }
1220 :
1221 : daqdataformats::TimeSlice
1222 0 : HDF5RawDataFile::get_timeslice(const daqdataformats::timeslice_number_t ts_num)
1223 : {
1224 0 : daqdataformats::TimeSlice timeslice(*get_tsh_ptr(ts_num));
1225 0 : for (auto const& frag_path : get_fragment_dataset_paths(ts_num)) {
1226 0 : timeslice.add_fragment(get_frag_ptr(frag_path));
1227 0 : }
1228 :
1229 0 : return timeslice;
1230 0 : }
1231 :
1232 : std::vector<uint64_t> // NOLINT(build/unsigned)
1233 0 : HDF5RawDataFile::get_geo_ids_for_source_id(const record_id_t& rid, const daqdataformats::SourceID& source_id)
1234 : {
1235 0 : auto rec_id = get_all_record_ids().find(rid);
1236 0 : if (rec_id == get_all_record_ids().end())
1237 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1238 :
1239 0 : add_record_level_info_to_caches_if_needed(rid);
1240 :
1241 0 : return m_source_id_geo_id_cache[rid][source_id];
1242 : }
1243 :
1244 : daqdataformats::SourceID
1245 0 : HDF5RawDataFile::get_source_id_for_geo_id(const record_id_t& rid,
1246 : const uint64_t requested_geo_id) // NOLINT(build/unsigned)
1247 : {
1248 0 : auto rec_id = get_all_record_ids().find(rid);
1249 0 : if (rec_id == get_all_record_ids().end())
1250 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1251 :
1252 0 : add_record_level_info_to_caches_if_needed(rid);
1253 :
1254 : // if we want to make this faster, we could build a reverse lookup cache in
1255 : // add_record_level_info_to_caches_if_needed() and just look up the requested geo_id here
1256 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
1257 0 : auto geoid_list = map_entry.second;
1258 0 : for (auto const& geoid_from_list : geoid_list) {
1259 0 : if (geoid_from_list == requested_geo_id) {
1260 0 : return map_entry.first;
1261 : }
1262 : }
1263 0 : }
1264 :
1265 0 : daqdataformats::SourceID empty_sid;
1266 0 : return empty_sid;
1267 : }
1268 :
1269 : } // namespace hdf5libs
1270 : } // namespace dunedaq
|
