Line data Source code
1 : /**
2 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
3 : * Licensing/copyright details are in the COPYING file that you should have
4 : * received with this code.
5 : *
6 : */
7 :
8 : #include "hdf5libs/HDF5RawDataFile.hpp"
9 :
10 : #include "logging/Logging.hpp"
11 :
12 : #include <algorithm>
13 : #include <filesystem>
14 : #include <map>
15 : #include <memory>
16 : #include <set>
17 : #include <sstream>
18 : #include <string>
19 : #include <utility>
20 : #include <vector>
21 :
22 : namespace dunedaq {
23 : namespace hdf5libs {
24 :
25 : constexpr uint32_t MAX_FILELAYOUT_VERSION = 4294967295; // NOLINT(build/unsigned)
26 :
27 : /**
28 : * @brief Constructor for writing a new file
29 : */
30 30 : HDF5RawDataFile::HDF5RawDataFile(const std::string file_name,
31 : const daqdataformats::run_number_t run_number,
32 : const size_t file_index,
33 : const std::string application_name,
34 : const HDF5FileLayoutParameters fl_params,
35 : const HDF5SourceIDHandler::source_id_geo_id_map_t srcid_geoid_map,
36 : const unsigned compression_level,
37 : const std::string inprogress_filename_suffix,
38 30 : const unsigned open_flags)
39 30 : : m_bare_file_name(file_name)
40 30 : , m_compression_level(compression_level)
41 60 : , m_open_flags(open_flags)
42 : {
43 :
44 : // check and make sure that the file isn't ReadOnly
45 30 : if (m_open_flags == HighFive::File::ReadOnly) {
46 0 : throw IncompatibleOpenFlags(ERS_HERE, file_name, m_open_flags);
47 : }
48 :
49 30 : auto filename_to_open = m_bare_file_name + inprogress_filename_suffix;
50 :
51 : // do the file open
52 30 : try {
53 30 : m_file_ptr.reset(new HighFive::File(filename_to_open, m_open_flags));
54 0 : } catch (std::exception const& excpt) {
55 0 : throw FileOpenFailed(ERS_HERE, filename_to_open, excpt.what());
56 0 : }
57 :
58 30 : m_recorded_size = 0;
59 30 : m_uncompressed_raw_data_size = 0;
60 30 : m_total_file_size = 0;
61 :
62 30 : size_t file_creation_timestamp =
63 30 : std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
64 :
65 30 : TLOG_DEBUG(TLVL_BASIC) << "Created HDF5 file (" << file_name << ") at time " << file_creation_timestamp << " .";
66 :
67 : // write some file attributes
68 30 : write_attribute("run_number", run_number);
69 30 : write_attribute("file_index", file_index);
70 30 : write_attribute("creation_timestamp", file_creation_timestamp);
71 30 : write_attribute("application_name", application_name);
72 :
73 : // set the file layout contents
74 30 : m_file_layout_ptr.reset(new HDF5FileLayout(fl_params));
75 30 : write_file_layout();
76 :
77 : // write the SourceID-related attributes
78 : // HDF5SourceIDHandler::populate_source_id_geo_id_map(srcid_geoid_map, m_file_level_source_id_geo_id_map);
79 30 : m_file_level_source_id_geo_id_map = srcid_geoid_map;
80 30 : HDF5SourceIDHandler::store_file_level_geo_id_info(*m_file_ptr, m_file_level_source_id_geo_id_map);
81 :
82 : // write the record type
83 30 : m_record_type = fl_params.record_name_prefix;
84 30 : write_attribute("record_type", m_record_type);
85 :
86 : // write the compression level
87 30 : write_attribute("compression_level", m_compression_level);
88 30 : }
89 :
90 44 : HDF5RawDataFile::~HDF5RawDataFile()
91 : {
92 44 : if (m_file_ptr.get() != nullptr && m_open_flags != HighFive::File::ReadOnly) {
93 30 : if (!m_file_ptr->hasAttribute("recorded_size")) {
94 30 : write_attribute("recorded_size", m_recorded_size);
95 : }
96 :
97 30 : if (!m_file_ptr->hasAttribute("uncompressed_raw_data_size")) {
98 30 : write_attribute("uncompressed_raw_data_size", m_uncompressed_raw_data_size);
99 : }
100 :
101 30 : if (!m_file_ptr->hasAttribute("total_file_size")) {
102 30 : write_attribute("total_file_size", m_total_file_size);
103 : }
104 :
105 30 : if (!m_file_ptr->hasAttribute("closing_timestamp")) {
106 30 : size_t file_closing_timestamp =
107 30 : std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
108 30 : write_attribute("closing_timestamp", file_closing_timestamp);
109 : }
110 :
111 30 : m_file_ptr->flush();
112 :
113 : // rename file to the bare name, if needed
114 30 : if (m_file_ptr->getName() != m_bare_file_name) {
115 30 : std::filesystem::rename(m_file_ptr->getName(), m_bare_file_name);
116 : }
117 : }
118 :
119 : // explicit destruction; not really needed, but nice to be clear...
120 44 : m_file_ptr.reset();
121 44 : m_file_layout_ptr.reset();
122 44 : }
123 :
124 : /**
125 : * @brief Fetch the list of all file-level Attribute names.
126 : */
127 : std::vector<std::string>
128 0 : HDF5RawDataFile::HDF5RawDataFile::get_attribute_names()
129 : {
130 0 : return m_file_ptr->listAttributeNames();
131 : }
132 :
133 : /**
134 : * @brief Write a TriggerRecord to the file.
135 : */
136 : void
137 75 : HDF5RawDataFile::write(const daqdataformats::TriggerRecord& tr)
138 : {
139 : // the source_id_path map that we will build up as we write the TR header
140 : // and fragments (and then write the map into the HDF5 TR_record Group)
141 75 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
142 :
143 : // the map of fragment types to SourceIDS
144 75 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
145 :
146 : // the map of subdetectors to SourceIDS
147 75 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
148 :
149 : // write the record header into the HDF5 file/group
150 75 : HighFive::Group record_level_group = write(tr.get_header_ref(), source_id_path_map);
151 :
152 : // store the SourceID of the record header in the HDF5 file/group
153 : // (since there should only be one entry in the map at this point, we'll take advantage of that...)
154 150 : for (auto const& source_id_path : source_id_path_map) {
155 75 : HDF5SourceIDHandler::store_record_header_source_id(record_level_group, source_id_path.first);
156 : }
157 :
158 : // write all of the fragments into the HDF5 file/group
159 1415 : for (auto const& frag_ptr : tr.get_fragments_ref()) {
160 1340 : write(*frag_ptr, source_id_path_map);
161 1340 : HDF5SourceIDHandler::add_fragment_type_source_id_to_map(
162 1340 : fragment_type_source_id_map, frag_ptr->get_fragment_type(), frag_ptr->get_element_id());
163 1340 : HDF5SourceIDHandler::add_subdetector_source_id_to_map(
164 : subdetector_source_id_map,
165 1340 : static_cast<detdataformats::DetID::Subdetector>(frag_ptr->get_detector_id()),
166 1340 : frag_ptr->get_element_id());
167 : }
168 :
169 : // store all of the record-level maps in the HDF5 file/group
170 75 : HDF5SourceIDHandler::store_record_level_path_info(record_level_group, source_id_path_map);
171 75 : HDF5SourceIDHandler::store_record_level_fragment_type_map(record_level_group, fragment_type_source_id_map);
172 75 : HDF5SourceIDHandler::store_record_level_subdetector_map(record_level_group, subdetector_source_id_map);
173 75 : }
174 :
175 : /**
176 : * @brief Write a TimeSlice to the file.
177 : */
178 : void
179 31 : HDF5RawDataFile::write(const daqdataformats::TimeSlice& ts)
180 : {
181 : // the source_id_path map that we will build up as we write the TR header
182 : // and fragments (and then write the map into the HDF5 TR_record Group)
183 31 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
184 :
185 : // the map of fragment types to SourceIDS
186 31 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
187 :
188 : // the map of subdetectors to SourceIDS
189 31 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
190 :
191 : // write the record header into the HDF5 file/group
192 31 : HighFive::Group record_level_group = write(ts.get_header(), source_id_path_map);
193 :
194 : // store the SourceID of the record header in the HDF5 file/group
195 : // (since there should only be one entry in the map at this point, we'll take advantage of that...)
196 62 : for (auto const& source_id_path : source_id_path_map) {
197 31 : HDF5SourceIDHandler::store_record_header_source_id(record_level_group, source_id_path.first);
198 : }
199 :
200 : // write all of the fragments into the HDF5 file/group
201 271 : for (auto const& frag_ptr : ts.get_fragments_ref()) {
202 240 : write(*frag_ptr, source_id_path_map);
203 240 : HDF5SourceIDHandler::add_fragment_type_source_id_to_map(
204 240 : fragment_type_source_id_map, frag_ptr->get_fragment_type(), frag_ptr->get_element_id());
205 240 : HDF5SourceIDHandler::add_subdetector_source_id_to_map(
206 : subdetector_source_id_map,
207 240 : static_cast<detdataformats::DetID::Subdetector>(frag_ptr->get_detector_id()),
208 240 : frag_ptr->get_element_id());
209 : }
210 :
211 : // store all of the record-level maps in the HDF5 file/group
212 31 : HDF5SourceIDHandler::store_record_level_path_info(record_level_group, source_id_path_map);
213 31 : HDF5SourceIDHandler::store_record_level_fragment_type_map(record_level_group, fragment_type_source_id_map);
214 31 : HDF5SourceIDHandler::store_record_level_subdetector_map(record_level_group, subdetector_source_id_map);
215 31 : }
216 :
217 : /**
218 : * @brief Write a TriggerRecordHeader to the file.
219 : */
220 : HighFive::Group
221 75 : HDF5RawDataFile::write(const daqdataformats::TriggerRecordHeader& trh,
222 : HDF5SourceIDHandler::source_id_path_map_t& path_map)
223 : {
224 75 : std::tuple<size_t, std::string, HighFive::Group> write_results =
225 75 : do_write(m_file_layout_ptr->get_path_elements(trh),
226 75 : static_cast<const char*>(trh.get_storage_location()),
227 : trh.get_total_size_bytes(),
228 75 : m_compression_level);
229 75 : m_recorded_size += std::get<0>(write_results);
230 75 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, trh.get_header().element_id, std::get<1>(write_results));
231 150 : return std::get<2>(write_results);
232 75 : }
233 :
234 : /**
235 : * @brief Write a TimeSliceHeader to the file.
236 : */
237 : HighFive::Group
238 31 : HDF5RawDataFile::write(const daqdataformats::TimeSliceHeader& tsh, HDF5SourceIDHandler::source_id_path_map_t& path_map)
239 : {
240 31 : std::tuple<size_t, std::string, HighFive::Group> write_results = do_write(m_file_layout_ptr->get_path_elements(tsh),
241 : (const char*)(&tsh),
242 : sizeof(daqdataformats::TimeSliceHeader),
243 31 : m_compression_level);
244 31 : m_recorded_size += std::get<0>(write_results);
245 31 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, tsh.element_id, std::get<1>(write_results));
246 62 : return std::get<2>(write_results);
247 31 : }
248 :
249 : /**
250 : * @brief Write a Fragment to the file.
251 : */
252 : void
253 1580 : HDF5RawDataFile::write(const daqdataformats::Fragment& frag, HDF5SourceIDHandler::source_id_path_map_t& path_map)
254 : {
255 1580 : std::tuple<size_t, std::string, HighFive::Group> write_results =
256 1580 : do_write(m_file_layout_ptr->get_path_elements(frag.get_header()),
257 1580 : static_cast<const char*>(frag.get_storage_location()),
258 : frag.get_size(),
259 1580 : m_compression_level);
260 1580 : m_recorded_size += std::get<0>(write_results);
261 :
262 1580 : daqdataformats::SourceID source_id = frag.get_element_id();
263 1580 : HDF5SourceIDHandler::add_source_id_path_to_map(path_map, source_id, std::get<1>(write_results));
264 1580 : }
265 :
266 : /**
267 : * @brief write the file layout
268 : */
269 : void
270 30 : HDF5RawDataFile::write_file_layout()
271 : {
272 30 : auto fl_json = m_file_layout_ptr->get_file_layout_params().to_json();
273 30 : write_attribute("filelayout_params", fl_json.dump());
274 30 : write_attribute("filelayout_version", m_file_layout_ptr->get_version());
275 30 : }
276 :
277 : /**
278 : * @brief write bytes to a dataset in the file, at the appropriate path
279 : */
280 : std::tuple<size_t, std::string, HighFive::Group>
281 1686 : HDF5RawDataFile::do_write(std::vector<std::string> const& group_and_dataset_path_elements,
282 : const char* raw_data_ptr,
283 : size_t raw_data_size_bytes,
284 : unsigned compression_level)
285 : {
286 1686 : const std::string dataset_name = group_and_dataset_path_elements.back();
287 :
288 : // create top level group if needed
289 1686 : std::string const& top_level_group_name = group_and_dataset_path_elements.at(0);
290 1686 : if (!m_file_ptr->exist(top_level_group_name))
291 106 : m_file_ptr->createGroup(top_level_group_name);
292 :
293 : // setup sub_group to work with
294 1686 : HighFive::Group sub_group = m_file_ptr->getGroup(top_level_group_name);
295 1686 : if (!sub_group.isValid()) {
296 0 : throw InvalidHDF5Group(ERS_HERE, top_level_group_name);
297 : }
298 1686 : HighFive::Group top_level_group = sub_group;
299 :
300 : // Create the remaining subgroups
301 3372 : for (size_t idx = 1; idx < group_and_dataset_path_elements.size() - 1; ++idx) {
302 : // group_dataset.size()-1 because the last element is the dataset
303 1686 : std::string const& child_group_name = group_and_dataset_path_elements[idx];
304 1686 : if (child_group_name.empty()) {
305 0 : throw InvalidHDF5Group(ERS_HERE, child_group_name);
306 : }
307 1686 : if (!sub_group.exist(child_group_name)) {
308 106 : sub_group.createGroup(child_group_name);
309 : }
310 1686 : HighFive::Group child_group = sub_group.getGroup(child_group_name);
311 1686 : if (!child_group.isValid()) {
312 0 : throw InvalidHDF5Group(ERS_HERE, child_group_name);
313 : }
314 1686 : sub_group = child_group;
315 1686 : }
316 :
317 : // Create dataset
318 1686 : HighFive::DataSpace data_space = HighFive::DataSpace({ raw_data_size_bytes, 1 });
319 1686 : HighFive::DataSetCreateProps data_set_create_props;
320 1686 : HighFive::DataSetAccessProps data_set_access_props;
321 :
322 1686 : if (compression_level > 0) {
323 415 : std::vector<hsize_t> chunk_size = { raw_data_size_bytes, 1 };
324 415 : data_set_create_props.add(HighFive::Chunking(chunk_size));
325 415 : data_set_create_props.add(HighFive::Deflate(compression_level));
326 415 : }
327 :
328 1686 : m_uncompressed_raw_data_size += raw_data_size_bytes;
329 :
330 1686 : auto data_set = sub_group.createDataSet<char>(dataset_name, data_space, data_set_create_props, data_set_access_props);
331 :
332 1686 : if (data_set.isValid()) {
333 1686 : data_set.write_raw(raw_data_ptr);
334 1686 : m_total_file_size = m_file_ptr->getFileSize();
335 1686 : m_file_ptr->flush();
336 3372 : return std::make_tuple(data_set.getStorageSize(), data_set.getPath(), top_level_group);
337 : } else {
338 0 : throw InvalidHDF5Dataset(ERS_HERE, dataset_name, m_file_ptr->getName());
339 : }
340 1686 : }
341 :
342 : /**
343 : * @brief Constructor for reading (and optionally writing) a file
344 : */
345 14 : HDF5RawDataFile::HDF5RawDataFile(const std::string& file_name, bool allow_writing)
346 14 : : m_open_flags(HighFive::File::ReadOnly)
347 : {
348 14 : if (allow_writing) {
349 0 : m_open_flags = HighFive::File::ReadWrite;
350 : }
351 14 : m_bare_file_name = file_name;
352 14 : size_t pos = m_bare_file_name.rfind(s_inprogress_suffix);
353 14 : if (pos != std::string::npos) {
354 0 : m_bare_file_name.erase(pos);
355 : }
356 :
357 : // do the file open
358 14 : try {
359 14 : m_file_ptr = std::make_unique<HighFive::File>(file_name, m_open_flags);
360 0 : } catch (std::exception const& excpt) {
361 0 : throw FileOpenFailed(ERS_HERE, file_name, excpt.what());
362 0 : }
363 :
364 14 : if (m_file_ptr->hasAttribute("recorded_size"))
365 14 : m_recorded_size = get_attribute<size_t>("recorded_size");
366 : else
367 0 : m_recorded_size = 0;
368 :
369 14 : if (m_file_ptr->hasAttribute("uncompressed_raw_data_size"))
370 14 : m_uncompressed_raw_data_size = get_attribute<size_t>("uncompressed_raw_data_size");
371 : else
372 0 : m_uncompressed_raw_data_size = 0;
373 :
374 14 : if (m_file_ptr->hasAttribute("total_file_size"))
375 14 : m_total_file_size = get_attribute<size_t>("total_file_size");
376 : else
377 0 : m_total_file_size = 0;
378 :
379 14 : if (m_file_ptr->hasAttribute("compression_level"))
380 14 : m_compression_level = get_attribute<unsigned>("compression_level");
381 : else
382 0 : m_compression_level = 0;
383 :
384 14 : read_file_layout();
385 :
386 14 : if (m_file_ptr->hasAttribute("record_type"))
387 14 : m_record_type = get_attribute<std::string>("record_type");
388 : else
389 0 : m_record_type = m_file_layout_ptr->get_record_name_prefix();
390 :
391 14 : check_file_layout();
392 :
393 : // HDF5SourceIDHandler operations need to come *after* read_file_layout()
394 : // because they count on the filelayout_version, which is set in read_file_layout().
395 14 : HDF5SourceIDHandler sid_handler(get_version());
396 14 : sid_handler.fetch_file_level_geo_id_info(*m_file_ptr, m_file_level_source_id_geo_id_map);
397 14 : }
398 :
399 : void
400 14 : HDF5RawDataFile::read_file_layout()
401 : {
402 14 : HDF5FileLayoutParameters fl_params;
403 14 : uint32_t version = 0; // NOLINT(build/unsigned)
404 :
405 14 : std::string fl_str;
406 14 : try {
407 14 : fl_str = get_attribute<std::string>("filelayout_params");
408 14 : nlohmann::json fl_json = nlohmann::json::parse(fl_str);
409 14 : fl_params = HDF5FileLayoutParameters(fl_json);
410 :
411 14 : version = get_attribute<uint32_t>("filelayout_version"); // NOLINT(build/unsigned)
412 :
413 14 : } catch (InvalidHDF5Attribute const&) {
414 0 : ers::info(MissingFileLayout(ERS_HERE, version));
415 0 : }
416 :
417 : // now reset the HDF5Filelayout object
418 14 : m_file_layout_ptr.reset(new HDF5FileLayout(fl_params, version));
419 14 : }
420 :
421 : void
422 14 : HDF5RawDataFile::check_file_layout()
423 : {
424 14 : if (get_version() < 2)
425 0 : return;
426 :
427 14 : std::string record_type = get_attribute<std::string>("record_type");
428 14 : if (record_type.compare(m_file_layout_ptr->get_record_name_prefix()) != 0)
429 0 : throw BadRecordType(ERS_HERE, record_type, m_file_layout_ptr->get_record_name_prefix());
430 14 : }
431 :
432 : void
433 18 : HDF5RawDataFile::check_record_type(std::string rt_name)
434 : {
435 18 : if (get_version() < 2)
436 : return;
437 :
438 18 : if (m_file_layout_ptr->get_record_name_prefix().compare(rt_name) != 0)
439 0 : throw WrongRecordTypeRequested(ERS_HERE, rt_name, m_file_layout_ptr->get_record_name_prefix());
440 : }
441 :
442 : // HDF5 Utility function to recursively traverse a file
443 : void
444 176 : HDF5RawDataFile::explore_subgroup(const HighFive::Group& parent_group,
445 : std::string relative_path,
446 : std::vector<std::string>& path_list)
447 : {
448 176 : if (relative_path.size() > 0 && relative_path.compare(relative_path.size() - 1, 1, "/") == 0)
449 16 : relative_path.pop_back();
450 :
451 176 : std::vector<std::string> childNames = parent_group.listObjectNames();
452 :
453 1256 : for (auto& child_name : childNames) {
454 1080 : std::string full_path = relative_path + "/" + child_name;
455 1080 : HighFive::ObjectType child_type = parent_group.getObjectType(child_name);
456 :
457 1080 : if (child_type == HighFive::ObjectType::Dataset) {
458 920 : path_list.push_back(full_path);
459 160 : } else if (child_type == HighFive::ObjectType::Group) {
460 160 : HighFive::Group child_group = parent_group.getGroup(child_name);
461 : // start the recusion
462 160 : std::string new_path = relative_path + "/" + child_name;
463 160 : explore_subgroup(child_group, new_path, path_list);
464 160 : }
465 1080 : }
466 176 : }
467 :
468 : void
469 72 : HDF5RawDataFile::add_record_level_info_to_caches_if_needed(record_id_t rid)
470 : {
471 : // we should probably check that all relevant caches have an entry for the
472 : // specified record ID, but we will just check one, in the interest of
473 : // performance, and trust the "else" part of this routine to fill in *all*
474 : // of the appropriate caches
475 72 : if (m_source_id_path_cache.count(rid) != 0) {
476 32 : return;
477 : }
478 :
479 : // create the handler to do the work
480 40 : HDF5SourceIDHandler sid_handler(get_version());
481 :
482 : // determine the HDF5 Group that corresponds to the specified record
483 40 : std::string record_level_group_name = m_file_layout_ptr->get_record_number_string(rid.first, rid.second);
484 40 : HighFive::Group record_group = m_file_ptr->getGroup(record_level_group_name);
485 40 : if (!record_group.isValid()) {
486 0 : throw InvalidHDF5Group(ERS_HERE, record_level_group_name);
487 : }
488 :
489 : // start with a copy of the file-level source-id-to-geo-id map and give the
490 : // handler an opportunity to add any record-level additions
491 40 : HDF5SourceIDHandler::source_id_geo_id_map_t local_source_id_geo_id_map = m_file_level_source_id_geo_id_map;
492 40 : sid_handler.fetch_record_level_geo_id_info(record_group, local_source_id_geo_id_map);
493 :
494 : // fetch the record-level source-id-to-path map
495 40 : HDF5SourceIDHandler::source_id_path_map_t source_id_path_map;
496 40 : sid_handler.fetch_source_id_path_info(record_group, source_id_path_map);
497 :
498 : // fetch the record-level fragment-type-to-source-id map
499 40 : HDF5SourceIDHandler::fragment_type_source_id_map_t fragment_type_source_id_map;
500 40 : sid_handler.fetch_fragment_type_source_id_info(record_group, fragment_type_source_id_map);
501 :
502 : // fetch the record-level subdetector-to-source-id map
503 40 : HDF5SourceIDHandler::subdetector_source_id_map_t subdetector_source_id_map;
504 40 : sid_handler.fetch_subdetector_source_id_info(record_group, subdetector_source_id_map);
505 :
506 : // loop through the source-id-to-path map to create various lists of SourceIDs in the record
507 40 : daqdataformats::SourceID rh_sid = sid_handler.fetch_record_header_source_id(record_group);
508 40 : std::set<daqdataformats::SourceID> full_source_id_set;
509 40 : std::set<daqdataformats::SourceID> fragment_source_id_set;
510 40 : HDF5SourceIDHandler::subsystem_source_id_map_t subsystem_source_id_map;
511 500 : for (auto const& source_id_path : source_id_path_map) {
512 460 : full_source_id_set.insert(source_id_path.first);
513 460 : if (source_id_path.first != rh_sid) {
514 420 : fragment_source_id_set.insert(source_id_path.first);
515 : }
516 460 : HDF5SourceIDHandler::add_subsystem_source_id_to_map(
517 460 : subsystem_source_id_map, source_id_path.first.subsystem, source_id_path.first);
518 : }
519 :
520 : // note that even if the "fetch" methods above fail to add anything to the specified
521 : // maps, the maps will still be valid (though, possibly empty), and once we add them
522 : // to the caches here, we will be assured that lookups from the caches will not fail.
523 40 : m_source_id_cache[rid] = full_source_id_set;
524 40 : m_record_header_source_id_cache[rid] = rh_sid;
525 40 : m_fragment_source_id_cache[rid] = fragment_source_id_set;
526 40 : m_source_id_geo_id_cache[rid] = local_source_id_geo_id_map;
527 40 : m_source_id_path_cache[rid] = source_id_path_map;
528 40 : m_subsystem_source_id_cache[rid] = subsystem_source_id_map;
529 40 : m_fragment_type_source_id_cache[rid] = fragment_type_source_id_map;
530 40 : m_subdetector_source_id_cache[rid] = subdetector_source_id_map;
531 40 : }
532 :
533 : /**
534 : * @brief Return a vector of dataset names
535 : */
536 : std::vector<std::string>
537 16 : HDF5RawDataFile::get_dataset_paths(std::string top_level_group_name)
538 : {
539 16 : if (top_level_group_name.empty())
540 16 : top_level_group_name = m_file_ptr->getPath();
541 :
542 : // Vector containing the path list to the HDF5 datasets
543 16 : std::vector<std::string> path_list;
544 :
545 16 : HighFive::Group parent_group = m_file_ptr->getGroup(top_level_group_name);
546 16 : if (!parent_group.isValid())
547 0 : throw InvalidHDF5Group(ERS_HERE, top_level_group_name);
548 :
549 16 : explore_subgroup(parent_group, top_level_group_name, path_list);
550 :
551 16 : return path_list;
552 16 : }
553 :
554 : /**
555 : * @brief Return all of the record numbers in the file.
556 : */
557 : HDF5RawDataFile::record_id_set // NOLINT(build/unsigned)
558 242 : HDF5RawDataFile::get_all_record_ids()
559 : {
560 242 : if (!m_all_record_ids_in_file.empty())
561 232 : return m_all_record_ids_in_file;
562 :
563 : // records are at the top level
564 :
565 10 : HighFive::Group parent_group = m_file_ptr->getGroup(m_file_ptr->getPath());
566 :
567 10 : std::vector<std::string> childNames = parent_group.listObjectNames();
568 10 : const std::string record_prefix = m_file_layout_ptr->get_record_name_prefix();
569 10 : const size_t record_prefix_size = record_prefix.size();
570 :
571 60 : for (auto const& name : childNames) {
572 50 : auto loc = name.find(record_prefix);
573 :
574 50 : if (loc == std::string::npos)
575 0 : continue;
576 :
577 50 : auto rec_num_string = name.substr(loc + record_prefix_size);
578 :
579 50 : loc = rec_num_string.find(".");
580 50 : if (loc == std::string::npos) {
581 20 : m_all_record_ids_in_file.insert(std::make_pair(std::stoll(rec_num_string), 0));
582 : } else {
583 30 : auto seq_num_string = rec_num_string.substr(loc + 1);
584 30 : rec_num_string.resize(loc); // remove anything from '.' onwards
585 30 : m_all_record_ids_in_file.insert(std::make_pair(std::stoll(rec_num_string), std::stoi(seq_num_string)));
586 30 : }
587 :
588 50 : } // end loop over childNames
589 :
590 10 : return m_all_record_ids_in_file;
591 10 : }
592 :
593 : std::set<uint64_t>
594 4 : HDF5RawDataFile::get_all_record_numbers() // NOLINT(build/unsigned)
595 : {
596 12 : ers::warning(DeprecatedUsage(ERS_HERE,
597 : "get_all_record_numbers()",
598 12 : "Use get_all_record_ids(), which returns a record_number,sequence_number pair."));
599 :
600 4 : std::set<uint64_t> record_numbers; // NOLINT(build/unsigned)
601 24 : for (auto const& rid : get_all_record_ids())
602 24 : record_numbers.insert(rid.first);
603 :
604 4 : return record_numbers;
605 0 : }
606 :
607 : HDF5RawDataFile::record_id_set
608 6 : HDF5RawDataFile::get_all_trigger_record_ids()
609 : {
610 6 : check_record_type("TriggerRecord");
611 6 : return get_all_record_ids();
612 : }
613 :
614 : std::set<daqdataformats::trigger_number_t>
615 0 : HDF5RawDataFile::get_all_trigger_record_numbers()
616 : {
617 0 : ers::warning(
618 0 : DeprecatedUsage(ERS_HERE,
619 : "get_all_trigger_record_numbers()",
620 0 : "Use get_all_trigger_record_ids(), which returns a record_number,sequence_number pair."));
621 :
622 0 : return get_all_record_numbers();
623 : }
624 :
625 : HDF5RawDataFile::record_id_set
626 0 : HDF5RawDataFile::get_all_timeslice_ids()
627 : {
628 0 : check_record_type("TimeSlice");
629 0 : return get_all_record_ids();
630 : }
631 :
632 : std::set<daqdataformats::timeslice_number_t>
633 4 : HDF5RawDataFile::get_all_timeslice_numbers()
634 : {
635 4 : check_record_type("TimeSlice");
636 4 : return get_all_record_numbers();
637 : }
638 :
639 : /**
640 : * @brief Return a vector of dataset names that correspond to record headers
641 : */
642 : std::vector<std::string>
643 8 : HDF5RawDataFile::get_record_header_dataset_paths()
644 : {
645 :
646 8 : std::vector<std::string> rec_paths;
647 :
648 8 : if (get_version() >= 2) {
649 48 : for (auto const& rec_id : get_all_record_ids())
650 48 : rec_paths.push_back(get_record_header_dataset_path(rec_id));
651 : } else {
652 0 : for (auto const& path : get_dataset_paths()) {
653 0 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) != std::string::npos) {
654 0 : rec_paths.push_back(path);
655 : }
656 0 : }
657 : }
658 :
659 8 : return rec_paths;
660 0 : }
661 :
662 : std::vector<std::string>
663 4 : HDF5RawDataFile::get_trigger_record_header_dataset_paths()
664 : {
665 4 : check_record_type("TriggerRecord");
666 4 : return get_record_header_dataset_paths();
667 : }
668 :
669 : std::vector<std::string>
670 4 : HDF5RawDataFile::get_timeslice_header_dataset_paths()
671 : {
672 4 : check_record_type("TimeSlice");
673 4 : return get_record_header_dataset_paths();
674 : }
675 :
676 : std::string
677 40 : HDF5RawDataFile::get_record_header_dataset_path(const record_id_t& rid)
678 : {
679 40 : auto rec_id = get_all_record_ids().find(rid);
680 40 : if (rec_id == get_all_record_ids().end())
681 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
682 :
683 40 : if (get_version() <= 2) {
684 0 : return (m_file_ptr->getPath() + m_file_layout_ptr->get_record_header_path(rid.first, rid.second));
685 : } else {
686 40 : daqdataformats::SourceID source_id = get_record_header_source_id(rid);
687 40 : return m_source_id_path_cache[rid][source_id];
688 : }
689 : }
690 :
691 : std::string
692 0 : HDF5RawDataFile::get_record_header_dataset_path(const uint64_t rec_num, // NOLINT (build/unsigned)
693 : const daqdataformats::sequence_number_t seq_num)
694 : {
695 0 : return get_record_header_dataset_path(std::make_pair(rec_num, seq_num));
696 : }
697 :
698 : std::string
699 0 : HDF5RawDataFile::get_trigger_record_header_dataset_path(const record_id_t& rid)
700 : {
701 0 : check_record_type("TriggerRecord");
702 0 : return get_record_header_dataset_path(rid);
703 : }
704 :
705 : std::string
706 0 : HDF5RawDataFile::get_trigger_record_header_dataset_path(const daqdataformats::trigger_number_t trig_num,
707 : const daqdataformats::sequence_number_t seq_num)
708 : {
709 0 : check_record_type("TriggerRecord");
710 0 : return get_record_header_dataset_path(trig_num, seq_num);
711 : }
712 :
713 : std::string
714 0 : HDF5RawDataFile::get_timeslice_header_dataset_path(const record_id_t& rid)
715 : {
716 0 : check_record_type("TimeSlice");
717 0 : return get_record_header_dataset_path(rid.first, 0);
718 : }
719 :
720 : std::string
721 0 : HDF5RawDataFile::get_timeslice_header_dataset_path(const daqdataformats::timeslice_number_t ts_num)
722 : {
723 0 : check_record_type("TimeSlice");
724 0 : return get_record_header_dataset_path(ts_num);
725 : }
726 :
727 : /**
728 : * @brief Return a vector of dataset names that correspond to Fragemnts
729 : * Note: this gets all datsets, and then removes those that look like TriggerRecordHeader ones
730 : * one could instead loop through all system types and ask for appropriate datsets in those
731 : * however, probably that's more time consuming
732 : */
733 : std::vector<std::string>
734 8 : HDF5RawDataFile::get_all_fragment_dataset_paths()
735 : {
736 8 : std::vector<std::string> frag_paths;
737 :
738 468 : for (auto const& path : get_dataset_paths()) {
739 460 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) == std::string::npos)
740 420 : frag_paths.push_back(path);
741 8 : }
742 :
743 8 : return frag_paths;
744 0 : }
745 :
746 : // get all fragment dataset paths for given record ID
747 : std::vector<std::string>
748 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid)
749 : {
750 0 : auto rec_id = get_all_record_ids().find(rid);
751 0 : if (rec_id == get_all_record_ids().end())
752 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
753 :
754 0 : std::vector<std::string> frag_paths;
755 0 : if (get_version() <= 2) {
756 0 : std::string record_group_path =
757 0 : m_file_ptr->getPath() + m_file_layout_ptr->get_record_number_string(rid.first, rid.second);
758 :
759 0 : for (auto const& path : get_dataset_paths(record_group_path)) {
760 0 : if (path.find(m_file_layout_ptr->get_record_header_dataset_name()) == std::string::npos)
761 0 : frag_paths.push_back(path);
762 0 : }
763 0 : } else {
764 0 : std::set<daqdataformats::SourceID> source_id_list = get_fragment_source_ids(rid);
765 0 : for (auto const& source_id : source_id_list) {
766 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
767 : }
768 0 : }
769 0 : return frag_paths;
770 0 : }
771 :
772 : // get all fragment dataset paths for given record ID
773 : std::vector<std::string>
774 0 : HDF5RawDataFile::get_fragment_dataset_paths(const uint64_t rec_num, // NOLINT (build/unsigned)
775 : const daqdataformats::sequence_number_t seq_num)
776 : {
777 0 : return get_fragment_dataset_paths(std::make_pair(rec_num, seq_num));
778 : }
779 :
780 : // get all fragment dataset paths for a Subsystem
781 : std::vector<std::string>
782 0 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID::Subsystem subsystem)
783 : {
784 0 : std::vector<std::string> frag_paths;
785 0 : for (auto const& rid : get_all_record_ids()) {
786 0 : if (get_version() <= 2) {
787 0 : auto datasets = get_dataset_paths(m_file_ptr->getPath() +
788 0 : m_file_layout_ptr->get_fragment_type_path(rid.first, rid.second, subsystem));
789 0 : frag_paths.insert(frag_paths.end(), datasets.begin(), datasets.end());
790 0 : } else {
791 0 : std::set<daqdataformats::SourceID> source_id_list = get_source_ids_for_subsystem(rid, subsystem);
792 0 : for (auto const& source_id : source_id_list) {
793 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
794 : }
795 0 : }
796 0 : }
797 0 : return frag_paths;
798 0 : }
799 :
800 : std::vector<std::string>
801 0 : HDF5RawDataFile::get_fragment_dataset_paths(const std::string& subsystem_name)
802 : {
803 0 : daqdataformats::SourceID::Subsystem subsystem = daqdataformats::SourceID::string_to_subsystem(subsystem_name);
804 0 : return get_fragment_dataset_paths(subsystem);
805 : }
806 :
807 : std::vector<std::string>
808 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid, const daqdataformats::SourceID::Subsystem subsystem)
809 : {
810 0 : auto rec_id = get_all_record_ids().find(rid);
811 0 : if (rec_id == get_all_record_ids().end())
812 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
813 :
814 0 : if (get_version() <= 2) {
815 0 : return get_dataset_paths(m_file_ptr->getPath() +
816 0 : m_file_layout_ptr->get_fragment_type_path(rid.first, rid.second, subsystem));
817 : } else {
818 0 : std::vector<std::string> frag_paths;
819 0 : std::set<daqdataformats::SourceID> source_id_list = get_source_ids_for_subsystem(rid, subsystem);
820 0 : for (auto const& source_id : source_id_list) {
821 0 : frag_paths.push_back(m_source_id_path_cache[rid][source_id]);
822 : }
823 0 : return frag_paths;
824 0 : }
825 : }
826 :
827 : std::vector<std::string>
828 0 : HDF5RawDataFile::get_fragment_dataset_paths(const record_id_t& rid, const std::string& subsystem_name)
829 : {
830 0 : daqdataformats::SourceID::Subsystem subsystem = daqdataformats::SourceID::string_to_subsystem(subsystem_name);
831 0 : return get_fragment_dataset_paths(rid, subsystem);
832 : }
833 :
834 : #if 0
835 : // get all fragment dataset paths for a SourceID
836 : std::vector<std::string>
837 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID& source_id)
838 : {
839 : std::vector<std::string> frag_paths;
840 :
841 : for (auto const& rid : get_all_record_ids())
842 : frag_paths.push_back(m_file_ptr->getPath() +
843 : m_file_layout_ptr->get_fragment_path(rid.first, rid.second, source_id));
844 :
845 : return frag_paths;
846 : }
847 :
848 : std::vector<std::string>
849 : HDF5RawDataFile::get_fragment_dataset_paths(const daqdataformats::SourceID::Subsystem type,
850 : const uint32_t id) // NOLINT(build/unsigned)
851 : {
852 : return get_fragment_dataset_paths(daqdataformats::SourceID(type, source_id));
853 : }
854 : std::vector<std::string>
855 : HDF5RawDataFile::get_fragment_dataset_paths(const std::string& typestring,
856 : const uint32_t id) // NOLINT(build/unsigned)
857 : {
858 : return get_fragment_dataset_paths(
859 : daqdataformats::SourceID(daqdataformats::SourceID::string_to_subsystem(typestring), source_id));
860 : }
861 :
862 : std::set<daqdataformats::SourceID>
863 : HDF5RawDataFile::get_source_ids(std::vector<std::string> const& frag_dataset_paths)
864 : {
865 : std::set<daqdataformats::SourceID> source_ids;
866 : std::vector<std::string> path_elements;
867 : std::string s;
868 : for (auto const& frag_dataset : frag_dataset_paths) {
869 : path_elements.clear();
870 : std::istringstream iss(frag_dataset);
871 : while (std::getline(iss, s, '/')) {
872 : if (s.size() > 0)
873 : path_elements.push_back(s);
874 : }
875 : source_ids.insert(m_file_layout_ptr->get_source_id_from_path_elements(path_elements));
876 : }
877 :
878 : return source_ids;
879 : }
880 : #endif
881 :
882 : HDF5SourceIDHandler::source_id_geo_id_map_t
883 0 : HDF5RawDataFile::get_srcid_geoid_map() const
884 : {
885 :
886 0 : return m_file_level_source_id_geo_id_map;
887 : }
888 :
889 : std::set<uint64_t> // NOLINT(build/unsigned)
890 0 : HDF5RawDataFile::get_all_geo_ids() const
891 : {
892 0 : std::set<uint64_t> set_of_geo_ids;
893 : // 13-Sep-2022, KAB
894 : // It would be safer, but slower, to fetch all of the geo_ids from the
895 : // individual records, and we'll go with faster, for now. If/when we
896 : // change the way that we determine the file-level and record-level
897 : // source_id-to-geo_id maps, we may need to change this code.
898 0 : for (auto const& map_entry : m_file_level_source_id_geo_id_map) {
899 0 : for (auto const& geo_id : map_entry.second) {
900 0 : set_of_geo_ids.insert(geo_id);
901 : }
902 : }
903 0 : return set_of_geo_ids;
904 0 : }
905 :
906 : std::set<uint64_t> // NOLINT(build/unsigned)
907 0 : HDF5RawDataFile::get_geo_ids(const record_id_t& rid)
908 : {
909 0 : auto rec_id = get_all_record_ids().find(rid);
910 0 : if (rec_id == get_all_record_ids().end())
911 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
912 :
913 0 : add_record_level_info_to_caches_if_needed(rid);
914 :
915 0 : std::set<uint64_t> set_of_geo_ids;
916 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
917 0 : for (auto const& geo_id : map_entry.second) {
918 0 : set_of_geo_ids.insert(geo_id);
919 : }
920 : }
921 0 : return set_of_geo_ids;
922 0 : }
923 :
924 : std::set<uint64_t> // NOLINT(build/unsigned)
925 0 : HDF5RawDataFile::get_geo_ids_for_subdetector(const record_id_t& rid, const detdataformats::DetID::Subdetector subdet)
926 : {
927 0 : auto rec_id = get_all_record_ids().find(rid);
928 0 : if (rec_id == get_all_record_ids().end())
929 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
930 :
931 0 : add_record_level_info_to_caches_if_needed(rid);
932 :
933 0 : std::set<uint64_t> set_of_geo_ids;
934 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
935 0 : for (auto const& geo_id : map_entry.second) {
936 : // FIXME: replace with a proper coder/decoder
937 :
938 0 : uint16_t det_id = 0xffff & geo_id;
939 0 : if (det_id == static_cast<uint16_t>(subdet)) {
940 0 : set_of_geo_ids.insert(geo_id);
941 : }
942 : }
943 : }
944 0 : return set_of_geo_ids;
945 0 : }
946 :
947 : // get all SourceIDs for given record ID
948 : std::set<daqdataformats::SourceID>
949 0 : HDF5RawDataFile::get_source_ids(const record_id_t& rid)
950 : {
951 0 : auto rec_id = get_all_record_ids().find(rid);
952 0 : if (rec_id == get_all_record_ids().end())
953 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
954 :
955 0 : add_record_level_info_to_caches_if_needed(rid);
956 :
957 0 : return m_source_id_cache[rid];
958 : }
959 :
960 : daqdataformats::SourceID
961 40 : HDF5RawDataFile::get_record_header_source_id(const record_id_t& rid)
962 : {
963 40 : auto rec_id = get_all_record_ids().find(rid);
964 40 : if (rec_id == get_all_record_ids().end())
965 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
966 :
967 40 : add_record_level_info_to_caches_if_needed(rid);
968 :
969 40 : return m_record_header_source_id_cache[rid];
970 : }
971 :
972 : std::set<daqdataformats::SourceID>
973 0 : HDF5RawDataFile::get_fragment_source_ids(const record_id_t& rid)
974 : {
975 0 : auto rec_id = get_all_record_ids().find(rid);
976 0 : if (rec_id == get_all_record_ids().end())
977 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
978 :
979 0 : add_record_level_info_to_caches_if_needed(rid);
980 :
981 0 : return m_fragment_source_id_cache[rid];
982 : }
983 :
984 : std::set<daqdataformats::SourceID>
985 0 : HDF5RawDataFile::get_source_ids_for_subsystem(const record_id_t& rid,
986 : const daqdataformats::SourceID::Subsystem subsystem)
987 : {
988 0 : auto rec_id = get_all_record_ids().find(rid);
989 0 : if (rec_id == get_all_record_ids().end())
990 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
991 :
992 0 : add_record_level_info_to_caches_if_needed(rid);
993 :
994 0 : return m_subsystem_source_id_cache[rid][subsystem];
995 : }
996 :
997 : std::set<daqdataformats::SourceID>
998 0 : HDF5RawDataFile::get_source_ids_for_fragment_type(const record_id_t& rid, const daqdataformats::FragmentType frag_type)
999 : {
1000 0 : auto rec_id = get_all_record_ids().find(rid);
1001 0 : if (rec_id == get_all_record_ids().end())
1002 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1003 :
1004 0 : add_record_level_info_to_caches_if_needed(rid);
1005 :
1006 0 : return m_fragment_type_source_id_cache[rid][frag_type];
1007 : }
1008 :
1009 : std::set<daqdataformats::SourceID>
1010 0 : HDF5RawDataFile::get_source_ids_for_fragtype_and_subdetector(const record_id_t& rid,
1011 : const std::string& frag_type_name,
1012 : const std::string& subdet_name)
1013 : {
1014 0 : daqdataformats::FragmentType frag_type = daqdataformats::string_to_fragment_type(frag_type_name);
1015 0 : if (frag_type == daqdataformats::FragmentType::kUnknown)
1016 0 : throw InvalidFragmentTypeString(ERS_HERE, frag_type_name);
1017 0 : detdataformats::DetID::Subdetector subdet = detdataformats::DetID::string_to_subdetector(subdet_name);
1018 0 : if (subdet == detdataformats::DetID::Subdetector::kUnknown)
1019 0 : throw InvalidSubdetectorString(ERS_HERE, subdet_name);
1020 :
1021 0 : auto rec_id = get_all_record_ids().find(rid);
1022 0 : if (rec_id == get_all_record_ids().end())
1023 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1024 :
1025 0 : add_record_level_info_to_caches_if_needed(rid);
1026 :
1027 0 : std::set<daqdataformats::SourceID> fragtype_match_sids = m_fragment_type_source_id_cache[rid][frag_type];
1028 0 : std::set<daqdataformats::SourceID> detid_match_sids = m_subdetector_source_id_cache[rid][subdet];
1029 0 : std::set<daqdataformats::SourceID> combined_set_sids;
1030 0 : for (auto ftsid : fragtype_match_sids) {
1031 0 : if (detid_match_sids.contains(ftsid)) {
1032 0 : combined_set_sids.insert(ftsid);
1033 : }
1034 : }
1035 0 : return combined_set_sids;
1036 0 : }
1037 :
1038 : std::set<daqdataformats::SourceID>
1039 0 : HDF5RawDataFile::get_source_ids_for_subdetector(const record_id_t& rid, const detdataformats::DetID::Subdetector subdet)
1040 : {
1041 0 : auto rec_id = get_all_record_ids().find(rid);
1042 0 : if (rec_id == get_all_record_ids().end())
1043 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1044 :
1045 0 : add_record_level_info_to_caches_if_needed(rid);
1046 :
1047 0 : return m_subdetector_source_id_cache[rid][subdet];
1048 : }
1049 :
1050 : std::unique_ptr<char[]>
1051 48 : HDF5RawDataFile::get_dataset_raw_data(const std::string& dataset_path)
1052 : {
1053 48 : HighFive::Group parent_group = m_file_ptr->getGroup("/");
1054 48 : HighFive::DataSet data_set = parent_group.getDataSet(dataset_path);
1055 :
1056 48 : if (!data_set.isValid())
1057 0 : throw InvalidHDF5Dataset(ERS_HERE, dataset_path, get_file_name());
1058 :
1059 48 : size_t data_size = data_set.getSpace().getElementCount() * sizeof(char);
1060 :
1061 48 : auto membuffer = std::make_unique<char[]>(data_size);
1062 :
1063 48 : data_set.read(membuffer.get());
1064 48 : return membuffer;
1065 48 : }
1066 :
1067 : std::unique_ptr<daqdataformats::Fragment>
1068 32 : HDF5RawDataFile::get_frag_ptr(const std::string& dataset_name)
1069 : {
1070 32 : auto membuffer = get_dataset_raw_data(dataset_name);
1071 32 : auto frag_ptr = std::make_unique<daqdataformats::Fragment>(
1072 32 : membuffer.release(), dunedaq::daqdataformats::Fragment::BufferAdoptionMode::kTakeOverBuffer);
1073 32 : return frag_ptr;
1074 32 : }
1075 :
1076 : std::unique_ptr<daqdataformats::Fragment>
1077 24 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid, const daqdataformats::SourceID& source_id)
1078 : {
1079 24 : if (get_version() < 2)
1080 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1081 :
1082 24 : auto rec_id = get_all_record_ids().find(rid);
1083 24 : if (rec_id == get_all_record_ids().end())
1084 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1085 :
1086 24 : add_record_level_info_to_caches_if_needed(rid);
1087 :
1088 24 : return get_frag_ptr(m_source_id_path_cache[rid][source_id]);
1089 : }
1090 :
1091 : std::unique_ptr<daqdataformats::Fragment>
1092 8 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1093 : const daqdataformats::sequence_number_t seq_num,
1094 : const daqdataformats::SourceID& source_id)
1095 : {
1096 8 : record_id_t rid = std::make_pair(rec_num, seq_num);
1097 8 : return get_frag_ptr(rid, source_id);
1098 : }
1099 :
1100 : std::unique_ptr<daqdataformats::Fragment>
1101 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1102 : const daqdataformats::SourceID::Subsystem type,
1103 : const uint32_t id) // NOLINT(build/unsigned)
1104 : {
1105 0 : daqdataformats::SourceID source_id(type, id);
1106 0 : return get_frag_ptr(rid, source_id);
1107 : }
1108 :
1109 : std::unique_ptr<daqdataformats::Fragment>
1110 0 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1111 : const daqdataformats::sequence_number_t seq_num,
1112 : const daqdataformats::SourceID::Subsystem type,
1113 : const uint32_t id) // NOLINT(build/unsigned)
1114 : {
1115 0 : record_id_t rid = std::make_pair(rec_num, seq_num);
1116 0 : daqdataformats::SourceID source_id(type, id);
1117 0 : return get_frag_ptr(rid, source_id);
1118 : }
1119 :
1120 : std::unique_ptr<daqdataformats::Fragment>
1121 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1122 : const std::string& typestring,
1123 : const uint32_t id) // NOLINT(build/unsigned)
1124 : {
1125 0 : daqdataformats::SourceID source_id(daqdataformats::SourceID::string_to_subsystem(typestring), id);
1126 0 : return get_frag_ptr(rid, source_id);
1127 : }
1128 :
1129 : std::unique_ptr<daqdataformats::Fragment>
1130 16 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1131 : const daqdataformats::sequence_number_t seq_num,
1132 : const std::string& typestring,
1133 : const uint32_t id) // NOLINT(build/unsigned)
1134 : {
1135 16 : record_id_t rid = std::make_pair(rec_num, seq_num);
1136 16 : daqdataformats::SourceID source_id(daqdataformats::SourceID::string_to_subsystem(typestring), id);
1137 16 : return get_frag_ptr(rid, source_id);
1138 : }
1139 :
1140 : std::unique_ptr<daqdataformats::Fragment>
1141 0 : HDF5RawDataFile::get_frag_ptr(const record_id_t& rid,
1142 : const uint64_t geo_id) // NOLINT(build/unsigned)
1143 : {
1144 0 : daqdataformats::SourceID sid = get_source_id_for_geo_id(rid, geo_id);
1145 0 : return get_frag_ptr(rid, sid);
1146 : }
1147 :
1148 : std::unique_ptr<daqdataformats::Fragment>
1149 0 : HDF5RawDataFile::get_frag_ptr(const uint64_t rec_num, // NOLINT(build/unsigned)
1150 : const daqdataformats::sequence_number_t seq_num,
1151 : const uint64_t geo_id) // NOLINT(build/unsigned)
1152 : {
1153 0 : record_id_t rid = std::make_pair(rec_num, seq_num);
1154 0 : return get_frag_ptr(rid, geo_id);
1155 : }
1156 :
1157 : std::unique_ptr<daqdataformats::TriggerRecordHeader>
1158 8 : HDF5RawDataFile::get_trh_ptr(const std::string& dataset_name)
1159 : {
1160 8 : auto membuffer = get_dataset_raw_data(dataset_name);
1161 8 : auto trh_ptr = std::make_unique<daqdataformats::TriggerRecordHeader>(membuffer.release(), true);
1162 8 : return trh_ptr;
1163 8 : }
1164 :
1165 : std::unique_ptr<daqdataformats::TriggerRecordHeader>
1166 4 : HDF5RawDataFile::get_trh_ptr(const record_id_t& rid)
1167 : {
1168 4 : if (get_version() < 2)
1169 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1170 :
1171 4 : auto rec_id = get_all_record_ids().find(rid);
1172 4 : if (rec_id == get_all_record_ids().end())
1173 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1174 :
1175 4 : add_record_level_info_to_caches_if_needed(rid);
1176 :
1177 4 : daqdataformats::SourceID rh_source_id = m_record_header_source_id_cache[rid];
1178 4 : return get_trh_ptr(m_source_id_path_cache[rid][rh_source_id]);
1179 : }
1180 :
1181 : std::unique_ptr<daqdataformats::TimeSliceHeader>
1182 8 : HDF5RawDataFile::get_tsh_ptr(const std::string& dataset_name)
1183 : {
1184 8 : auto membuffer = get_dataset_raw_data(dataset_name);
1185 8 : auto tsh_ptr = std::make_unique<daqdataformats::TimeSliceHeader>(
1186 8 : *(reinterpret_cast<daqdataformats::TimeSliceHeader*>(membuffer.release()))); // NOLINT
1187 8 : return tsh_ptr;
1188 8 : }
1189 :
1190 : std::unique_ptr<daqdataformats::TimeSliceHeader>
1191 4 : HDF5RawDataFile::get_tsh_ptr(const record_id_t& rid)
1192 : {
1193 4 : if (get_version() < 2)
1194 0 : throw IncompatibleFileLayoutVersion(ERS_HERE, get_version(), 2, MAX_FILELAYOUT_VERSION);
1195 :
1196 4 : auto rec_id = get_all_record_ids().find(rid);
1197 4 : if (rec_id == get_all_record_ids().end())
1198 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1199 :
1200 4 : add_record_level_info_to_caches_if_needed(rid);
1201 :
1202 4 : daqdataformats::SourceID rh_source_id = m_record_header_source_id_cache[rid];
1203 4 : return get_tsh_ptr(m_source_id_path_cache[rid][rh_source_id]);
1204 : }
1205 :
1206 : daqdataformats::TriggerRecord
1207 0 : HDF5RawDataFile::get_trigger_record(const record_id_t& rid)
1208 : {
1209 0 : daqdataformats::TriggerRecord trigger_record(*get_trh_ptr(rid));
1210 0 : for (auto const& frag_path : get_fragment_dataset_paths(rid)) {
1211 0 : trigger_record.add_fragment(get_frag_ptr(frag_path));
1212 0 : }
1213 :
1214 0 : return trigger_record;
1215 0 : }
1216 :
1217 : daqdataformats::TimeSlice
1218 0 : HDF5RawDataFile::get_timeslice(const daqdataformats::timeslice_number_t ts_num)
1219 : {
1220 0 : daqdataformats::TimeSlice timeslice(*get_tsh_ptr(ts_num));
1221 0 : for (auto const& frag_path : get_fragment_dataset_paths(ts_num)) {
1222 0 : timeslice.add_fragment(get_frag_ptr(frag_path));
1223 0 : }
1224 :
1225 0 : return timeslice;
1226 0 : }
1227 :
1228 : std::vector<uint64_t> // NOLINT(build/unsigned)
1229 0 : HDF5RawDataFile::get_geo_ids_for_source_id(const record_id_t& rid, const daqdataformats::SourceID& source_id)
1230 : {
1231 0 : auto rec_id = get_all_record_ids().find(rid);
1232 0 : if (rec_id == get_all_record_ids().end())
1233 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1234 :
1235 0 : add_record_level_info_to_caches_if_needed(rid);
1236 :
1237 0 : return m_source_id_geo_id_cache[rid][source_id];
1238 : }
1239 :
1240 : daqdataformats::SourceID
1241 0 : HDF5RawDataFile::get_source_id_for_geo_id(const record_id_t& rid,
1242 : const uint64_t requested_geo_id) // NOLINT(build/unsigned)
1243 : {
1244 0 : auto rec_id = get_all_record_ids().find(rid);
1245 0 : if (rec_id == get_all_record_ids().end())
1246 0 : throw RecordIDNotFound(ERS_HERE, rid.first, rid.second);
1247 :
1248 0 : add_record_level_info_to_caches_if_needed(rid);
1249 :
1250 : // if we want to make this faster, we could build a reverse lookup cache in
1251 : // add_record_level_info_to_caches_if_needed() and just look up the requested geo_id here
1252 0 : for (auto const& map_entry : m_source_id_geo_id_cache[rid]) {
1253 0 : auto geoid_list = map_entry.second;
1254 0 : for (auto const& geoid_from_list : geoid_list) {
1255 0 : if (geoid_from_list == requested_geo_id) {
1256 0 : return map_entry.first;
1257 : }
1258 : }
1259 0 : }
1260 :
1261 0 : daqdataformats::SourceID empty_sid;
1262 0 : return empty_sid;
1263 : }
1264 :
1265 : } // namespace hdf5libs
1266 : } // namespace dunedaq
|
