Line data Source code
1 : /**
2 : * @file HDF5Write_test.cxx Application that tests and demonstrates
3 : * the write functionality of the HDF5DataStore class.
4 : *
5 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
6 : * Licensing/copyright details are in the COPYING file that you should have
7 : * received with this code.
8 : */
9 :
10 : #include "dfmodules/DataStore.hpp"
11 :
12 : // We need the actual HDF5DataStore.hpp plugin header so the unit tests can access its exceptions
13 : #include "../plugins/HDF5DataStore.hpp" // NOLINT(build/include_path)
14 :
15 : #include "appmodel/DataStoreConf.hpp"
16 : #include "appmodel/DataWriterConf.hpp"
17 : #include "appmodel/DataWriterModule.hpp"
18 : #include "appmodel/FilenameParams.hpp"
19 : #include "confmodel/DetectorConfig.hpp"
20 : #include "confmodel/Session.hpp"
21 : #include "detdataformats/DetID.hpp"
22 :
23 : #define BOOST_TEST_MODULE HDF5Write_test // NOLINT
24 :
25 : #include "boost/test/unit_test.hpp"
26 :
27 : #include <cstdlib>
28 : #include <filesystem>
29 : #include <fstream>
30 : #include <iostream>
31 : #include <limits>
32 : #include <memory>
33 : #include <regex>
34 : #include <string>
35 : #include <utility>
36 : #include <vector>
37 :
38 : using namespace dunedaq::dfmodules;
39 :
40 : std::vector<std::string>
41 11 : get_files_matching_pattern(const std::string& path, const std::string& pattern)
42 : {
43 11 : std::regex regex_search_pattern(pattern);
44 11 : std::vector<std::string> file_list;
45 4553 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
46 4542 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
47 20 : file_list.push_back(entry.path());
48 : }
49 11 : }
50 11 : return file_list;
51 11 : }
52 :
53 : std::vector<std::string>
54 16 : delete_files_matching_pattern(const std::string& path, const std::string& pattern)
55 : {
56 16 : std::regex regex_search_pattern(pattern);
57 16 : std::vector<std::string> file_list;
58 6609 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
59 6593 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
60 16 : if (std::filesystem::remove(entry.path())) {
61 16 : file_list.push_back(entry.path());
62 : }
63 : }
64 16 : }
65 16 : return file_list;
66 16 : }
67 :
68 : dunedaq::daqdataformats::TriggerRecord
69 35 : create_trigger_record(int trig_num, int fragment_size, int element_count)
70 : {
71 35 : const int run_number = 53;
72 35 : const dunedaq::daqdataformats::SourceID::Subsystem stype_to_use =
73 : dunedaq::daqdataformats::SourceID::Subsystem::kDetectorReadout;
74 :
75 : // setup our dummy_data
76 35 : std::vector<char> dummy_vector(fragment_size);
77 35 : char* dummy_data = dummy_vector.data();
78 :
79 : // get a timestamp for this trigger
80 70 : uint64_t ts = std::chrono::duration_cast<std::chrono::milliseconds>( // NOLINT(build/unsigned)
81 35 : system_clock::now().time_since_epoch())
82 35 : .count();
83 :
84 : // create TriggerRecordHeader
85 35 : dunedaq::daqdataformats::TriggerRecordHeaderData trh_data;
86 35 : trh_data.trigger_number = trig_num;
87 35 : trh_data.trigger_timestamp = ts;
88 35 : trh_data.num_requested_components = element_count;
89 35 : trh_data.run_number = run_number;
90 35 : trh_data.sequence_number = 0;
91 35 : trh_data.max_sequence_number = 1;
92 35 : trh_data.element_id = dunedaq::daqdataformats::SourceID(dunedaq::daqdataformats::SourceID::Subsystem::kTRBuilder, 0);
93 :
94 35 : dunedaq::daqdataformats::TriggerRecordHeader trh(&trh_data);
95 :
96 : // create out TriggerRecord
97 35 : dunedaq::daqdataformats::TriggerRecord tr(trh);
98 :
99 : // loop over elements
100 855 : for (int ele_num = 0; ele_num < element_count; ++ele_num) {
101 :
102 : // create our fragment
103 820 : dunedaq::daqdataformats::FragmentHeader fh;
104 820 : fh.trigger_number = trig_num;
105 820 : fh.trigger_timestamp = ts;
106 820 : fh.window_begin = ts - 10;
107 820 : fh.window_end = ts;
108 820 : fh.run_number = run_number;
109 820 : fh.sequence_number = 0;
110 820 : fh.fragment_type =
111 : static_cast<dunedaq::daqdataformats::fragment_type_t>(dunedaq::daqdataformats::FragmentType::kWIB);
112 820 : fh.detector_id = static_cast<uint16_t>(dunedaq::detdataformats::DetID::Subdetector::kHD_TPC);
113 820 : fh.element_id = dunedaq::daqdataformats::SourceID(stype_to_use, ele_num);
114 820 : std::unique_ptr<dunedaq::daqdataformats::Fragment> frag_ptr(
115 820 : new dunedaq::daqdataformats::Fragment(dummy_data, fragment_size));
116 820 : frag_ptr->set_header_fields(fh);
117 :
118 : // add fragment to TriggerRecord
119 820 : tr.add_fragment(std::move(frag_ptr));
120 :
121 820 : } // end loop over elements
122 :
123 35 : return tr;
124 35 : }
125 :
126 : struct CfgFixture
127 : {
128 8 : CfgFixture(std::string sessionName)
129 8 : {
130 8 : TLOG_DEBUG(4) << "Creating CfgFixture";
131 8 : setenv("DUNEDAQ_SESSION", sessionName.c_str(), 1);
132 8 : std::string oksConfig = "oksconflibs:test/config/hdf5write_test.data.xml";
133 8 : std::string appName = "TestApp";
134 8 : cfgMgr = std::make_shared<dunedaq::appfwk::ConfigurationManager>(oksConfig, appName, sessionName);
135 8 : TLOG_DEBUG(4) << "Done with CfgFixture";
136 8 : }
137 :
138 : const dunedaq::confmodel::DetectorConfig* get_detector_config()
139 : {
140 : return cfgMgr->get_session()->get_detector_configuration();
141 : }
142 :
143 : std::shared_ptr<dunedaq::appfwk::ConfigurationManager> cfgMgr;
144 : };
145 :
146 : BOOST_AUTO_TEST_SUITE(HDF5Write_test)
147 :
148 2 : BOOST_AUTO_TEST_CASE(NullConfiguration)
149 : {
150 5 : BOOST_CHECK_THROW(make_data_store("HDF5DataStore", "dummy", nullptr, "dummy"),
151 : dunedaq::dfmodules::FileDataStoreImplBadConfiguration);
152 1 : }
153 :
154 2 : BOOST_AUTO_TEST_CASE(WriteEventFiles)
155 : {
156 1 : std::string file_path(std::filesystem::temp_directory_path());
157 :
158 1 : const int trigger_count = 5;
159 1 : const int apa_count = 3;
160 1 : const int link_count = 1;
161 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
162 :
163 : // delete any pre-existing files so that we start with a clean slate
164 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
165 1 : delete_files_matching_pattern(file_path, delete_pattern);
166 :
167 : // create the DataStore
168 1 : CfgFixture cfg("test-session-3-1");
169 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
170 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
171 :
172 1 : auto data_store_conf_obj = data_store_conf->config_object();
173 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
174 1 : data_store_conf_obj.set_by_val<std::string>("mode", "one-event-per-file");
175 :
176 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
177 :
178 : // write several events, each with several fragments
179 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
180 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, link_count * apa_count));
181 :
182 1 : data_store_ptr.reset(); // explicit destruction
183 :
184 : // check that the expected number of files was created
185 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
186 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
187 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
188 :
189 : // clean up the files that were created
190 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
191 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
192 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
193 1 : }
194 :
195 2 : BOOST_AUTO_TEST_CASE(WriteOneFile)
196 : {
197 1 : std::string file_path(std::filesystem::temp_directory_path());
198 :
199 1 : const int trigger_count = 5;
200 1 : const int apa_count = 3;
201 1 : const int link_count = 1;
202 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
203 :
204 : // delete any pre-existing files so that we start with a clean slate
205 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
206 1 : delete_files_matching_pattern(file_path, delete_pattern);
207 :
208 : // create the DataStore
209 1 : CfgFixture cfg("test-session-3-1");
210 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
211 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
212 :
213 1 : auto data_store_conf_obj = data_store_conf->config_object();
214 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
215 :
216 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
217 :
218 : // write several events, each with several fragments
219 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
220 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
221 :
222 1 : data_store_ptr.reset(); // explicit destruction
223 :
224 : // check that the expected number of files was created
225 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
226 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
227 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
228 :
229 : // clean up the files that were created
230 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
231 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
232 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
233 1 : }
234 :
235 2 : BOOST_AUTO_TEST_CASE(CheckWritingSuffix)
236 : {
237 1 : std::string file_path(std::filesystem::temp_directory_path());
238 :
239 1 : const int trigger_count = 5;
240 1 : const int apa_count = 3;
241 1 : const int link_count = 1;
242 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
243 :
244 : // delete any pre-existing files so that we start with a clean slate
245 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
246 1 : delete_files_matching_pattern(file_path, delete_pattern);
247 :
248 : // create the DataStore
249 1 : CfgFixture cfg("test-session-3-1");
250 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
251 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
252 :
253 1 : auto data_store_conf_obj = data_store_conf->config_object();
254 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
255 :
256 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
257 :
258 : // write several events, each with several fragments
259 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number) {
260 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
261 :
262 : // check that the .writing file is there
263 5 : std::string search_pattern = "hdf5writetest.*\\.writing";
264 5 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
265 5 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
266 5 : }
267 :
268 1 : data_store_ptr.reset(); // explicit destruction
269 :
270 : // check that the expected number of files was created
271 1 : std::string search_pattern = "hdf5writetest.*\\.writing";
272 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
273 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 0);
274 :
275 : // clean up the files that were created
276 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
277 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
278 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
279 1 : }
280 :
281 2 : BOOST_AUTO_TEST_CASE(NoDuplicateTimeSlices)
282 : {
283 1 : std::string file_path(std::filesystem::temp_directory_path());
284 :
285 1 : CfgFixture cfg("test-session-3-1");
286 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
287 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
288 :
289 1 : auto data_store_conf_obj = data_store_conf->config_object();
290 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
291 :
292 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
293 :
294 1 : dunedaq::daqdataformats::TimeSlice timeslice{ 999, 999 }; // timeslice #, run #
295 1 : dunedaq::daqdataformats::TimeSlice identical_timeslice{ 999, 999 };
296 :
297 1 : data_store_ptr->write(timeslice);
298 :
299 1 : std::string search_pattern = "hdf5writetest.*\\.writing";
300 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
301 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
302 :
303 1 : BOOST_CHECK_THROW(data_store_ptr->write(identical_timeslice), dunedaq::dfmodules::IgnorableDataStoreProblem);
304 :
305 1 : delete_files_matching_pattern(file_path, "hdf5writetest.*\\.hdf5");
306 1 : }
307 :
308 2 : BOOST_AUTO_TEST_CASE(WriteToBadOutputArea)
309 : {
310 1 : std::string file_path("/this/path/does/not/exist");
311 :
312 1 : CfgFixture cfg("test-session-3-1");
313 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
314 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
315 :
316 1 : auto data_store_conf_obj = data_store_conf->config_object();
317 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
318 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
319 :
320 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InvalidOutputPath);
321 1 : }
322 :
323 2 : BOOST_AUTO_TEST_CASE(EnormousMaxFileSize)
324 : {
325 1 : CfgFixture cfg("test-session-3-1");
326 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
327 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
328 :
329 1 : auto data_store_conf_obj = data_store_conf->config_object();
330 1 : data_store_conf_obj.set_by_val<size_t>("max_file_size", std::numeric_limits<size_t>::max());
331 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
332 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InsufficientDiskSpace);
333 1 : }
334 :
335 2 : BOOST_AUTO_TEST_CASE(FileSizeLimitResultsInMultipleFiles)
336 : {
337 1 : std::string file_path(std::filesystem::temp_directory_path());
338 :
339 1 : const int trigger_count = 15;
340 1 : const int apa_count = 5;
341 1 : const int link_count = 10;
342 1 : const int fragment_size = 10000;
343 :
344 : // 5 APAs times 10 links times 10000 bytes per fragment gives 500,000 bytes per TR
345 : // So, 15 TRs would give 7,500,000 bytes total.
346 :
347 : // delete any pre-existing files so that we start with a clean slate
348 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
349 1 : delete_files_matching_pattern(file_path, delete_pattern);
350 :
351 : // create the DataStore
352 1 : CfgFixture cfg("test-session-5-10");
353 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
354 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
355 :
356 1 : auto data_store_conf_obj = data_store_conf->config_object();
357 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
358 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 3000000); // goal is 6 events per file
359 :
360 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
361 :
362 : // write several events, each with several fragments
363 16 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
364 15 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
365 :
366 1 : data_store_ptr.reset(); // explicit destruction
367 :
368 : // check that the expected number of files was created
369 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
370 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
371 : // 7,500,000 bytes stored in files of size 3,000,000 should result in three files.
372 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
373 :
374 : // clean up the files that were created
375 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
376 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
377 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
378 1 : }
379 :
380 2 : BOOST_AUTO_TEST_CASE(SmallFileSizeLimitDataBlockListWrite)
381 : {
382 1 : std::string file_path(std::filesystem::temp_directory_path());
383 :
384 1 : const int trigger_count = 5;
385 1 : const int apa_count = 5;
386 1 : const int link_count = 1;
387 1 : const int fragment_size = 100000;
388 :
389 : // 5 APAs times 100000 bytes per fragment gives 500,000 bytes per TR
390 :
391 : // delete any pre-existing files so that we start with a clean slate
392 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
393 1 : delete_files_matching_pattern(file_path, delete_pattern);
394 :
395 : // create the DataStore
396 1 : CfgFixture cfg("test-session-5-1");
397 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
398 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
399 :
400 1 : auto data_store_conf_obj = data_store_conf->config_object();
401 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
402 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 150000); // ~1.5 Fragment, ~0.3 TR
403 :
404 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
405 :
406 : // write several events, each with several fragments
407 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
408 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
409 :
410 1 : data_store_ptr.reset(); // explicit destruction
411 :
412 : // check that the expected number of files was created
413 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
414 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
415 : // each TriggerRecord should be stored in its own file
416 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
417 :
418 : // clean up the files that were created
419 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
420 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
421 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
422 1 : }
423 :
424 : BOOST_AUTO_TEST_SUITE_END()
|
