Line data Source code
1 : /**
2 : * @file HDF5Write_test.cxx Application that tests and demonstrates
3 : * the write functionality of the HDF5DataStore class.
4 : *
5 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
6 : * Licensing/copyright details are in the COPYING file that you should have
7 : * received with this code.
8 : */
9 :
10 : #include "dfmodules/DataStore.hpp"
11 :
12 : // We need the actual HDF5DataStore.hpp plugin header so the unit tests can access its exceptions
13 : #include "../plugins/HDF5DataStore.hpp" // NOLINT(build/include_path)
14 :
15 : #include "appmodel/DataWriterModule.hpp"
16 : #include "appmodel/DataWriterConf.hpp"
17 : #include "appmodel/FilenameParams.hpp"
18 : #include "confmodel/DetectorConfig.hpp"
19 : #include "confmodel/Session.hpp"
20 : #include "appmodel/DataStoreConf.hpp"
21 : #include "detdataformats/DetID.hpp"
22 :
23 : #define BOOST_TEST_MODULE HDF5Write_test // NOLINT
24 :
25 : #include "boost/test/unit_test.hpp"
26 :
27 : #include <cstdlib>
28 : #include <filesystem>
29 : #include <fstream>
30 : #include <iostream>
31 : #include <limits>
32 : #include <memory>
33 : #include <regex>
34 : #include <string>
35 : #include <utility>
36 : #include <vector>
37 :
38 : using namespace dunedaq::dfmodules;
39 :
40 : std::vector<std::string>
41 11 : get_files_matching_pattern(const std::string& path, const std::string& pattern)
42 : {
43 11 : std::regex regex_search_pattern(pattern);
44 11 : std::vector<std::string> file_list;
45 3266 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
46 3255 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
47 20 : file_list.push_back(entry.path());
48 : }
49 11 : }
50 11 : return file_list;
51 11 : }
52 :
53 : std::vector<std::string>
54 16 : delete_files_matching_pattern(const std::string& path, const std::string& pattern)
55 : {
56 16 : std::regex regex_search_pattern(pattern);
57 16 : std::vector<std::string> file_list;
58 4737 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
59 4721 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
60 16 : if (std::filesystem::remove(entry.path())) {
61 16 : file_list.push_back(entry.path());
62 : }
63 : }
64 16 : }
65 16 : return file_list;
66 16 : }
67 :
68 : dunedaq::daqdataformats::TriggerRecord
69 35 : create_trigger_record(int trig_num, int fragment_size, int element_count)
70 : {
71 35 : const int run_number = 53;
72 35 : const dunedaq::daqdataformats::SourceID::Subsystem stype_to_use =
73 : dunedaq::daqdataformats::SourceID::Subsystem::kDetectorReadout;
74 :
75 : // setup our dummy_data
76 35 : std::vector<char> dummy_vector(fragment_size);
77 35 : char* dummy_data = dummy_vector.data();
78 :
79 : // get a timestamp for this trigger
80 70 : uint64_t ts = std::chrono::duration_cast<std::chrono::milliseconds>( // NOLINT(build/unsigned)
81 35 : system_clock::now().time_since_epoch())
82 35 : .count();
83 :
84 : // create TriggerRecordHeader
85 35 : dunedaq::daqdataformats::TriggerRecordHeaderData trh_data;
86 35 : trh_data.trigger_number = trig_num;
87 35 : trh_data.trigger_timestamp = ts;
88 35 : trh_data.num_requested_components = element_count;
89 35 : trh_data.run_number = run_number;
90 35 : trh_data.sequence_number = 0;
91 35 : trh_data.max_sequence_number = 1;
92 35 : trh_data.element_id = dunedaq::daqdataformats::SourceID(dunedaq::daqdataformats::SourceID::Subsystem::kTRBuilder, 0);
93 :
94 35 : dunedaq::daqdataformats::TriggerRecordHeader trh(&trh_data);
95 :
96 : // create out TriggerRecord
97 35 : dunedaq::daqdataformats::TriggerRecord tr(trh);
98 :
99 : // loop over elements
100 855 : for (int ele_num = 0; ele_num < element_count; ++ele_num) {
101 :
102 : // create our fragment
103 820 : dunedaq::daqdataformats::FragmentHeader fh;
104 820 : fh.trigger_number = trig_num;
105 820 : fh.trigger_timestamp = ts;
106 820 : fh.window_begin = ts - 10;
107 820 : fh.window_end = ts;
108 820 : fh.run_number = run_number;
109 820 : fh.sequence_number = 0;
110 820 : fh.fragment_type =
111 : static_cast<dunedaq::daqdataformats::fragment_type_t>(dunedaq::daqdataformats::FragmentType::kWIB);
112 820 : fh.detector_id = static_cast<uint16_t>(dunedaq::detdataformats::DetID::Subdetector::kHD_TPC);
113 820 : fh.element_id = dunedaq::daqdataformats::SourceID(stype_to_use, ele_num);
114 820 : std::unique_ptr<dunedaq::daqdataformats::Fragment> frag_ptr(
115 820 : new dunedaq::daqdataformats::Fragment(dummy_data, fragment_size));
116 820 : frag_ptr->set_header_fields(fh);
117 :
118 : // add fragment to TriggerRecord
119 820 : tr.add_fragment(std::move(frag_ptr));
120 :
121 820 : } // end loop over elements
122 :
123 35 : return tr;
124 35 : }
125 :
126 : struct CfgFixture
127 : {
128 8 : CfgFixture(std::string sessionName)
129 8 : {
130 8 : TLOG_DEBUG(4) << "Creating CfgFixture";
131 8 : setenv("DUNEDAQ_SESSION", sessionName.c_str(), 1);
132 8 : std::string oksConfig = "oksconflibs:test/config/hdf5write_test.data.xml";
133 8 : std::string appName = "TestApp";
134 8 : cfgMgr = std::make_shared<dunedaq::appfwk::ConfigurationManager>(oksConfig, appName, sessionName);
135 8 : TLOG_DEBUG(4) << "Done with CfgFixture";
136 8 : }
137 :
138 : const dunedaq::confmodel::DetectorConfig* get_detector_config()
139 : {
140 : return cfgMgr->get_session()->get_detector_configuration();
141 : }
142 :
143 : std::shared_ptr<dunedaq::appfwk::ConfigurationManager> cfgMgr;
144 : };
145 :
146 : BOOST_AUTO_TEST_SUITE(HDF5Write_test)
147 :
148 2 : BOOST_AUTO_TEST_CASE(WriteEventFiles)
149 : {
150 1 : std::string file_path(std::filesystem::temp_directory_path());
151 :
152 1 : const int trigger_count = 5;
153 1 : const int apa_count = 3;
154 1 : const int link_count = 1;
155 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
156 :
157 : // delete any pre-existing files so that we start with a clean slate
158 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
159 1 : delete_files_matching_pattern(file_path, delete_pattern);
160 :
161 : // create the DataStore
162 1 : CfgFixture cfg("test-session-3-1");
163 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
164 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
165 :
166 1 : auto data_store_conf_obj = data_store_conf->config_object();
167 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
168 1 : data_store_conf_obj.set_by_val<std::string>("mode", "one-event-per-file");
169 :
170 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
171 :
172 : // write several events, each with several fragments
173 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
174 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, link_count * apa_count));
175 :
176 1 : data_store_ptr.reset(); // explicit destruction
177 :
178 : // check that the expected number of files was created
179 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
180 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
181 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
182 :
183 : // clean up the files that were created
184 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
185 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
186 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
187 1 : }
188 :
189 2 : BOOST_AUTO_TEST_CASE(WriteOneFile)
190 : {
191 1 : std::string file_path(std::filesystem::temp_directory_path());
192 :
193 1 : const int trigger_count = 5;
194 1 : const int apa_count = 3;
195 1 : const int link_count = 1;
196 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
197 :
198 : // delete any pre-existing files so that we start with a clean slate
199 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
200 1 : delete_files_matching_pattern(file_path, delete_pattern);
201 :
202 : // create the DataStore
203 1 : CfgFixture cfg("test-session-3-1");
204 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
205 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
206 :
207 1 : auto data_store_conf_obj = data_store_conf->config_object();
208 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
209 :
210 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
211 :
212 : // write several events, each with several fragments
213 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
214 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
215 :
216 1 : data_store_ptr.reset(); // explicit destruction
217 :
218 : // check that the expected number of files was created
219 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
220 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
221 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
222 :
223 : // clean up the files that were created
224 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
225 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
226 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
227 1 : }
228 :
229 2 : BOOST_AUTO_TEST_CASE(CheckWritingSuffix)
230 : {
231 1 : std::string file_path(std::filesystem::temp_directory_path());
232 :
233 1 : const int trigger_count = 5;
234 1 : const int apa_count = 3;
235 1 : const int link_count = 1;
236 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
237 :
238 : // delete any pre-existing files so that we start with a clean slate
239 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
240 1 : delete_files_matching_pattern(file_path, delete_pattern);
241 :
242 : // create the DataStore
243 1 : CfgFixture cfg("test-session-3-1");
244 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
245 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
246 :
247 1 : auto data_store_conf_obj = data_store_conf->config_object();
248 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
249 :
250 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
251 :
252 : // write several events, each with several fragments
253 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number) {
254 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
255 :
256 : // check that the .writing file is there
257 5 : std::string search_pattern = "hdf5writetest.*\\.writing";
258 5 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
259 5 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
260 5 : }
261 :
262 1 : data_store_ptr.reset(); // explicit destruction
263 :
264 : // check that the expected number of files was created
265 1 : std::string search_pattern = "hdf5writetest.*\\.writing";
266 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
267 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 0);
268 :
269 : // clean up the files that were created
270 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
271 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
272 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
273 1 : }
274 :
275 2 : BOOST_AUTO_TEST_CASE(NoDuplicateTimeSlices)
276 : {
277 1 : std::string file_path(std::filesystem::temp_directory_path());
278 :
279 1 : CfgFixture cfg("test-session-3-1");
280 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
281 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
282 :
283 1 : auto data_store_conf_obj = data_store_conf->config_object();
284 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
285 :
286 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
287 :
288 1 : dunedaq::daqdataformats::TimeSlice timeslice {999, 999}; // timeslice #, run #
289 1 : dunedaq::daqdataformats::TimeSlice identical_timeslice {999, 999};
290 :
291 1 : data_store_ptr->write(timeslice);
292 :
293 1 : std::string search_pattern = "hdf5writetest.*\\.writing";
294 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
295 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
296 :
297 1 : BOOST_CHECK_THROW(data_store_ptr->write(identical_timeslice), dunedaq::dfmodules::IgnorableDataStoreProblem);
298 :
299 1 : delete_files_matching_pattern(file_path, "hdf5writetest.*\\.hdf5");
300 1 : }
301 :
302 2 : BOOST_AUTO_TEST_CASE(WriteToBadOutputArea)
303 : {
304 1 : std::string file_path("/this/path/does/not/exist");
305 :
306 1 : CfgFixture cfg("test-session-3-1");
307 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
308 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
309 :
310 1 : auto data_store_conf_obj = data_store_conf->config_object();
311 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
312 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
313 :
314 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InvalidOutputPath);
315 1 : }
316 :
317 2 : BOOST_AUTO_TEST_CASE(EnormousMaxFileSize)
318 : {
319 1 : CfgFixture cfg("test-session-3-1");
320 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
321 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
322 :
323 1 : auto data_store_conf_obj = data_store_conf->config_object();
324 1 : data_store_conf_obj.set_by_val<size_t>("max_file_size", std::numeric_limits<size_t>::max());
325 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
326 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InsufficientDiskSpace);
327 1 : }
328 :
329 :
330 2 : BOOST_AUTO_TEST_CASE(FileSizeLimitResultsInMultipleFiles)
331 : {
332 1 : std::string file_path(std::filesystem::temp_directory_path());
333 :
334 1 : const int trigger_count = 15;
335 1 : const int apa_count = 5;
336 1 : const int link_count = 10;
337 1 : const int fragment_size = 10000;
338 :
339 : // 5 APAs times 10 links times 10000 bytes per fragment gives 500,000 bytes per TR
340 : // So, 15 TRs would give 7,500,000 bytes total.
341 :
342 : // delete any pre-existing files so that we start with a clean slate
343 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
344 1 : delete_files_matching_pattern(file_path, delete_pattern);
345 :
346 : // create the DataStore
347 1 : CfgFixture cfg("test-session-5-10");
348 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
349 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
350 :
351 1 : auto data_store_conf_obj = data_store_conf->config_object();
352 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
353 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 3000000); // goal is 6 events per file
354 :
355 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
356 :
357 : // write several events, each with several fragments
358 16 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
359 15 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
360 :
361 1 : data_store_ptr.reset(); // explicit destruction
362 :
363 : // check that the expected number of files was created
364 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
365 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
366 : // 7,500,000 bytes stored in files of size 3,000,000 should result in three files.
367 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
368 :
369 : // clean up the files that were created
370 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
371 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
372 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
373 1 : }
374 :
375 2 : BOOST_AUTO_TEST_CASE(SmallFileSizeLimitDataBlockListWrite)
376 : {
377 1 : std::string file_path(std::filesystem::temp_directory_path());
378 :
379 1 : const int trigger_count = 5;
380 1 : const int apa_count = 5;
381 1 : const int link_count = 1;
382 1 : const int fragment_size = 100000;
383 :
384 : // 5 APAs times 100000 bytes per fragment gives 500,000 bytes per TR
385 :
386 : // delete any pre-existing files so that we start with a clean slate
387 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
388 1 : delete_files_matching_pattern(file_path, delete_pattern);
389 :
390 : // create the DataStore
391 1 : CfgFixture cfg("test-session-5-1");
392 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
393 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
394 :
395 1 : auto data_store_conf_obj = data_store_conf->config_object();
396 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
397 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 150000); // ~1.5 Fragment, ~0.3 TR
398 :
399 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr,"dwm-01");
400 :
401 : // write several events, each with several fragments
402 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
403 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
404 :
405 1 : data_store_ptr.reset(); // explicit destruction
406 :
407 : // check that the expected number of files was created
408 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
409 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
410 : // each TriggerRecord should be stored in its own file
411 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
412 :
413 : // clean up the files that were created
414 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
415 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
416 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
417 1 : }
418 :
419 : BOOST_AUTO_TEST_SUITE_END()
|
