Line data Source code
1 : /**
2 : * @file HDF5Write_test.cxx Application that tests and demonstrates
3 : * the write functionality of the HDF5DataStore class.
4 : *
5 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
6 : * Licensing/copyright details are in the COPYING file that you should have
7 : * received with this code.
8 : */
9 :
10 : #include "dfmodules/DataStore.hpp"
11 :
12 : #include "appmodel/DataWriterModule.hpp"
13 : #include "appmodel/DataWriterConf.hpp"
14 : #include "appmodel/FilenameParams.hpp"
15 : #include "confmodel/DetectorConfig.hpp"
16 : #include "confmodel/Session.hpp"
17 : #include "appmodel/DataStoreConf.hpp"
18 : #include "detdataformats/DetID.hpp"
19 :
20 : #define BOOST_TEST_MODULE HDF5Write_test // NOLINT
21 :
22 : #include "boost/test/unit_test.hpp"
23 :
24 : #include <cstdlib>
25 : #include <filesystem>
26 : #include <fstream>
27 : #include <iostream>
28 : #include <memory>
29 : #include <regex>
30 : #include <string>
31 : #include <utility>
32 : #include <vector>
33 :
34 : using namespace dunedaq::dfmodules;
35 :
36 : std::vector<std::string>
37 10 : get_files_matching_pattern(const std::string& path, const std::string& pattern)
38 : {
39 10 : std::regex regex_search_pattern(pattern);
40 10 : std::vector<std::string> file_list;
41 162710 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
42 162700 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
43 19 : file_list.push_back(entry.path());
44 : }
45 10 : }
46 10 : return file_list;
47 10 : }
48 :
49 : std::vector<std::string>
50 15 : delete_files_matching_pattern(const std::string& path, const std::string& pattern)
51 : {
52 15 : std::regex regex_search_pattern(pattern);
53 15 : std::vector<std::string> file_list;
54 244050 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
55 244035 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
56 15 : if (std::filesystem::remove(entry.path())) {
57 15 : file_list.push_back(entry.path());
58 : }
59 : }
60 15 : }
61 15 : return file_list;
62 15 : }
63 :
64 : dunedaq::daqdataformats::TriggerRecord
65 35 : create_trigger_record(int trig_num, int fragment_size, int element_count)
66 : {
67 35 : const int run_number = 53;
68 35 : const dunedaq::daqdataformats::SourceID::Subsystem stype_to_use =
69 : dunedaq::daqdataformats::SourceID::Subsystem::kDetectorReadout;
70 :
71 : // setup our dummy_data
72 35 : std::vector<char> dummy_vector(fragment_size);
73 35 : char* dummy_data = dummy_vector.data();
74 :
75 : // get a timestamp for this trigger
76 70 : uint64_t ts = std::chrono::duration_cast<std::chrono::milliseconds>( // NOLINT(build/unsigned)
77 35 : system_clock::now().time_since_epoch())
78 35 : .count();
79 :
80 : // create TriggerRecordHeader
81 35 : dunedaq::daqdataformats::TriggerRecordHeaderData trh_data;
82 35 : trh_data.trigger_number = trig_num;
83 35 : trh_data.trigger_timestamp = ts;
84 35 : trh_data.num_requested_components = element_count;
85 35 : trh_data.run_number = run_number;
86 35 : trh_data.sequence_number = 0;
87 35 : trh_data.max_sequence_number = 1;
88 35 : trh_data.element_id = dunedaq::daqdataformats::SourceID(dunedaq::daqdataformats::SourceID::Subsystem::kTRBuilder, 0);
89 :
90 35 : dunedaq::daqdataformats::TriggerRecordHeader trh(&trh_data);
91 :
92 : // create out TriggerRecord
93 35 : dunedaq::daqdataformats::TriggerRecord tr(trh);
94 :
95 : // loop over elements
96 855 : for (int ele_num = 0; ele_num < element_count; ++ele_num) {
97 :
98 : // create our fragment
99 820 : dunedaq::daqdataformats::FragmentHeader fh;
100 820 : fh.trigger_number = trig_num;
101 820 : fh.trigger_timestamp = ts;
102 820 : fh.window_begin = ts - 10;
103 820 : fh.window_end = ts;
104 820 : fh.run_number = run_number;
105 820 : fh.sequence_number = 0;
106 820 : fh.fragment_type =
107 : static_cast<dunedaq::daqdataformats::fragment_type_t>(dunedaq::daqdataformats::FragmentType::kWIB);
108 820 : fh.detector_id = static_cast<uint16_t>(dunedaq::detdataformats::DetID::Subdetector::kHD_TPC);
109 820 : fh.element_id = dunedaq::daqdataformats::SourceID(stype_to_use, ele_num);
110 820 : std::unique_ptr<dunedaq::daqdataformats::Fragment> frag_ptr(
111 820 : new dunedaq::daqdataformats::Fragment(dummy_data, fragment_size));
112 820 : frag_ptr->set_header_fields(fh);
113 :
114 : // add fragment to TriggerRecord
115 820 : tr.add_fragment(std::move(frag_ptr));
116 :
117 820 : } // end loop over elements
118 :
119 35 : return tr;
120 35 : }
121 :
122 : struct CfgFixture
123 : {
124 5 : CfgFixture(std::string sessionName)
125 5 : {
126 5 : TLOG_DEBUG(4) << "Creating CfgFixture";
127 5 : setenv("DUNEDAQ_SESSION", sessionName.c_str(), 1);
128 5 : std::string oksConfig = "oksconflibs:test/config/hdf5write_test.data.xml";
129 5 : std::string appName = "TestApp";
130 5 : cfgMgr = std::make_shared<dunedaq::appfwk::ConfigurationManager>(oksConfig, appName, sessionName);
131 5 : TLOG_DEBUG(4) << "Done with CfgFixture";
132 5 : }
133 :
134 : const dunedaq::confmodel::DetectorConfig* get_detector_config()
135 : {
136 : return cfgMgr->get_session()->get_detector_configuration();
137 : }
138 :
139 : std::shared_ptr<dunedaq::appfwk::ConfigurationManager> cfgMgr;
140 : };
141 :
142 : BOOST_AUTO_TEST_SUITE(HDF5Write_test)
143 :
144 2 : BOOST_AUTO_TEST_CASE(WriteEventFiles)
145 : {
146 1 : std::string file_path(std::filesystem::temp_directory_path());
147 :
148 1 : const int trigger_count = 5;
149 1 : const int apa_count = 3;
150 1 : const int link_count = 1;
151 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
152 :
153 : // delete any pre-existing files so that we start with a clean slate
154 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
155 1 : delete_files_matching_pattern(file_path, delete_pattern);
156 :
157 : // create the DataStore
158 1 : CfgFixture cfg("test-session-3-1");
159 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
160 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
161 :
162 1 : auto data_store_conf_obj = data_store_conf->config_object();
163 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
164 1 : data_store_conf_obj.set_by_val<std::string>("mode", "one-event-per-file");
165 :
166 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
167 :
168 : // write several events, each with several fragments
169 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
170 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, link_count * apa_count));
171 :
172 1 : data_store_ptr.reset(); // explicit destruction
173 :
174 : // check that the expected number of files was created
175 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
176 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
177 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
178 :
179 : // clean up the files that were created
180 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
181 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
182 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
183 1 : }
184 :
185 2 : BOOST_AUTO_TEST_CASE(WriteOneFile)
186 : {
187 1 : std::string file_path(std::filesystem::temp_directory_path());
188 :
189 1 : const int trigger_count = 5;
190 1 : const int apa_count = 3;
191 1 : const int link_count = 1;
192 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
193 :
194 : // delete any pre-existing files so that we start with a clean slate
195 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
196 1 : delete_files_matching_pattern(file_path, delete_pattern);
197 :
198 : // create the DataStore
199 1 : CfgFixture cfg("test-session-3-1");
200 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
201 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
202 :
203 1 : auto data_store_conf_obj = data_store_conf->config_object();
204 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
205 :
206 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
207 :
208 : // write several events, each with several fragments
209 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
210 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
211 :
212 1 : data_store_ptr.reset(); // explicit destruction
213 :
214 : // check that the expected number of files was created
215 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
216 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
217 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
218 :
219 : // clean up the files that were created
220 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
221 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
222 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
223 1 : }
224 :
225 2 : BOOST_AUTO_TEST_CASE(CheckWritingSuffix)
226 : {
227 1 : std::string file_path(std::filesystem::temp_directory_path());
228 :
229 1 : const int trigger_count = 5;
230 1 : const int apa_count = 3;
231 1 : const int link_count = 1;
232 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
233 :
234 : // delete any pre-existing files so that we start with a clean slate
235 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
236 1 : delete_files_matching_pattern(file_path, delete_pattern);
237 :
238 : // create the DataStore
239 1 : CfgFixture cfg("test-session-3-1");
240 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
241 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
242 :
243 1 : auto data_store_conf_obj = data_store_conf->config_object();
244 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
245 :
246 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
247 :
248 : // write several events, each with several fragments
249 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number) {
250 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
251 :
252 : // check that the .writing file is there
253 5 : std::string search_pattern = "hdf5writetest.*\\.writing";
254 5 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
255 5 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
256 5 : }
257 :
258 1 : data_store_ptr.reset(); // explicit destruction
259 :
260 : // check that the expected number of files was created
261 1 : std::string search_pattern = "hdf5writetest.*\\.writing";
262 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
263 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 0);
264 :
265 : // clean up the files that were created
266 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
267 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
268 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
269 1 : }
270 :
271 2 : BOOST_AUTO_TEST_CASE(FileSizeLimitResultsInMultipleFiles)
272 : {
273 1 : std::string file_path(std::filesystem::temp_directory_path());
274 :
275 1 : const int trigger_count = 15;
276 1 : const int apa_count = 5;
277 1 : const int link_count = 10;
278 1 : const int fragment_size = 10000;
279 :
280 : // 5 APAs times 10 links times 10000 bytes per fragment gives 500,000 bytes per TR
281 : // So, 15 TRs would give 7,500,000 bytes total.
282 :
283 : // delete any pre-existing files so that we start with a clean slate
284 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
285 1 : delete_files_matching_pattern(file_path, delete_pattern);
286 :
287 : // create the DataStore
288 1 : CfgFixture cfg("test-session-5-10");
289 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
290 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
291 :
292 1 : auto data_store_conf_obj = data_store_conf->config_object();
293 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
294 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 3000000); // goal is 6 events per file
295 :
296 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
297 :
298 : // write several events, each with several fragments
299 16 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
300 15 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
301 :
302 1 : data_store_ptr.reset(); // explicit destruction
303 :
304 : // check that the expected number of files was created
305 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
306 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
307 : // 7,500,000 bytes stored in files of size 3,000,000 should result in three files.
308 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
309 :
310 : // clean up the files that were created
311 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
312 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
313 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 3);
314 1 : }
315 :
316 2 : BOOST_AUTO_TEST_CASE(SmallFileSizeLimitDataBlockListWrite)
317 : {
318 1 : std::string file_path(std::filesystem::temp_directory_path());
319 :
320 1 : const int trigger_count = 5;
321 1 : const int apa_count = 5;
322 1 : const int link_count = 1;
323 1 : const int fragment_size = 100000;
324 :
325 : // 5 APAs times 100000 bytes per fragment gives 500,000 bytes per TR
326 :
327 : // delete any pre-existing files so that we start with a clean slate
328 1 : std::string delete_pattern = "hdf5writetest.*\\.hdf5";
329 1 : delete_files_matching_pattern(file_path, delete_pattern);
330 :
331 : // create the DataStore
332 1 : CfgFixture cfg("test-session-5-1");
333 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
334 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
335 :
336 1 : auto data_store_conf_obj = data_store_conf->config_object();
337 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
338 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 150000); // ~1.5 Fragment, ~0.3 TR
339 :
340 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr,"dwm-01");
341 :
342 : // write several events, each with several fragments
343 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
344 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
345 :
346 1 : data_store_ptr.reset(); // explicit destruction
347 :
348 : // check that the expected number of files was created
349 1 : std::string search_pattern = "hdf5writetest.*\\.hdf5";
350 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
351 : // each TriggerRecord should be stored in its own file
352 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
353 :
354 : // clean up the files that were created
355 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
356 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
357 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
358 1 : }
359 :
360 : BOOST_AUTO_TEST_SUITE_END()
|
