Line data Source code
1 : /**
2 : * @file SummaryTextWrite_test.cxx Application that ostensibly tests and
3 : * demonstrates the write functionality of the SummaryTextDataStore class,
4 : * but can be more properly thought of as a test of the base class
5 : * from which it inherits, FileDataStoreImpl
6 : *
7 : * This is heavily modeled on the HDF5Write unit tests, with
8 : * adjustments where necessary (max file size different in the
9 : * multiple-output-file test; no check for a ".writing" suffix for
10 : * files in the process of being written to)
11 : *
12 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
13 : * Licensing/copyright details are in the COPYING file that you should have
14 : * received with this code.
15 : */
16 :
17 : #include "dfmodules/DataStore.hpp"
18 : #include "dfmodules/DataStoreConfTestDeriv.hpp"
19 :
20 : // We need the actual SummaryTextDataStore.hpp plugin header so the unit tests can access its exceptions
21 : #include "../plugins/SummaryTextDataStore.hpp" // NOLINT(build/include_path)
22 :
23 : #include "appmodel/DataStoreConf.hpp"
24 : #include "appmodel/DataWriterConf.hpp"
25 : #include "appmodel/DataWriterModule.hpp"
26 : #include "appmodel/FilenameParams.hpp"
27 : #include "confmodel/DetectorConfig.hpp"
28 : #include "confmodel/Session.hpp"
29 : #include "detdataformats/DetID.hpp"
30 :
31 : #define BOOST_TEST_MODULE SummaryTextWrite_test // NOLINT
32 :
33 : #include "boost/test/unit_test.hpp"
34 :
35 : #include <cstdlib>
36 : #include <filesystem>
37 : #include <fstream>
38 : #include <iostream>
39 : #include <limits>
40 : #include <memory>
41 : #include <regex>
42 : #include <string>
43 : #include <utility>
44 : #include <vector>
45 :
46 : using namespace dunedaq::dfmodules;
47 :
48 : std::vector<std::string>
49 5 : get_files_matching_pattern(const std::string& path, const std::string& pattern)
50 : {
51 5 : std::regex regex_search_pattern(pattern);
52 5 : std::vector<std::string> file_list;
53 2087 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
54 2082 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
55 27 : file_list.push_back(entry.path());
56 : }
57 5 : }
58 5 : return file_list;
59 5 : }
60 :
61 : std::vector<std::string>
62 14 : delete_files_matching_pattern(const std::string& path, const std::string& pattern)
63 : {
64 14 : std::regex regex_search_pattern(pattern);
65 14 : std::vector<std::string> file_list;
66 5795 : for (const auto& entry : std::filesystem::directory_iterator(path)) {
67 5781 : if (std::regex_match(entry.path().filename().string(), regex_search_pattern)) {
68 27 : if (std::filesystem::remove(entry.path())) {
69 27 : file_list.push_back(entry.path());
70 : }
71 : }
72 14 : }
73 14 : return file_list;
74 14 : }
75 :
76 : dunedaq::daqdataformats::TriggerRecord
77 30 : create_trigger_record(int trig_num, int fragment_size, int element_count)
78 : {
79 30 : const int run_number = 53;
80 30 : const dunedaq::daqdataformats::SourceID::Subsystem stype_to_use =
81 : dunedaq::daqdataformats::SourceID::Subsystem::kDetectorReadout;
82 :
83 : // setup our dummy_data
84 30 : std::vector<char> dummy_vector(fragment_size);
85 30 : char* dummy_data = dummy_vector.data();
86 :
87 : // get a timestamp for this trigger
88 60 : uint64_t ts = std::chrono::duration_cast<std::chrono::milliseconds>( // NOLINT(build/unsigned)
89 30 : system_clock::now().time_since_epoch())
90 30 : .count();
91 :
92 : // create TriggerRecordHeader
93 30 : dunedaq::daqdataformats::TriggerRecordHeaderData trh_data;
94 30 : trh_data.trigger_number = trig_num;
95 30 : trh_data.trigger_timestamp = ts;
96 30 : trh_data.num_requested_components = element_count;
97 30 : trh_data.run_number = run_number;
98 30 : trh_data.sequence_number = 0;
99 30 : trh_data.max_sequence_number = 1;
100 30 : trh_data.element_id = dunedaq::daqdataformats::SourceID(dunedaq::daqdataformats::SourceID::Subsystem::kTRBuilder, 0);
101 :
102 30 : dunedaq::daqdataformats::TriggerRecordHeader trh(&trh_data);
103 :
104 : // create out TriggerRecord
105 30 : dunedaq::daqdataformats::TriggerRecord tr(trh);
106 :
107 : // loop over elements
108 160 : for (int ele_num = 0; ele_num < element_count; ++ele_num) {
109 :
110 : // create our fragment
111 130 : dunedaq::daqdataformats::FragmentHeader fh;
112 130 : fh.trigger_number = trig_num;
113 130 : fh.trigger_timestamp = ts;
114 130 : fh.window_begin = ts - 10;
115 130 : fh.window_end = ts;
116 130 : fh.run_number = run_number;
117 130 : fh.sequence_number = 0;
118 130 : fh.fragment_type =
119 : static_cast<dunedaq::daqdataformats::fragment_type_t>(dunedaq::daqdataformats::FragmentType::kWIB);
120 130 : fh.detector_id =
121 : static_cast<uint16_t>(dunedaq::detdataformats::DetID::Subdetector::kHD_TPC); // NOLINT(build/unsigned)
122 130 : fh.element_id = dunedaq::daqdataformats::SourceID(stype_to_use, ele_num);
123 130 : std::unique_ptr<dunedaq::daqdataformats::Fragment> frag_ptr(
124 130 : new dunedaq::daqdataformats::Fragment(dummy_data, fragment_size));
125 130 : frag_ptr->set_header_fields(fh);
126 :
127 : // add fragment to TriggerRecord
128 130 : tr.add_fragment(std::move(frag_ptr));
129 :
130 130 : } // end loop over elements
131 :
132 30 : return tr;
133 30 : }
134 :
135 : struct CfgFixture
136 : {
137 8 : explicit CfgFixture(std::string sessionName)
138 8 : {
139 8 : TLOG_DEBUG(4) << "Creating CfgFixture";
140 8 : setenv("DUNEDAQ_SESSION", sessionName.c_str(), 1);
141 8 : std::string oksConfig = "oksconflibs:test/config/summarytextwrite_test.data.xml";
142 8 : std::string appName = "TestApp";
143 8 : cfgMgr = std::make_shared<dunedaq::appfwk::ConfigurationManager>(oksConfig, appName, sessionName);
144 8 : TLOG_DEBUG(4) << "Done with CfgFixture";
145 8 : }
146 :
147 : const dunedaq::confmodel::DetectorConfig* get_detector_config()
148 : {
149 : return cfgMgr->get_session()->get_detector_configuration();
150 : }
151 :
152 : std::shared_ptr<dunedaq::appfwk::ConfigurationManager> cfgMgr;
153 : };
154 :
155 : BOOST_AUTO_TEST_SUITE(SummaryTextWrite_test)
156 :
157 2 : BOOST_AUTO_TEST_CASE(NullConfiguration)
158 : {
159 5 : BOOST_CHECK_THROW(make_data_store("SummaryTextDataStore", "dummy", nullptr, "dummy"),
160 : dunedaq::dfmodules::FileDataStoreImplBadConfiguration);
161 1 : }
162 :
163 2 : BOOST_AUTO_TEST_CASE(CheckDerivation)
164 : {
165 1 : CfgFixture cfg("test-session-3-1");
166 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
167 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
168 :
169 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
170 1 : BOOST_REQUIRE(data_store_ptr != nullptr);
171 :
172 1 : const auto stub_data_store_ptr{ dynamic_cast<const dunedaq::dfmodules::SummaryTextDataStore*>(
173 1 : data_store_ptr.get()) }; // NOLINT(runtime/rtti)
174 1 : BOOST_REQUIRE(stub_data_store_ptr != nullptr);
175 :
176 1 : BOOST_REQUIRE_EQUAL(stub_data_store_ptr->get_derivval(), 773); // 773 should be the value in the config
177 1 : }
178 :
179 2 : BOOST_AUTO_TEST_CASE(WriteEventFiles)
180 : {
181 1 : std::string file_path(std::filesystem::temp_directory_path());
182 :
183 1 : const int trigger_count = 5;
184 1 : const int apa_count = 3;
185 1 : const int link_count = 1;
186 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
187 :
188 : // delete any pre-existing files so that we start with a clean slate
189 1 : std::string delete_pattern = "stubwritetest.*\\.txt";
190 1 : delete_files_matching_pattern(file_path, delete_pattern);
191 :
192 : // create the DataStore
193 1 : CfgFixture cfg("test-session-3-1");
194 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
195 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
196 :
197 1 : auto data_store_conf_obj = data_store_conf->config_object();
198 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
199 1 : data_store_conf_obj.set_by_val<std::string>("mode", "one-event-per-file");
200 :
201 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
202 :
203 : // write several events, each with several fragments
204 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
205 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, link_count * apa_count));
206 :
207 1 : data_store_ptr.reset();
208 :
209 : // check that the expected number of files was created
210 1 : std::string search_pattern = "stubwritetest.*\\.txt";
211 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
212 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
213 :
214 : // clean up the files that were created
215 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
216 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
217 1 : BOOST_REQUIRE_EQUAL(file_list.size(), trigger_count);
218 1 : }
219 :
220 2 : BOOST_AUTO_TEST_CASE(WriteOneFile)
221 : {
222 1 : std::string file_path(std::filesystem::temp_directory_path());
223 :
224 1 : const int trigger_count = 5;
225 1 : const int apa_count = 3;
226 1 : const int link_count = 1;
227 1 : const int fragment_size = 10 + sizeof(dunedaq::daqdataformats::FragmentHeader);
228 :
229 : // delete any pre-existing files so that we start with a clean slate
230 1 : std::string delete_pattern = "stubwritetest.*\\.txt";
231 1 : delete_files_matching_pattern(file_path, delete_pattern);
232 :
233 : // create the DataStore
234 1 : CfgFixture cfg("test-session-3-1");
235 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
236 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
237 :
238 1 : auto data_store_conf_obj = data_store_conf->config_object();
239 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
240 :
241 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
242 :
243 : // write several events, each with several fragments
244 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
245 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
246 :
247 1 : data_store_ptr.reset();
248 :
249 : // check that the expected number of files was created
250 1 : std::string search_pattern = "stubwritetest.*\\.txt";
251 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
252 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
253 :
254 : // clean up the files that were created
255 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
256 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
257 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
258 1 : }
259 :
260 2 : BOOST_AUTO_TEST_CASE(NoDuplicateTimeSlices)
261 : {
262 1 : std::string file_path(std::filesystem::temp_directory_path());
263 :
264 : // delete any pre-existing files so that we start with a clean slate
265 1 : std::string delete_pattern = "stubwritetest.*\\.txt";
266 1 : delete_files_matching_pattern(file_path, delete_pattern);
267 :
268 1 : CfgFixture cfg("test-session-3-1");
269 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
270 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
271 :
272 1 : auto data_store_conf_obj = data_store_conf->config_object();
273 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
274 :
275 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
276 :
277 1 : dunedaq::daqdataformats::TimeSlice timeslice{ 999, 999 }; // timeslice #, run #
278 1 : dunedaq::daqdataformats::TimeSlice identical_timeslice{ 999, 999 };
279 :
280 1 : data_store_ptr->write(timeslice);
281 :
282 1 : std::string search_pattern = "stubwritetest.*\\.txt";
283 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
284 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 1);
285 :
286 1 : BOOST_CHECK_THROW(data_store_ptr->write(identical_timeslice), dunedaq::dfmodules::IgnorableDataStoreProblem);
287 :
288 1 : delete_files_matching_pattern(file_path, delete_pattern);
289 1 : }
290 :
291 2 : BOOST_AUTO_TEST_CASE(WriteToBadOutputArea)
292 : {
293 1 : std::string file_path("/this/path/does/not/exist");
294 :
295 1 : CfgFixture cfg("test-session-3-1");
296 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
297 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
298 :
299 1 : auto data_store_conf_obj = data_store_conf->config_object();
300 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
301 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
302 :
303 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InvalidOutputPath);
304 1 : }
305 :
306 2 : BOOST_AUTO_TEST_CASE(EnormousMaxFileSize)
307 : {
308 1 : CfgFixture cfg("test-session-3-1");
309 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
310 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
311 :
312 1 : auto data_store_conf_obj = data_store_conf->config_object();
313 1 : data_store_conf_obj.set_by_val<size_t>("max_file_size", std::numeric_limits<size_t>::max());
314 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
315 1 : BOOST_CHECK_THROW(data_store_ptr->prepare_for_run(1, true), dunedaq::dfmodules::InsufficientDiskSpace);
316 1 : }
317 :
318 2 : BOOST_AUTO_TEST_CASE(FileSizeLimitResultsInMultipleFiles)
319 : {
320 1 : std::string file_path(std::filesystem::temp_directory_path());
321 :
322 1 : const int trigger_count = 15;
323 1 : const int apa_count = 5;
324 1 : const int link_count = 1;
325 1 : const int fragment_size = 10000;
326 :
327 : // delete any pre-existing files so that we start with a clean slate
328 1 : std::string delete_pattern = "stubwritetest.*\\.txt";
329 1 : delete_files_matching_pattern(file_path, delete_pattern);
330 :
331 : // create the DataStore
332 1 : CfgFixture cfg("test-session-5-10");
333 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
334 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
335 :
336 1 : auto data_store_conf_obj = data_store_conf->config_object();
337 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
338 :
339 : // Tiny max file size needed since each trigger record just gets a simple line of text written out
340 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 100);
341 :
342 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
343 :
344 : // write several events, each with several fragments
345 16 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
346 15 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
347 :
348 1 : data_store_ptr.reset();
349 :
350 : // check that the expected number of files was created
351 1 : std::string search_pattern = "stubwritetest.*\\.txt";
352 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
353 : // With very small max_file_size (100 bytes) and minimal SUMMARYTEXT output, we should have multiple files
354 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 15);
355 :
356 : // clean up the files that were created
357 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
358 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
359 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 15);
360 1 : }
361 :
362 2 : BOOST_AUTO_TEST_CASE(SmallFileSizeLimitDataBlockListWrite)
363 : {
364 1 : std::string file_path(std::filesystem::temp_directory_path());
365 :
366 1 : const int trigger_count = 5;
367 1 : const int apa_count = 5;
368 1 : const int link_count = 1;
369 1 : const int fragment_size = 100000;
370 :
371 : // 5 APAs times 100000 bytes per fragment gives 500,000 bytes per TR
372 :
373 : // delete any pre-existing files so that we start with a clean slate
374 1 : std::string delete_pattern = "stubwritetest.*\\.txt";
375 1 : delete_files_matching_pattern(file_path, delete_pattern);
376 :
377 : // create the DataStore
378 1 : CfgFixture cfg("test-session-5-1");
379 1 : auto data_writer_conf = cfg.cfgMgr->get_dal<dunedaq::appmodel::DataWriterModule>("dwm-01")->get_configuration();
380 1 : auto data_store_conf = data_writer_conf->get_data_store_params();
381 :
382 1 : auto data_store_conf_obj = data_store_conf->config_object();
383 1 : data_store_conf_obj.set_by_val<std::string>("directory_path", file_path);
384 1 : data_store_conf_obj.set_by_val<int>("max_file_size", 150000); // ~1.5 Fragment, ~0.3 TR
385 :
386 1 : auto data_store_ptr = make_data_store(data_store_conf->get_type(), data_store_conf->UID(), cfg.cfgMgr, "dwm-01");
387 :
388 : // write several events, each with several fragments
389 6 : for (int trigger_number = 1; trigger_number <= trigger_count; ++trigger_number)
390 5 : data_store_ptr->write(create_trigger_record(trigger_number, fragment_size, apa_count * link_count));
391 :
392 1 : data_store_ptr.reset();
393 :
394 : // check that the expected number of files was created
395 1 : std::string search_pattern = "stubwritetest.*\\.txt";
396 1 : std::vector<std::string> file_list = get_files_matching_pattern(file_path, search_pattern);
397 : // each TriggerRecord should be stored in its own file
398 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
399 :
400 : // clean up the files that were created
401 1 : file_list = delete_files_matching_pattern(file_path, delete_pattern);
402 1 : delete_files_matching_pattern(file_path, "HardwareMap.*\\.txt");
403 1 : BOOST_REQUIRE_EQUAL(file_list.size(), 5);
404 1 : }
405 :
406 : BOOST_AUTO_TEST_SUITE_END()
|
