file_quality_checker Namespace Reference
Functions | |
| main (filenames) | |
Function Documentation
◆ main()
| main | ( | filenames | ) |
This script provides a high-level summary of the records in an output HDF5 file and the fragments which they contain. It simply takes a filename, or list of filenames, as argument(s) and summarizes each one sequentially. For info on how to interpret the output, look at rawdatautils documentation: https://dune-daq-sw.readthedocs.io/en/latest/packages/rawdatautils/
Definition at line 33 of file file_quality_checker.py.
34 """
35This script provides a high-level summary of the records in an output HDF5 file and the fragments which they contain.
36
37It simply takes a filename, or list of filenames, as argument(s) and summarizes each one sequentially.
38
39For info on how to interpret the output, look at rawdatautils documentation:
40https://dune-daq-sw.readthedocs.io/en/latest/packages/rawdatautils/
41
42"""
43
44 for filename in filenames:
45
46 if not os.path.exists(filename):
47 sys.exit(f"ERROR: file \"{filename}\" doesn't appear to exist")
48
49 try:
50 h5file = HDF5RawDataFile(filename)
51 except:
52 print(traceback.format_exc())
53 sys.exit(f"ERROR: file \"{filename}\" couldn't be opened; is it an HDF5 file?")
54
55 print(f"Processing {filename}...")
56
57 is_trigger_records = True
58
59 try:
60 records = h5file.get_all_trigger_record_ids()
61 except RuntimeError:
62 is_trigger_records = False
63 except:
64 print(traceback.format_exc())
65 sys.exit("ERROR: Something went wrong when calling h5file.get_all_trigger_record_ids(); file may contain junk data or be corrupted. Exiting...\n")
66
67 if not is_trigger_records:
68 try:
69 records = h5file.get_all_timeslice_ids()
70 except RuntimeError:
71 sys.exit(f"Neither get_all_trigger_record_ids() nor get_all_timeslice_ids() returned records. Exiting...\n")
72 except:
73 print(traceback.format_exc())
74 sys.exit("ERROR: Something went wrong when calling h5file.get_all_timeslice_ids(); file may contain junk data or be corrupted. Exiting...\n")
75
76 if is_trigger_records:
77 print(f'Will process {len(records)} trigger records.')
78 else:
79 print(f'Will process {len(records)} timeslice records.')
80
81 sequence_ids = []
82 record_ids = []
83 tr_global_stats = {}
84
85 assumed_sequence_id_step = -1
86 assumed_record_id_step = -1
87 first_sequence_id = -1
88 first_record_id = -1
89
90 for i_r, r in enumerate(records):
91
92 for i_quadrant in range(1,4):
93 if i_r == i_quadrant * int(len(records)/4):
94 print(f"Processed {i_r} of {len(records)} records...")
95
96 record_ids.append(r[0])
97 sequence_ids.append(r[1])
98
99 if i_r == 0:
100 first_record_id = r[0]
101 first_sequence_id = r[1]
102 elif i_r == 1:
103 assumed_record_id_step = r[0] - first_record_id
104 assumed_sequence_id_step = r[1] - first_sequence_id
105
106 dset_paths = h5file.get_fragment_dataset_paths(r)
107 tr_stats = {}
108
109 for dset_path in dset_paths:
110 frag = h5file.get_frag(dset_path)
111 if frag.get_fragment_type() in tr_stats:
112 tr_stats[ frag.get_fragment_type()]["count" ] += 1
113 if frag.get_size() > tr_stats[ frag.get_fragment_type() ]["max_size" ]:
114 tr_stats[ frag.get_fragment_type() ][ "max_size" ] = frag.get_size()
115 if frag.get_size() < tr_stats[ frag.get_fragment_type()]["min_size" ]:
116 tr_stats[ frag.get_fragment_type()]["min_size" ] = frag.get_size()
117 else:
118 tr_stats[ frag.get_fragment_type() ] = { "count": 1, "nonzero_status_bits_count": 0, "max_size": frag.get_size(), "min_size": frag.get_size() }
119
120 if frag.get_header().status_bits != 0:
121 tr_stats[ frag.get_fragment_type() ]["nonzero_status_bits_count"] += 1
122
123
124 for frag_type in tr_stats:
125 if frag_type in tr_global_stats:
126 if tr_stats[frag_type]["count"] > tr_global_stats[frag_type]["max_count"]:
127 tr_global_stats[frag_type]["max_count"] = tr_stats[frag_type]["count"]
128 if tr_stats[frag_type]["count"] < tr_global_stats[frag_type]["min_count"]:
129 tr_global_stats[frag_type]["min_count"] = tr_stats[frag_type]["count"]
130 if tr_stats[frag_type]["nonzero_status_bits_count"] > tr_global_stats[frag_type]["nonzero_status_bits_max_count"]:
131 tr_global_stats[frag_type]["nonzero_status_bits_max_count"] = tr_stats[frag_type]["nonzero_status_bits_count"]
132 if tr_stats[frag_type]["nonzero_status_bits_count"] < tr_global_stats[frag_type]["nonzero_status_bits_min_count"]:
133 tr_global_stats[frag_type]["nonzero_status_bits_min_count"] = tr_stats[frag_type]["nonzero_status_bits_count"]
134
135 if tr_stats[frag_type]["max_size"] > tr_global_stats[frag_type]["max_size"]:
136 tr_global_stats[frag_type]["max_size"] = tr_stats[frag_type]["max_size"]
137 if tr_stats[frag_type]["min_size"] < tr_global_stats[frag_type]["min_size"]:
138 tr_global_stats[frag_type]["min_size"] = tr_stats[frag_type]["min_size"]
139
140 else:
141 tr_global_stats[frag_type] = { "max_count": tr_stats[frag_type]["count"],
142 "min_count": tr_stats[frag_type]["count"],
143 "max_size": tr_stats[frag_type]["max_size"],
144 "min_size": tr_stats[frag_type]["min_size"],
145 "nonzero_status_bits_min_count": tr_stats[frag_type]["nonzero_status_bits_count"],
146 "nonzero_status_bits_max_count": tr_stats[frag_type]["nonzero_status_bits_count"]
147 }
148
149 print(f"Processed {len(records)} of {len(records)} records...")
150 print("")
151
152 sequence_ids_ok = True
153 for i_s, seqid in enumerate(sequence_ids):
154 if i_s > 0:
155 if seqid - sequence_ids[i_s - 1] != assumed_sequence_id_step:
156 sequence_ids_ok = False
157
158 record_ids_ok = True
159 for i_r, recid in enumerate(record_ids):
160 if i_r > 0:
161 if recid - record_ids[i_r - 1] != assumed_record_id_step:
162 record_ids_ok = False
163
164 if record_ids_ok:
165 print(f"Progression of record ids over records looks ok (expected change of {assumed_record_id_step} for each new record)")
166 else:
167 print("Non-constant progression of record ids over records. This may or may not be a problem.")
168 print(" ".join([str(recid) for recid in record_ids]))
169
170 if sequence_ids_ok:
171 print(f"Progression of sequence ids over records looks ok (expected change of {assumed_sequence_id_step} for each new record)")
172 else:
173 print("Non-contant progression of sequence ids over records. This may or may not be a problem.")
174 print(" ".join([str(seqid) for seqid in sequence_ids]))
175
176 print("")
177
178
179 frag_type_phrase_length = max([len(str(fragname)) - len("FragmentType.") for fragname in tr_global_stats]) + 1
180 max_count_phrase = "max # in rec"
181 min_count_phrase = "min # in rec"
182 max_size_phrase = "largest (B)"
183 min_size_phrase = "smallest (B)"
184 max_errs_phrase = "max err in rec"
185 min_errs_phrase = "min err in rec"
186
187 fmtstring=f"%-{frag_type_phrase_length}s|%s|%s|%s|%s|%s|%s|"
188 print(fmtstring % (" FragType ", min_count_phrase, max_count_phrase, min_size_phrase, max_size_phrase, min_errs_phrase, max_errs_phrase))
189
190 divider = "-" * (frag_type_phrase_length + len(min_count_phrase) + len(max_count_phrase) +
191 len(min_size_phrase) + len(max_size_phrase) + len(max_errs_phrase) + len(min_errs_phrase) + 7)
192 print(divider)
193
194 for frag_type in tr_global_stats:
195 fmtstring = f"%-{frag_type_phrase_length}s|%-{len(min_count_phrase)}s|%-{len(max_count_phrase)}s|%-{len(min_size_phrase)}s|%-{len(max_size_phrase)}s|%-{len(min_errs_phrase)}s|%-{len(max_errs_phrase)}s|"
196 print(fmtstring % (str(frag_type).replace("FragmentType.",""),
197 tr_global_stats[frag_type]["min_count"],
198 tr_global_stats[frag_type]["max_count"],
199 tr_global_stats[frag_type]["min_size"],
200 tr_global_stats[frag_type]["max_size"],
201 tr_global_stats[frag_type]["nonzero_status_bits_min_count"],
202 tr_global_stats[frag_type]["nonzero_status_bits_max_count"])
203 )
204
205 print(divider)
206 print("")
207
208 del h5file
209 gc.collect()
210
Generated on for DUNE-DAQ by
1.17.0