generate Namespace Reference
Functions | |
| generate_dataflow (oksfile, include, n_dfapps, tpwriting_enabled, generate_segment, n_data_writers=1, trmon_app=False) | |
| generate_hsi (oksfile, include, generate_segment) | |
| generate_readout (readoutmap, oksfile, include, generate_segment, emulated_file_name, tpg_enabled=True, hosts_to_use=[]) | |
| generate_fakedata (oksfile, include, generate_segment, n_streams, n_apps, det_id, fragment_type=None) | |
| generate_trigger (oksfile, include, generate_segment, tpg_enabled=True, hsi_enabled=False) | |
| generate_session (oksfile, include, session_name, op_env, connectivity_service_is_infrastructure_app=True, disable_connectivity_service=False) | |
Function Documentation
◆ generate_dataflow()
| generate_dataflow | ( | oksfile, | |
| include, | |||
| n_dfapps, | |||
| tpwriting_enabled, | |||
| generate_segment, | |||
| n_data_writers = 1, | |||
| trmon_app = False ) |
Simple script to create an OKS configuration file for a dataflow segment. The file will automatically include the relevant schema files and any other OKS files you specify.
Definition at line 10 of file generate.py.
18):
19 """Simple script to create an OKS configuration file for a dataflow segment.
20
21 The file will automatically include the relevant schema files and
22 any other OKS files you specify.
23 """
24
25 includefiles = [
26 "schema/confmodel/dunedaq.schema.xml",
27 "schema/appmodel/application.schema.xml",
28 ]
29
30 res, extra_includes = find_oksincludes(include, os.path.dirname(oksfile))
31 if res:
32 includefiles += extra_includes
33 else:
34 return
35
38 if not oksfile.endswith(".data.xml"):
39 oksfile = oksfile + ".data.xml"
40 print(f"Creating OKS database file {oksfile}")
41 db.create_db(oksfile, includefiles)
42 db.set_active(oksfile)
43
44 hosts = []
45 for vhost in db.get_dals(class_name="VirtualHost"):
46 hosts.append(vhost.id)
47 if vhost.id == "vlocalhost":
48 host = vhost
49 if "vlocalhost" not in hosts:
50 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
51 db.update_dal(cpus)
52 phdal = dal.PhysicalHost("localhost", contains=[cpus])
53 db.update_dal(phdal)
54 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
55 db.update_dal(host)
56 hosts.append("vlocalhost")
57
58 # Services
59 daqapp_control = db.get_dal(class_name="Service", uid="daqapp_control")
60 rccontroller_control = db.get_dal(class_name="Service", uid="rccontroller_control")
61
62 # Source IDs
63 tpw_source_id = db.get_dal("SourceIDConf", uid="srcid-tp-stream-writer")
64
65 # Queue Rules
66 trigger_record_q_rule = db.get_dal(
67 class_name="QueueConnectionRule", uid="trigger-record-q-rule"
68 )
69 dfapp_qrules = [trigger_record_q_rule]
70
71 # Net Rules
72 frag_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="frag-net-rule")
73 df_token_net_rule = db.get_dal(
74 class_name="NetworkConnectionRule", uid="df-token-net-rule"
75 )
76 tpset_net_rule = db.get_dal(
77 class_name="NetworkConnectionRule", uid="tpset-net-rule"
78 )
79 ti_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="ti-net-rule")
80 td_dfo_net_rule = db.get_dal(
81 class_name="NetworkConnectionRule", uid="td-dfo-net-rule"
82 )
83 td_trb_net_rule = db.get_dal(
84 class_name="NetworkConnectionRule", uid="td-trb-net-rule"
85 )
86 data_req_trig_net_rule = db.get_dal(
87 class_name="NetworkConnectionRule", uid="data-req-trig-net-rule"
88 )
89 data_req_hsi_net_rule = db.get_dal(
90 class_name="NetworkConnectionRule", uid="data-req-hsi-net-rule"
91 )
92 data_req_readout_net_rule = db.get_dal(
93 class_name="NetworkConnectionRule", uid="data-req-readout-net-rule"
94 )
95 dfapp_netrules = [
96 td_trb_net_rule,
97 frag_net_rule,
98 df_token_net_rule,
99 data_req_hsi_net_rule,
100 data_req_readout_net_rule,
101 data_req_trig_net_rule,
102 ]
103
104 trmon_netrules = []
105 trmon_qrules = []
106 if trmon_app:
107 trmon_req_net_rule = db.get_dal(
108 class_name="NetworkConnectionRule", uid="trmon-req-net-rule"
109 )
110 trigger_record_net_rule = db.get_dal(
111 class_name="NetworkConnectionRule", uid="trigger-record-net-rule"
112 )
113 dfapp_netrules.append(trmon_req_net_rule)
114 dfapp_netrules.append(trigger_record_net_rule)
115
116 trmon_netrules.append(trigger_record_net_rule)
117 trigger_decision_token_q_rule = db.get_dal(
118 class_name="QueueConnectionRule", uid="trigger-decision-token-q-rule"
119 )
120 trmon_qrules.append(trigger_decision_token_q_rule)
121
122 dfo_netrules = [td_dfo_net_rule, ti_net_rule, df_token_net_rule]
123 tpw_netrules = [tpset_net_rule]
124
125 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
126
127 dfo_conf = db.get_dal(class_name="DFOConf", uid="dfoconf-01")
128 dfo = dal.DFOApplication(
129 "dfo-01",
130 runs_on=host,
131 application_name="daq_application",
132 exposes_service=[daqapp_control],
133 network_rules=dfo_netrules,
134 opmon_conf=opmon_conf,
135 dfo=dfo_conf,
136 )
137 db.update_dal(dfo)
138
139 trb_conf = db.get_dal(class_name="TRBConf", uid="trb-01")
140 dw_conf = db.get_dal(class_name="DataWriterConf", uid="dw-01")
141 dfhw = db.get_dal(class_name="DFHWConf", uid="dfhw-01")
142 dfapps = []
143 for dfapp_idx in range(n_dfapps):
144 dfapp_id = dfapp_idx + 1
145
146 # Offset sids by one so that TPW sourceID can stay at 1
147 dfapp_source_id = dal.SourceIDConf(
148 f"srcid-df-{dfapp_id:02}", sid=dfapp_id + 1, subsystem="TR_Builder"
149 )
150 db.update_dal(dfapp_source_id)
151
152 dfapp = dal.DFApplication(
153 f"df-{dfapp_id:02}",
154 runs_on=host,
155 application_name="daq_application",
156 exposes_service=[daqapp_control],
157 source_id=dfapp_source_id,
158 queue_rules=dfapp_qrules,
159 network_rules=dfapp_netrules,
160 opmon_conf=opmon_conf,
161 trb=trb_conf,
162 data_writers=[dw_conf] * n_data_writers,
163 uses=dfhw,
164 )
165 db.update_dal(dfapp)
166 dfapps.append(dfapp)
167
168 tpwapps = []
169 if tpwriting_enabled:
170 tpw_writer_conf = db.get_dal(
171 class_name="TPStreamWriterConf", uid="tp-stream-writer-conf"
172 )
173
174 tpwapp = dal.TPStreamWriterApplication(
175 "tp-stream-writer",
176 runs_on=host,
177 application_name="daq_application",
178 exposes_service=[daqapp_control],
179 source_id=tpw_source_id,
180 network_rules=tpw_netrules,
181 opmon_conf=opmon_conf,
182 tp_writer=tpw_writer_conf,
183 )
184 db.update_dal(tpwapp)
185 tpwapps.append(tpwapp)
186
187 trmonapps = []
188 if trmon_app:
189 trmonconf = db.get_dal(class_name="TRMonRequestorConf", uid="trmr-01")
190 trmondwconf = db.get_dal(class_name="DataWriterConf", uid="tr_mon_dw-01")
191
192 trmonapp = dal.TRMonReqApplication(
193 "trmon-01",
194 runs_on=host,
195 application_name="daq_application",
196 exposes_service=[daqapp_control],
197 network_rules=trmon_netrules,
198 queue_rules=trmon_qrules,
199 opmon_conf=opmon_conf,
200 trmonreq=trmonconf,
201 data_writer=trmondwconf,
202 uses=dfhw,
203 )
204 db.update_dal(trmonapp)
205 trmonapps.append(trmonapp)
206
207 if generate_segment:
208 fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
209 controller = dal.RCApplication(
210 "df-controller",
211 application_name="drunc-controller",
212 runs_on=host,
213 fsm=fsm,
214 opmon_conf=opmon_conf,
215 exposes_service=[rccontroller_control],
216 )
217 db.update_dal(controller)
218
219 seg = dal.Segment(
220 f"df-segment",
221 controller=controller,
222 applications=[dfo] + dfapps + tpwapps + trmonapps,
223 )
224 db.update_dal(seg)
225
226 try:
227 db.commit()
228 except RuntimeError as err:
229 print("Failed to commit Dataflow DB!")
230 print(traceback.format_exc())
231 raise err
232 return
233
234
module(name, schema, other_dals=[], backend='oksconflibs', db=None)
Definition dal.py:678
◆ generate_fakedata()
| generate_fakedata | ( | oksfile, | |
| include, | |||
| generate_segment, | |||
| n_streams, | |||
| n_apps, | |||
| det_id, | |||
| fragment_type = None ) |
Simple script to create an OKS configuration file for a FakeDataProd-based readout segment. The file will automatically include the relevant schema files and any other OKS files you specify.
Definition at line 795 of file generate.py.
797):
798 """Simple script to create an OKS configuration file for a FakeDataProd-based readout segment.
799
800 The file will automatically include the relevant schema files and
801 any other OKS files you specify.
802
803 """
804
805 includefiles = [
806 "schema/confmodel/dunedaq.schema.xml",
807 "schema/appmodel/application.schema.xml",
808 ]
809
810 res, extra_includes = find_oksincludes(include, os.path.dirname(oksfile))
811 if res:
812 includefiles += extra_includes
813 else:
814 return
815
818 if not oksfile.endswith(".data.xml"):
819 oksfile = oksfile + ".data.xml"
820 print(f"Creating OKS database file {oksfile}")
821 db.create_db(oksfile, includefiles)
822 db.set_active(oksfile)
823
824 hosts = []
825 for vhost in db.get_dals(class_name="VirtualHost"):
826 hosts.append(vhost.id)
827 if vhost.id == "vlocalhost":
828 host = vhost
829 if "vlocalhost" not in hosts:
830 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
831 db.update_dal(cpus)
832 phdal = dal.PhysicalHost("localhost", contains=[cpus])
833 db.update_dal(phdal)
834 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
835 db.update_dal(host)
836 hosts.append("vlocalhost")
837
838 source_id = 0
839 fakeapps = []
840 # Services
841 daqapp_control = db.get_dal(class_name="Service", uid="daqapp_control")
842 rccontroller_control = db.get_dal(class_name="Service", uid="rccontroller_control")
843 dataRequests = db.get_dal(class_name="Service", uid="dataRequests")
844 timeSyncs = db.get_dal(class_name="Service", uid="timeSyncs")
845 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
846 fragagg = db.get_dal(class_name="FragmentAggregatorConf", uid="frag-agg-01")
847
848 rule = db.get_dal(
849 class_name="NetworkConnectionRule", uid="data-req-readout-net-rule"
850 )
851 netrules = [rule]
852 for rule in ["ts-fdp-net-rule"]:
853 netrules.append(db.get_dal(class_name="NetworkConnectionRule", uid=rule))
854
855 try:
856 rule = db.get_dal(
857 class_name="QueueConnectionRule", uid="fpdm-data-requests-queue-rule"
858 )
859 except:
860 print(
861 'Expected QueueConnectionRule "fpdm-data-requests-queue-rule" not found in input databases!'
862 )
863 else:
864 qrules = [rule]
865 for rule in [
866 "fa-queue-rule",
867 ]:
868 qrules.append(db.get_dal(class_name="QueueConnectionRule", uid=rule))
869
870 frame_size = 0
871 if det_id == 3:
872 frame_size = 7200
873 time_tick_diff = 32 * 64
874 response_delay = 0
875 fragment_type = "WIBEth"
876 elif det_id == 2:
877 if fragment_type == "DAPHNEEthStream":
878 frame_size = 2008
879 time_tick_diff = 280
880 response_delay = 0
881 elif fragment_type == "DAPHNEEth":
882 frame_size = 1864
883 time_tick_diff = 1024
884 response_delay = 0
885 else:
886 raise Exception(
887 f"FakeDataProd fragment_type '{fragment_type}' not recognized for detector ID {det_id}; "
888 f"expected 'DAPHNEEthStream' or 'DAPHNEEth'"
889 )
890 else:
891 raise Exception(
892 f"FakeDataProd parameters not configured for detector ID {det_id}"
893 )
894
895 for appidx in range(n_apps):
896
897 fakeapp = dal.FakeDataApplication(
898 f"fakedata_{appidx}",
899 runs_on=host,
900 application_name="daq_application",
901 exposes_service=[daqapp_control, dataRequests, timeSyncs],
902 queue_rules=qrules,
903 network_rules=netrules,
904 fragment_aggregator=fragagg,
905 opmon_conf=opmon_conf,
906 )
907
908 for streamidx in range(n_streams):
909 stream = dal.FakeDataProdConf(
910 f"fakedata_{appidx}_stream_{streamidx}",
911 system_type="Detector_Readout",
912 source_id=source_id,
913 time_tick_diff=time_tick_diff,
914 frame_size=frame_size,
915 response_delay=response_delay,
916 fragment_type=fragment_type,
917 )
918 db.update_dal(stream)
919 fakeapp.producers.append(stream)
920 source_id = source_id + 1
921
922 db.update_dal(fakeapp)
923 fakeapps.append(fakeapp)
924
925 if generate_segment:
926 fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
927 controller = dal.RCApplication(
928 "ru-controller",
929 application_name="drunc-controller",
930 opmon_conf=opmon_conf,
931 runs_on=host,
932 fsm=fsm,
933 exposes_service=[rccontroller_control],
934 )
935 db.update_dal(controller)
936
937 seg = dal.Segment(
938 f"ru-segment",
939 controller=controller,
940 applications=fakeapps,
941 )
942 db.update_dal(seg)
943
944 db.commit()
945 return
946
947
◆ generate_hsi()
| generate_hsi | ( | oksfile, | |
| include, | |||
| generate_segment ) |
Simple script to create an OKS configuration file for a FakeHSI segment. The file will automatically include the relevant schema files and any other OKS files you specify.
Definition at line 235 of file generate.py.
239):
240 """Simple script to create an OKS configuration file for a FakeHSI segment.
241
242 The file will automatically include the relevant schema files and
243 any other OKS files you specify.
244
245
246 """
247
248 includefiles = [
249 "schema/confmodel/dunedaq.schema.xml",
250 "schema/appmodel/application.schema.xml",
251 "schema/appmodel/trigger.schema.xml",
252 ]
253
254 res, extra_includes = find_oksincludes(include, os.path.dirname(oksfile))
255 if res:
256 includefiles += extra_includes
257 else:
258 return
259
262 if not oksfile.endswith(".data.xml"):
263 oksfile = oksfile + ".data.xml"
264 print(f"Creating OKS database file {oksfile}")
265 db.create_db(oksfile, includefiles)
266 db.set_active(oksfile)
267
268 hosts = []
269 for vhost in db.get_dals(class_name="VirtualHost"):
270 hosts.append(vhost.id)
271 if vhost.id == "vlocalhost":
272 host = vhost
273 if "vlocalhost" not in hosts:
274 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
275 db.update_dal(cpus)
276 phdal = dal.PhysicalHost("localhost", contains=[cpus])
277 db.update_dal(phdal)
278 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
279 db.update_dal(host)
280 hosts.append("vlocalhost")
281
282 # Services
283 daqapp_control = db.get_dal(class_name="Service", uid="daqapp_control")
284 rccontroller_control = db.get_dal(class_name="Service", uid="rccontroller_control")
285 dataRequests = db.get_dal(class_name="Service", uid="dataRequests")
286 hsievents = db.get_dal(class_name="Service", uid="HSIEvents")
287
288 # Source IDs
289 hsi_source_id = db.get_dal(class_name="SourceIDConf", uid="hsi-srcid-01")
290 hsi_tc_source_id = db.get_dal(class_name="SourceIDConf", uid="hsi-tc-srcid-1")
291
292 # Queue Rules
293 hsi_dlh_queue_rule = db.get_dal(
294 class_name="QueueConnectionRule", uid="hsi-dlh-data-requests-queue-rule"
295 )
296 hsi_qrules = [hsi_dlh_queue_rule]
297
298 # Net Rules
299 tc_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="tc-net-rule")
300 hsi_rule = db.get_dal(class_name="NetworkConnectionRule", uid="hsi-rule")
301 ts_hsi_net_rule = db.get_dal(
302 class_name="NetworkConnectionRule", uid="ts-hsi-net-rule"
303 )
304 data_req_hsi_net_rule = db.get_dal(
305 class_name="NetworkConnectionRule", uid="data-req-hsi-net-rule"
306 )
307 hsi_netrules = [hsi_rule, data_req_hsi_net_rule, ts_hsi_net_rule]
308 tc_netrules = [hsi_rule, tc_net_rule]
309
310 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
311 hsi_handler = db.get_dal(class_name="DataHandlerConf", uid="def-hsi-handler")
312 fakehsi = db.get_dal(class_name="FakeHSIEventGeneratorConf", uid="fakehsi")
313
314 hsi = dal.FakeHSIApplication(
315 "hsi-01",
316 runs_on=host,
317 application_name="daq_application",
318 exposes_service=[daqapp_control],
319 source_id=hsi_source_id,
320 queue_rules=hsi_qrules,
321 network_rules=hsi_netrules,
322 opmon_conf=opmon_conf,
323 link_handler=hsi_handler,
324 generator=fakehsi,
325 )
326 db.update_dal(hsi)
327
328 hsi_to_tc_conf = db.get_dal(class_name="HSI2TCTranslatorConf", uid="hsi-to-tc-conf")
329
330 hsi_to_tc = dal.HSIEventToTCApplication(
331 "hsi-to-tc-app",
332 runs_on=host,
333 application_name="daq_application",
334 exposes_service=[dataRequests, hsievents, daqapp_control],
335 source_id=hsi_tc_source_id,
336 network_rules=tc_netrules,
337 opmon_conf=opmon_conf,
338 hsevent_to_tc_conf=hsi_to_tc_conf,
339 )
340 db.update_dal(hsi_to_tc)
341
342 if generate_segment:
343 fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
344 controller = dal.RCApplication(
345 "hsi-controller",
346 application_name="drunc-controller",
347 runs_on=host,
348 fsm=fsm,
349 opmon_conf=opmon_conf,
350 exposes_service=[rccontroller_control],
351 )
352 db.update_dal(controller)
353
354 seg = dal.Segment(
355 f"hsi-segment", controller=controller, applications=[hsi, hsi_to_tc]
356 )
357 db.update_dal(seg)
358
359 db.commit()
360 return
361
362
◆ generate_readout()
| generate_readout | ( | readoutmap, | |
| oksfile, | |||
| include, | |||
| generate_segment, | |||
| emulated_file_name, | |||
| tpg_enabled = True, | |||
| hosts_to_use = [] ) |
Simple script to create an OKS configuration file for all ReadoutApplications defined in a readout map. The file will automatically include the relevant schema files and any other OKS files you specify. Example: generate_readoutOKS -i hosts \ -i appmodel/connections.data.xml -i appmodel/moduleconfs \ config/np04readoutmap.data.xml readoutApps.data.xml Will load hosts, connections and moduleconfs data files as well as the readoutmap (config/np04readoutmap.data.xml) and write the generated apps to readoutApps.data.xml. generate_readoutOKS --session --segment \ -i appmodel/fsm -i hosts \ -i appmodel/connections.data.xml -i appmodel/moduleconfs \ config/np04readoutmap.data.xml np04readout-session.data.xml Will do the same but in addition it will generate a containing Segment for the apps and a containing Session for the Segment. NB: Currently FSM generation is not implemented so you must include an fsm file in order to generate a Segment
Definition at line 363 of file generate.py.
371):
372 """Simple script to create an OKS configuration file for all
373 ReadoutApplications defined in a readout map.
374
375 The file will automatically include the relevant schema files and
376 any other OKS files you specify.
377
378 Example:
379 generate_readoutOKS -i hosts \
380 -i appmodel/connections.data.xml -i appmodel/moduleconfs \
381 config/np04readoutmap.data.xml readoutApps.data.xml
382
383 Will load hosts, connections and moduleconfs data files as well as
384 the readoutmap (config/np04readoutmap.data.xml) and write the
385 generated apps to readoutApps.data.xml.
386
387 generate_readoutOKS --session --segment \
388 -i appmodel/fsm -i hosts \
389 -i appmodel/connections.data.xml -i appmodel/moduleconfs \
390 config/np04readoutmap.data.xml np04readout-session.data.xml
391
392 Will do the same but in addition it will generate a containing
393 Segment for the apps and a containing Session for the Segment.
394
395 NB: Currently FSM generation is not implemented so you must include
396 an fsm file in order to generate a Segment
397
398 """
399
400 if not readoutmap.endswith(".data.xml"):
401 readoutmap = readoutmap + ".data.xml"
402
403 print(f"Readout map file {readoutmap}")
404
405 includefiles = (
406 [
407 "schema/confmodel/dunedaq.schema.xml",
408 "schema/appmodel/application.schema.xml",
409 "schema/appmodel/trigger.schema.xml",
410 "schema/appmodel/fdmodules.schema.xml",
411 "schema/appmodel/wiec.schema.xml",
412 ]
413 + [readoutmap]
414 if os.path.exists(readoutmap)
415 else []
416 )
417
418 searchdirs = [path for path in os.environ["DUNEDAQ_DB_PATH"].split(":")]
419 searchdirs.append(os.path.dirname(oksfile))
420 for inc in include:
421 # print (f"Searching for {inc}")
422 match = False
423 inc = inc.removesuffix(".xml")
424 if inc.endswith(".data"):
425 sub_dirs = ["config", "data"]
426 elif inc.endswith(".schema"):
427 sub_dirs = ["schema"]
428 else:
429 sub_dirs = ["*"]
430 inc = inc + "*"
431 for path in searchdirs:
432 # print (f" {path}/{inc}.xml")
433 matches = glob.glob(f"{inc}.xml", root_dir=path)
434 if len(matches) == 0:
435 for search_dir in sub_dirs:
436 # print (f" {path}/{search_dir}/{inc}.xml")
437 matches = glob.glob(f"{search_dir}/{inc}.xml", root_dir=path)
438 for filename in matches:
439 if filename not in includefiles:
440 print(f"Adding {filename} to include list")
441 includefiles.append(filename)
442 else:
443 print(f"{filename} already in include list")
444 match = True
445 break
446 if match:
447 break
448 if match:
449 break
450 else:
451 for filename in matches:
452 if filename not in includefiles:
453 print(f"Adding {filename} to include list")
454 includefiles.append(filename)
455 else:
456 print(f"{filename} already in include list")
457 match = True
458 break
459
460 if not match:
461 print(f"Error could not find include file for {inc}")
462 return
463
466 if not oksfile.endswith(".data.xml"):
467 oksfile = oksfile + ".data.xml"
468 print(f"Creating OKS database file {oksfile}")
469 db.create_db(oksfile, includefiles)
470 db.set_active(oksfile)
471
472 detector_connections = db.get_dals(class_name="DetectorToDaqConnection")
473 daqapp_control = db.get_dal(class_name="Service", uid="daqapp_control")
474 rccontroller_control = db.get_dal(class_name="Service", uid="rccontroller_control")
475
476 try:
477 rule = db.get_dal(
478 class_name="NetworkConnectionRule", uid="data-req-readout-net-rule"
479 )
480 except:
481 print(
482 'Expected NetworkConnectionRule "data-req-readout-net-rule" not found in input databases!'
483 )
484 else:
485 netrules = [rule]
486 # Assume we have all the other rules we need
487 for rule in ["tpset-net-rule", "ts-net-rule", "ta-net-rule"]:
488 netrules.append(db.get_dal(class_name="NetworkConnectionRule", uid=rule))
489
490 try:
491 rule = db.get_dal(
492 class_name="QueueConnectionRule", uid="fd-dlh-data-requests-queue-rule"
493 )
494 except:
495 print(
496 'Expected QueueConnectionRule "fd-dlh-data-requests-queue-rule" not found in input databases!'
497 )
498 else:
499 qrules = [rule]
500 for rule in [
501 "fa-queue-rule",
502 "tp-queue-rule",
503 ]:
504 qrules.append(db.get_dal(class_name="QueueConnectionRule", uid=rule))
505
506 hosts = []
507 if len(hosts_to_use) == 0:
508 for vhost in db.get_dals(class_name="VirtualHost"):
509 if vhost.id == "vlocalhost":
510 hosts.append(vhost.id)
511 if "vlocalhost" not in hosts:
512 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
513 db.update_dal(cpus)
514 phdal = dal.PhysicalHost("localhost", contains=[cpus])
515 db.update_dal(phdal)
516 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
517 db.update_dal(host)
518 hosts.append("vlocalhost")
519 else:
520 for vhost in db.get_dals(class_name="VirtualHost"):
521 if vhost.id in hosts_to_use:
522 hosts.append(vhost.id)
523 assert len(hosts) > 0
524
525 rohw = dal.RoHwConfig(f"rohw-{detector_connections[0].id}")
526 db.update_dal(rohw)
527
528 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
529 fragagg = db.get_dal(class_name="FragmentAggregatorConf", uid="frag-agg-01")
530
531 appnum = 0
532 nicrec = None
533 flxcard = None
534 wm_conf = None
535 hermes_conf = None
536 ruapps = []
537 for connection in detector_connections:
538
539 geo_id = connection.get("GeoId")
540 det_id = geo_id[0].detector_id
541 if det_id == 0:
542 raise Exception(f"Unable to determine detector ID from Hardware Map!")
543
544 tphandler = db.get_dal(class_name="DataHandlerConf", uid="def-tp-handler")
545
546 if det_id == 2:
547 if "DAPHNEStream" in emulated_file_name:
548 linkhandler = db.get_dal(
549 class_name="DataHandlerConf", uid="def-pds-stream-link-handler"
550 )
551 cb_desc = db.get_dal(
552 class_name="DataMoveCallbackDescriptor", uid="pds-stream-raw-input"
553 )
554 else:
555 linkhandler = db.get_dal(
556 class_name="DataHandlerConf", uid="def-pds-link-handler"
557 )
558 cb_desc = db.get_dal(
559 class_name="DataMoveCallbackDescriptor", uid="pds-raw-input"
560 )
561
562 elif det_id == 3 or det_id == 10:
563 linkhandler = db.get_dal(
564 class_name="DataHandlerConf", uid="def-link-handler"
565 )
566 cb_desc = db.get_dal(
567 class_name="DataMoveCallbackDescriptor", uid="wib-eth-raw-input"
568 )
569 elif det_id == 11:
570 linkhandler = db.get_dal(
571 class_name="DataHandlerConf", uid="def-tde-link-handler"
572 )
573 cb_desc = db.get_dal(
574 class_name="DataMoveCallbackDescriptor", uid="tde-raw-input"
575 )
576 elif det_id == 12:
577 linkhandler = db.get_dal(
578 class_name="DataHandlerConf", uid="def-crt-bern-link-handler"
579 )
580 # Not used, but needed for ReadoutApplication
581 cb_desc = db.get_dal(
582 class_name="DataMoveCallbackDescriptor", uid="crt-bern-raw-input"
583 )
584 elif det_id == 13:
585 linkhandler = db.get_dal(
586 class_name="DataHandlerConf", uid="def-crt-grenoble-link-handler"
587 )
588 # Not used, but needed for ReadoutApplication
589 cb_desc = db.get_dal(
590 class_name="DataMoveCallbackDescriptor", uid="crt-grenoble-raw-input"
591 )
592
593 hostnum = appnum % len(hosts)
594 # print(f"Looking up host[{hostnum}] ({hosts[hostnum]})")
595 host = db.get_dal(class_name="VirtualHost", uid=hosts[hostnum])
596
597 # Find which type of DataReceiver we need for this connection
598 if connection.className() == "NetworkDetectorToDaqConnection":
599 receiver = connection.net_receiver
600 elif connection.className() == "FelixDetectorToDaqConnection":
601 receiver = connection.felix_receiver
602
603 # Action Plans
604 readout_start = db.get_dal(class_name="ActionPlan", uid="readout-start")
605 readout_stop = db.get_dal(class_name="ActionPlan", uid="readout-stop")
606
607 # Emulated stream
608 if type(receiver).__name__ == "FakeDataReceiver":
609 if nicrec == None:
610 try:
611 stream_emu = db.get_dal(
612 class_name="StreamEmulationParameters", uid="stream-emu"
613 )
614 stream_emu.data_file_name = resolve_asset_file(emulated_file_name)
615 db.update_dal(stream_emu)
616 except:
617 stream_emu = dal.StreamEmulationParameters(
618 "stream-emu",
619 data_file_name=resolve_asset_file(emulated_file_name),
620 input_file_size_limit=5777280,
621 set_t0=True,
622 random_population_size=100000,
623 frame_error_rate_hz=0,
624 generate_periodic_adc_pattern=True,
625 TP_rate_per_channel=1,
626 )
627 db.update_dal(stream_emu)
628
629 print("Generating fake DataReaderConf")
630 nicrec = dal.DPDKReaderConf(
631 f"nicrcvr-fake-gen",
632 template_for="FDFakeReaderModule",
633 emulation_mode=1,
634 emulation_conf=stream_emu,
635 )
636 db.update_dal(nicrec)
637 datareader = nicrec
638 elif type(receiver).__name__ == "DPDKReceiver":
639 if nicrec == None:
640 print("Generating DPDKReaderConf")
641 nicrec = dal.DPDKReaderConf(
642 f"nicrcvr-dpdk-gen", template_for="DPDKReaderModule"
643 )
644 db.update_dal(nicrec)
645 if wm_conf == None:
646 try:
647 wm_conf = db.get_dal("WIBModuleConf", "def-wib-conf")
648 except:
649 print(
650 'Expected WIBModuleConf "def-wib-conf" not found in input databases!'
651 )
652 if hermes_conf == None:
653 try:
654 hermes_conf = db.get_dal("HermesModuleConf", "def-hermes-conf")
655 except:
656 print(
657 'Expected HermesModuleConf "def-hermes-conf" not found in input databases!'
658 )
659
660 datareader = nicrec
661
662 wiec_app = dal.WIECApplication(
663 f"wiec-{connection.id}",
664 application_name="daq_application",
665 runs_on=host,
666 detector_connections=[connection],
667 wib_module_conf=wm_conf,
668 hermes_module_conf=hermes_conf,
669 exposes_service=[daqapp_control],
670 )
671 db.update_dal(wiec_app)
672
673 elif type(receiver).__name__ == "FelixInterface":
674 if flxcard == None:
675 print("Generating Felix DataReaderConf")
676 flxcard = dal.DataReaderConf(
677 f"flxConf-1", template_for="FelixReaderModule"
678 )
679 db.update_dal(flxcard)
680 datareader = flxcard
681 elif type(receiver).__name__ == "FileReaderReceiver":
682 if nicrec == None:
683 try:
684 snb_files = db.get_dal(
685 class_name="SNBFileSourceParameters",
686 uid=f"snb-files-{connection.id}",
687 )
688 snb_files.data_files = [resolve_asset_file(emulated_file_name)]
689 db.update_dal(snb_files)
690 except:
691 snb_files = dal.SNBFileSourceParameters(
692 "snb-files-0",
693 data_files=[resolve_asset_file(emulated_file_name)],
694 input_buffer_size=5777280,
695 file_compression_algorithm="None",
696 )
697 db.update_dal(snb_files)
698
699 print("Generating fake DataReaderConf")
700 nicrec = dal.SNBFileReaderConf(
701 f"nicrcvr-file-reader",
702 template_for="SNBFileReaderModule",
703 emulation_mode=1,
704 snb_conf=snb_files,
705 )
706 db.update_dal(nicrec)
707 datareader = nicrec
708
709 print(f"Using SNB DataHandler")
710 linkhandler.template_for = "SNBDataHandlerModule"
711 db.update_dal(linkhandler)
712
713 readout_start = db.get_dal(class_name="ActionPlan", uid="snb-readout-start")
714 readout_stop = db.get_dal(class_name="ActionPlan", uid="snb-readout-stop")
715 else:
716 print(
717 f"ReadoutGroup contains unknown interface type {type(receiver).__name__}"
718 )
719 continue
720
721 db.commit()
722
723 # Services
724 dataRequests = db.get_dal(class_name="Service", uid="dataRequests")
725 timeSyncs = db.get_dal(class_name="Service", uid="timeSyncs")
726 triggerActivities = db.get_dal(class_name="Service", uid="triggerActivities")
727 triggerPrimitives = db.get_dal(class_name="Service", uid="triggerPrimitives")
728
729 ru = dal.ReadoutApplication(
730 f"ru-{connection.id}",
731 application_name="daq_application",
732 runs_on=host,
733 detector_connections=[connection],
734 network_rules=netrules,
735 queue_rules=qrules,
736 link_handler=linkhandler,
737 data_reader=datareader,
738 fragment_aggregator=fragagg,
739 opmon_conf=opmon_conf,
740 tp_generation_enabled=tpg_enabled,
741 ta_generation_enabled=tpg_enabled,
742 uses=rohw,
743 exposes_service=[daqapp_control, dataRequests, timeSyncs],
744 action_plans=[readout_start, readout_stop],
745 callback_desc=cb_desc,
746 )
747 if tpg_enabled:
748 ru.tp_handler = tphandler
749 tp_sources = []
750 tpbaseid = (appnum * 3) + 100
751 # 30-Apr-2025, KAB: added support for 1 "plane" of non-TPC TPs (e.g. PDS).
752 # That is compared with the usual 3 planes of TPs for TPC detectors.
753 for plane in range(1 if det_id not in [3, 10, 11] else 3):
754 s_id = tpbaseid + plane
755 tps_dal = dal.SourceIDConf(
756 f"tp-srcid-{s_id}", sid=s_id, subsystem="Trigger"
757 )
758 db.update_dal(tps_dal)
759 tp_sources.append(tps_dal)
760 ru.tp_source_ids = tp_sources
761 ru.exposes_service += [triggerActivities, triggerPrimitives]
762 appnum = appnum + 1
763 print(f"{ru=}")
764 db.update_dal(ru)
765 db.commit()
766 ruapps.append(ru)
767 if appnum == 0:
768 print(f"No ReadoutApplications generated\n")
769 return
770
771 db.commit()
772
773 if generate_segment:
774 # fsm = db.get_dal(class_name="FSMconfiguration", uid="fsmConf-test")
775 fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
776 controller = dal.RCApplication(
777 "ru-controller",
778 application_name="drunc-controller",
779 runs_on=host,
780 fsm=fsm,
781 opmon_conf=opmon_conf,
782 exposes_service=[rccontroller_control],
783 )
784 db.update_dal(controller)
785 db.commit()
786
787 seg = dal.Segment(f"ru-segment", controller=controller, applications=ruapps)
788 db.update_dal(seg)
789 db.commit()
790
791 db.commit()
792 return
793
794
◆ generate_session()
| generate_session | ( | oksfile, | |
| include, | |||
| session_name, | |||
| op_env, | |||
| connectivity_service_is_infrastructure_app = True, | |||
| disable_connectivity_service = False ) |
Simple script to create an OKS configuration file for a session. The file will automatically include the relevant schema files and any other OKS files you specify.
Definition at line 1124 of file generate.py.
1131):
1132 """Simple script to create an OKS configuration file for a session.
1133
1134 The file will automatically include the relevant schema files and
1135 any other OKS files you specify.
1136
1137 """
1138
1139 includefiles = [
1140 "schema/confmodel/dunedaq.schema.xml",
1141 "schema/appmodel/application.schema.xml",
1142 ]
1143 res, extra_includes = find_oksincludes(include, os.path.dirname(oksfile))
1144 if res:
1145 includefiles += extra_includes
1146 else:
1147 return
1148
1151 if not oksfile.endswith(".data.xml"):
1152 oksfile = oksfile + ".data.xml"
1153 print(f"Creating OKS database file {oksfile} with includes {includefiles}")
1154 db.create_db(oksfile, includefiles)
1155 db.set_active(oksfile)
1156
1157 hosts = []
1158 for vhost in db.get_dals(class_name="VirtualHost"):
1159 hosts.append(vhost.id)
1160 if vhost.id == "vlocalhost":
1161 host = vhost
1162 if "vlocalhost" not in hosts:
1163 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
1164 db.update_dal(cpus)
1165 phdal = dal.PhysicalHost("localhost", contains=[cpus])
1166 db.update_dal(phdal)
1167 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
1168 db.update_dal(host)
1169 hosts.append("vlocalhost")
1170
1171 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
1172
1173 fsm = db.get_dal(class_name="FSMconfiguration", uid="fsmConf-test")
1174 rccontroller_control = db.get_dal(class_name="Service", uid="root-rccontroller_control")
1175 controller = dal.RCApplication(
1176 "root-controller",
1177 application_name="drunc-controller",
1178 runs_on=host,
1179 fsm=fsm,
1180 opmon_conf=opmon_conf,
1181 exposes_service=[rccontroller_control],
1182 )
1183 db.update_dal(controller)
1184
1185 segments = db.get_dals(class_name="Segment")
1186
1187 seg = dal.Segment(f"root-segment", controller=controller, segments=segments)
1188 db.update_dal(seg)
1189
1190 detconf = db.get_dal(class_name="DetectorConfig", uid="dummy-detector")
1191
1192 detconf.op_env = op_env
1193 db.update_dal(detconf)
1194
1195 opmon_svc = db.get_dal(class_name="OpMonURI", uid="local-opmon-uri")
1196
1197 trace_file_var = None
1198 TRACE_FILE = os.getenv("TRACE_FILE")
1199 if TRACE_FILE is not None:
1200 trace_file_var = dal.Variable(
1201 "session-env-trace-file", name="TRACE_FILE", value=TRACE_FILE
1202 )
1203 db.update_dal(trace_file_var)
1204
1205 infrastructure_applications = []
1206 if connectivity_service_is_infrastructure_app:
1207 conn_svc = db.get_dal(
1208 class_name="ConnectionService", uid="local-connection-server"
1209 )
1210 infrastructure_applications.append(conn_svc)
1211
1212 env_vars_for_local_running = db.get_dal(
1213 class_name="VariableSet", uid="local-variables"
1214 ).contains
1215 if trace_file_var is not None:
1216 env_vars_for_local_running.append(trace_file_var)
1217
1218 sessiondal = dal.Session(
1219 session_name,
1220 environment=env_vars_for_local_running,
1221 segment=seg,
1222 detector_configuration=detconf,
1223 infrastructure_applications=infrastructure_applications,
1224 opmon_uri=opmon_svc,
1225 )
1226
1227 if not disable_connectivity_service:
1228 conn_svc_cfg = db.get_dal(
1229 class_name="ConnectivityService", uid="local-connectivity-service-config"
1230 )
1231 sessiondal.connectivity_service = conn_svc_cfg
1232
1233 db.update_dal(sessiondal)
1234
1235 db.commit()
1236 return
◆ generate_trigger()
| generate_trigger | ( | oksfile, | |
| include, | |||
| generate_segment, | |||
| tpg_enabled = True, | |||
| hsi_enabled = False ) |
Simple script to create an OKS configuration file for a trigger segment. The file will automatically include the relevant schema files and any other OKS files you specify.
Definition at line 948 of file generate.py.
954):
955 """Simple script to create an OKS configuration file for a trigger segment.
956
957 The file will automatically include the relevant schema files and
958 any other OKS files you specify.
959
960 """
961
962 includefiles = [
963 "schema/confmodel/dunedaq.schema.xml",
964 "schema/appmodel/application.schema.xml",
965 "schema/appmodel/trigger.schema.xml",
966 ]
967
968 res, extra_includes = find_oksincludes(include, os.path.dirname(oksfile))
969 if res:
970 includefiles += extra_includes
971 else:
972 return
973
976 if not oksfile.endswith(".data.xml"):
977 oksfile = oksfile + ".data.xml"
978 print(f"Creating OKS database file {oksfile}")
979 db.create_db(oksfile, includefiles)
980 db.set_active(oksfile)
981
982 hosts = []
983 for vhost in db.get_dals(class_name="VirtualHost"):
984 hosts.append(vhost.id)
985 if vhost.id == "vlocalhost":
986 host = vhost
987 if "vlocalhost" not in hosts:
988 cpus = dal.ProcessingResource("cpus", cpu_cores=[0, 1, 2, 3])
989 db.update_dal(cpus)
990 phdal = dal.PhysicalHost("localhost", contains=[cpus])
991 db.update_dal(phdal)
992 host = dal.VirtualHost("vlocalhost", runs_on=phdal, uses=[cpus])
993 db.update_dal(host)
994 hosts.append("vlocalhost")
995
996 # Services
997 daqapp_control = db.get_dal(class_name="Service", uid="daqapp_control")
998 rccontroller_control = db.get_dal(class_name="Service", uid="rccontroller_control")
999 dataRequests = db.get_dal(class_name="Service", uid="dataRequests")
1000 triggerActivities = db.get_dal(class_name="Service", uid="triggerActivities")
1001 triggerCandidates = db.get_dal(class_name="Service", uid="triggerCandidates")
1002 triggerInhibits = db.get_dal(class_name="Service", uid="triggerInhibits")
1003
1004 # Source IDs
1005 mlt_source_id = db.get_dal(class_name="SourceIDConf", uid="tc-srcid-1")
1006 tc_source_id = db.get_dal(class_name="SourceIDConf", uid="ta-srcid-1")
1007
1008 # Queue Rules
1009 tc_queue_rule = db.get_dal(class_name="QueueConnectionRule", uid="tc-queue-rule")
1010 td_queue_rule = db.get_dal(class_name="QueueConnectionRule", uid="td-queue-rule")
1011 ta_queue_rule = db.get_dal(class_name="QueueConnectionRule", uid="ta-queue-rule")
1012 mlt_qrules = [tc_queue_rule, td_queue_rule]
1013 tapp_qrules = [ta_queue_rule]
1014
1015 # Net Rules
1016 tc_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="tc-net-rule")
1017 ta_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="ta-net-rule")
1018 ts_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="ts-net-rule")
1019 ti_net_rule = db.get_dal(class_name="NetworkConnectionRule", uid="ti-net-rule")
1020 td_dfo_net_rule = db.get_dal(
1021 class_name="NetworkConnectionRule", uid="td-dfo-net-rule"
1022 )
1023 data_req_trig_net_rule = db.get_dal(
1024 class_name="NetworkConnectionRule", uid="data-req-trig-net-rule"
1025 )
1026 mlt_netrules = [
1027 tc_net_rule,
1028 ti_net_rule,
1029 td_dfo_net_rule,
1030 data_req_trig_net_rule,
1031 ts_net_rule,
1032 ]
1033 tapp_netrules = [ta_net_rule, tc_net_rule, data_req_trig_net_rule]
1034
1035 opmon_conf = db.get_dal(class_name="OpMonConf", uid="slow-all-monitoring")
1036 tc_subscriber = db.get_dal(class_name="DataReaderConf", uid="tc-subscriber-1")
1037 tc_handler = db.get_dal(class_name="DataHandlerConf", uid="def-tc-handler")
1038 mlt_conf = db.get_dal(class_name="MLTConf", uid="def-mlt-conf")
1039 random_tc_generator = db.get_dal(
1040 class_name="RandomTCMakerConf", uid="random-tc-generator"
1041 )
1042 tc_confs = []
1043
1044 try:
1045 fixedtime_tc_generator = db.get_dal(
1046 class_name="FixedTimeTCMakerModuleConf",
1047 uid="ft-trig-conf",
1048 )
1049 print(
1050 f"FixedTimeTCMakerModule has been configured, disabling random triggers and HSI"
1051 )
1052 tc_confs = [fixedtime_tc_generator]
1053 except:
1054 pass # No FixedTimeTCMaker
1055 if not hsi_enabled and len(tc_confs) == 0:
1056 tc_confs = [random_tc_generator]
1057
1058 mlt = dal.MLTApplication(
1059 "mlt",
1060 runs_on=host,
1061 application_name="daq_application",
1062 exposes_service=[
1063 daqapp_control,
1064 triggerCandidates,
1065 triggerInhibits,
1066 dataRequests,
1067 ],
1068 source_id=mlt_source_id,
1069 queue_rules=mlt_qrules,
1070 network_rules=mlt_netrules,
1071 opmon_conf=opmon_conf,
1072 data_subscriber=tc_subscriber,
1073 trigger_inputs_handler=tc_handler,
1074 mlt_conf=mlt_conf,
1075 standalone_candidate_maker_confs=tc_confs,
1076 )
1077 db.update_dal(mlt)
1078
1079 if tpg_enabled:
1080 ta_subscriber = db.get_dal(class_name="DataReaderConf", uid="ta-subscriber-1")
1081 ta_handler = db.get_dal(class_name="DataHandlerConf", uid="def-ta-handler")
1082
1083 # Action Plans
1084 tc_maker_start = db.get_dal(class_name="ActionPlan", uid="tc-maker-start")
1085
1086 tcmaker = dal.TriggerApplication(
1087 "tc-maker-1",
1088 runs_on=host,
1089 application_name="daq_application",
1090 exposes_service=[daqapp_control, triggerActivities, dataRequests],
1091 source_id=tc_source_id,
1092 queue_rules=tapp_qrules,
1093 network_rules=tapp_netrules,
1094 opmon_conf=opmon_conf,
1095 data_subscriber=ta_subscriber,
1096 trigger_inputs_handler=ta_handler,
1097 action_plans=[tc_maker_start],
1098 )
1099 db.update_dal(tcmaker)
1100
1101 if generate_segment:
1102 fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
1103 controller = dal.RCApplication(
1104 "trg-controller",
1105 application_name="drunc-controller",
1106 opmon_conf=opmon_conf,
1107 runs_on=host,
1108 fsm=fsm,
1109 exposes_service=[rccontroller_control],
1110 )
1111 db.update_dal(controller)
1112
1113 seg = dal.Segment(
1114 f"trg-segment",
1115 controller=controller,
1116 applications=[mlt] + ([tcmaker] if tpg_enabled else []),
1117 )
1118 db.update_dal(seg)
1119
1120 db.commit()
1121 return
1122
1123
Generated on for DUNE-DAQ by
1.17.0