Line data Source code
1 : /**
2 : * @file test_lb_allocation_app.cxx Test application for LB allocations.
3 : *
4 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
5 : * Licensing/copyright details are in the COPYING file that you should have
6 : * received with this code.
7 : */
8 :
9 : #include "datahandlinglibs/models/IterableQueueModel.hpp"
10 : #include "datahandlinglibs/models/SkipListLatencyBufferModel.hpp"
11 : #include "datahandlinglibs/concepts/RawDataProcessorConcept.hpp"
12 : #include "logging/Logging.hpp"
13 :
14 : #include "CLI/App.hpp"
15 : #include "CLI/Config.hpp"
16 : #include "CLI/Formatter.hpp"
17 :
18 : #include <atomic>
19 : #include <chrono>
20 : #include <memory>
21 : #include <random>
22 : #include <string>
23 : #include <vector>
24 :
25 : #ifdef WITH_LIBNUMA_SUPPORT
26 : #include <numaif.h>
27 : #endif
28 :
29 : //#define REGISTER (*(volatile unsigned char*)0x1234)
30 :
31 : using namespace dunedaq::datahandlinglibs;
32 :
33 : namespace {
34 :
35 : struct kBlock // Dummy data type for LB test
36 : {
37 0 : kBlock(){};
38 : char data[1024];
39 : };
40 :
41 : std::size_t lb_capacity = 1000; // LB capacity
42 :
43 : bool numa_aware_test = false;
44 : int num_numa_nodes = 2;
45 : bool intrinsic_test = false;
46 : bool aligned_test = false;
47 : bool prefill = false; // Prefill the LB
48 : std::size_t alignment_size = 4096;
49 : }
50 :
51 : int
52 0 : main(int argc, char** argv)
53 : {
54 0 : int runsecs = 5;
55 :
56 : // Run marker
57 0 : std::atomic<bool> marker{ true };
58 :
59 : // Counter for ops/s
60 0 : std::atomic<int> newops = 0;
61 :
62 0 : CLI::App app{"datahandlinglibs_test_lb_allocation"};
63 0 : app.add_option("-c", lb_capacity, "Capacity/size of latency buffer.");
64 0 : app.add_flag("--numa_aware", numa_aware_test, "Test NUMA aware allocator.");
65 0 : app.add_option("--num_numa_nodes", num_numa_nodes, "Number of NUMA nodes to test allocation on.");
66 0 : app.add_flag("--intrinsic", intrinsic_test, "Test Intrinsic allocator.");
67 0 : app.add_flag("--aligned", aligned_test, "Test aligned allocator.");
68 0 : app.add_option("--alignment_size", alignment_size, "Set alignment size. Default: 4096");
69 0 : app.add_flag("--prefill", prefill, "Interface to init");
70 0 : CLI11_PARSE(app, argc, argv);
71 :
72 0 : if (numa_aware_test) { // check if test is issued...
73 : #ifdef WITH_LIBNUMA_SUPPORT
74 0 : TLOG() << "NUMA aware allocator test...";
75 :
76 0 : for (int i=0; i<num_numa_nodes; ++i) { // for number of nodes...
77 0 : TLOG() << " # Allocating NUMA aware LB on node " << i;
78 0 : IterableQueueModel<kBlock> numaIQM(lb_capacity, true, i, false, 0);
79 :
80 0 : if (prefill) { // Force page-fault issues?
81 0 : TLOG() << " -> Prefilling LB...";
82 0 : numaIQM.force_pagefault();
83 : }
84 :
85 0 : for (std::size_t i=0; i<lb_capacity-1; ++i) { // Fill the LB
86 0 : numaIQM.write(kBlock());
87 : }
88 :
89 : // Test if the elements' pointers are on correct NUMA node.
90 : // Virtual addresses might be misleading between virt. and phys. addresses due to IOMMU!
91 0 : int numa_node = -1;
92 0 : get_mempolicy(&numa_node, NULL, 0, (void*)numaIQM.front(), MPOL_F_NODE | MPOL_F_ADDR);
93 0 : if (i != numa_node) { TLOG() << "Discrepancy in expected NUMA node and first element residency!"; }
94 0 : TLOG() << " -> NUMA " << i << " IQM front virt.addr.: " << std::hex << (void*)numaIQM.front() << " is on: " << numa_node << std::dec;
95 0 : get_mempolicy(&numa_node, NULL, 0, (void*)numaIQM.back(), MPOL_F_NODE | MPOL_F_ADDR);
96 0 : if (i != numa_node) { TLOG() << "Discrepancy in expected NUMA node and last element residency!"; }
97 0 : TLOG() << " -> NUMA " << i << " IQM back virt.addr.: " << std::hex << (void*)numaIQM.back() << " is on: " << numa_node << std::dec;
98 0 : }
99 0 : TLOG() << " -> Done.";
100 : #else
101 : TLOG() << " -> NUMA support is turned off build-time. Passed on NUMA demo.";
102 : #endif
103 : }
104 :
105 0 : if (intrinsic_test) {
106 0 : TLOG() << "Intrinsic allocator test...";
107 0 : IterableQueueModel<kBlock> intrIQM(lb_capacity, false, 0, true, alignment_size);
108 0 : if (prefill) {
109 0 : TLOG() << " -> Prefilling LB...";
110 0 : intrIQM.force_pagefault();
111 : }
112 :
113 0 : for (std::size_t i=0; i<lb_capacity-1; ++i) { // Fill the LB
114 0 : intrIQM.write(kBlock());
115 : }
116 :
117 0 : TLOG() << " -> Done.";
118 0 : }
119 :
120 0 : if (aligned_test) {
121 0 : TLOG() << "Aligned allocator test...";
122 0 : IterableQueueModel<kBlock> alignedIQM(lb_capacity, false, 0, false, alignment_size);
123 0 : if (prefill) {
124 0 : TLOG() << " -> Prefilling LB...";
125 0 : alignedIQM.force_pagefault();
126 : }
127 :
128 0 : for (std::size_t i=0; i<lb_capacity-1; ++i) { // Fill the LB
129 0 : alignedIQM.write(kBlock());
130 : }
131 :
132 0 : TLOG() << " -> Done.";
133 0 : }
134 :
135 : // Test app ends. Below there are general producer/consumer/stats threads.
136 : return 0;
137 :
138 : // Stats
139 : auto stats = std::thread([&]() {
140 : TLOG() << "Spawned stats thread...";
141 : while (marker) {
142 : TLOG() << "ops/s -> " << newops.exchange(0);
143 : std::this_thread::sleep_for(std::chrono::seconds(1));
144 : }
145 : });
146 :
147 : // Producer
148 : auto producer = std::thread([&]() {
149 : TLOG() << "Spawned producer thread...";
150 : while (marker) {
151 : std::this_thread::sleep_for(std::chrono::seconds(1));
152 : }
153 : });
154 :
155 : // Consumer
156 : auto consumer = std::thread([&]() {
157 : TLOG() << "Spawned consumer thread...";
158 : while (marker) {
159 : std::this_thread::sleep_for(std::chrono::seconds(1));
160 : }
161 : });
162 :
163 : // Killswitch that flips the run marker
164 : auto killswitch = std::thread([&]() {
165 : TLOG() << "Application will terminate in 5s...";
166 : std::this_thread::sleep_for(std::chrono::seconds(runsecs));
167 : marker.store(false);
168 : });
169 :
170 : // Join local threads
171 : TLOG() << "Flipping killswitch in order to stop...";
172 : if (killswitch.joinable()) {
173 : killswitch.join();
174 : }
175 :
176 : // Exit
177 : TLOG() << "Exiting.";
178 : return 0;
179 0 : }
|
