Line data Source code
1 : /**
2 : * @file test_lb_allocation_app.cxx Test application for LB allocations.
3 : *
4 : * This is part of the DUNE DAQ Application Framework, copyright 2020.
5 : * Licensing/copyright details are in the COPYING file that you should have
6 : * received with this code.
7 : */
8 :
9 : #include "datahandlinglibs/concepts/RawDataProcessorConcept.hpp"
10 : #include "datahandlinglibs/models/IterableQueueModel.hpp"
11 : #include "datahandlinglibs/models/SkipListLatencyBufferModel.hpp"
12 : #include "logging/Logging.hpp"
13 :
14 : #include "CLI/App.hpp"
15 : #include "CLI/Config.hpp"
16 : #include "CLI/Formatter.hpp"
17 :
18 : #include <atomic>
19 : #include <chrono>
20 : #include <memory>
21 : #include <random>
22 : #include <string>
23 : #include <vector>
24 :
25 : #ifdef WITH_LIBNUMA_SUPPORT
26 : #include <numaif.h>
27 : #endif
28 :
29 : // #define REGISTER (*(volatile unsigned char*)0x1234)
30 :
31 : using namespace dunedaq::datahandlinglibs;
32 :
33 : namespace {
34 :
35 : struct kBlock // Dummy data type for LB test
36 : {
37 0 : kBlock() {};
38 : char data[1024];
39 : };
40 :
41 : std::size_t lb_capacity = 1000; // LB capacity
42 :
43 : bool numa_aware_test = false;
44 : int num_numa_nodes = 2;
45 : bool intrinsic_test = false;
46 : bool aligned_test = false;
47 : bool prefill = false; // Prefill the LB
48 : std::size_t alignment_size = 4096;
49 : }
50 :
51 : int
52 0 : main(int argc, char** argv)
53 : {
54 0 : int runsecs = 5;
55 :
56 : // Run marker
57 0 : std::atomic<bool> marker{ true };
58 :
59 : // Counter for ops/s
60 0 : std::atomic<int> newops = 0;
61 :
62 0 : CLI::App app{ "datahandlinglibs_test_lb_allocation" };
63 0 : app.add_option("-c", lb_capacity, "Capacity/size of latency buffer.");
64 0 : app.add_flag("--numa_aware", numa_aware_test, "Test NUMA aware allocator.");
65 0 : app.add_option("--num_numa_nodes", num_numa_nodes, "Number of NUMA nodes to test allocation on.");
66 0 : app.add_flag("--intrinsic", intrinsic_test, "Test Intrinsic allocator.");
67 0 : app.add_flag("--aligned", aligned_test, "Test aligned allocator.");
68 0 : app.add_option("--alignment_size", alignment_size, "Set alignment size. Default: 4096");
69 0 : app.add_flag("--prefill", prefill, "Interface to init");
70 0 : CLI11_PARSE(app, argc, argv);
71 :
72 0 : if (numa_aware_test) { // check if test is issued...
73 : #ifdef WITH_LIBNUMA_SUPPORT
74 0 : TLOG() << "NUMA aware allocator test...";
75 :
76 0 : for (int i = 0; i < num_numa_nodes; ++i) { // for number of nodes...
77 0 : TLOG() << " # Allocating NUMA aware LB on node " << i;
78 0 : IterableQueueModel<kBlock> numaIQM(lb_capacity, true, i, false, 0);
79 :
80 0 : if (prefill) { // Force page-fault issues?
81 0 : TLOG() << " -> Prefilling LB...";
82 0 : numaIQM.force_pagefault();
83 : }
84 :
85 0 : for (std::size_t i = 0; i < lb_capacity - 1; ++i) { // Fill the LB
86 0 : numaIQM.write(kBlock());
87 : }
88 :
89 : // Test if the elements' pointers are on correct NUMA node.
90 : // Virtual addresses might be misleading between virt. and phys. addresses due to IOMMU!
91 0 : int numa_node = -1;
92 0 : get_mempolicy(&numa_node, NULL, 0, (void*)numaIQM.front(), MPOL_F_NODE | MPOL_F_ADDR);
93 0 : if (i != numa_node) {
94 0 : TLOG() << "Discrepancy in expected NUMA node and first element residency!";
95 : }
96 0 : TLOG() << " -> NUMA " << i << " IQM front virt.addr.: " << std::hex << (void*)numaIQM.front()
97 0 : << " is on: " << numa_node << std::dec;
98 0 : get_mempolicy(&numa_node, NULL, 0, (void*)numaIQM.back(), MPOL_F_NODE | MPOL_F_ADDR);
99 0 : if (i != numa_node) {
100 0 : TLOG() << "Discrepancy in expected NUMA node and last element residency!";
101 : }
102 0 : TLOG() << " -> NUMA " << i << " IQM back virt.addr.: " << std::hex << (void*)numaIQM.back()
103 0 : << " is on: " << numa_node << std::dec;
104 0 : }
105 0 : TLOG() << " -> Done.";
106 : #else
107 : TLOG() << " -> NUMA support is turned off build-time. Passed on NUMA demo.";
108 : #endif
109 : }
110 :
111 0 : if (intrinsic_test) {
112 0 : TLOG() << "Intrinsic allocator test...";
113 0 : IterableQueueModel<kBlock> intrIQM(lb_capacity, false, 0, true, alignment_size);
114 0 : if (prefill) {
115 0 : TLOG() << " -> Prefilling LB...";
116 0 : intrIQM.force_pagefault();
117 : }
118 :
119 0 : for (std::size_t i = 0; i < lb_capacity - 1; ++i) { // Fill the LB
120 0 : intrIQM.write(kBlock());
121 : }
122 :
123 0 : TLOG() << " -> Done.";
124 0 : }
125 :
126 0 : if (aligned_test) {
127 0 : TLOG() << "Aligned allocator test...";
128 0 : IterableQueueModel<kBlock> alignedIQM(lb_capacity, false, 0, false, alignment_size);
129 0 : if (prefill) {
130 0 : TLOG() << " -> Prefilling LB...";
131 0 : alignedIQM.force_pagefault();
132 : }
133 :
134 0 : for (std::size_t i = 0; i < lb_capacity - 1; ++i) { // Fill the LB
135 0 : alignedIQM.write(kBlock());
136 : }
137 :
138 0 : TLOG() << " -> Done.";
139 0 : }
140 :
141 : // Test app ends. Below there are general producer/consumer/stats threads.
142 : return 0;
143 :
144 : // Stats
145 : auto stats = std::thread([&]() {
146 : TLOG() << "Spawned stats thread...";
147 : while (marker) {
148 : TLOG() << "ops/s -> " << newops.exchange(0);
149 : std::this_thread::sleep_for(std::chrono::seconds(1));
150 : }
151 : });
152 :
153 : // Producer
154 : auto producer = std::thread([&]() {
155 : TLOG() << "Spawned producer thread...";
156 : while (marker) {
157 : std::this_thread::sleep_for(std::chrono::seconds(1));
158 : }
159 : });
160 :
161 : // Consumer
162 : auto consumer = std::thread([&]() {
163 : TLOG() << "Spawned consumer thread...";
164 : while (marker) {
165 : std::this_thread::sleep_for(std::chrono::seconds(1));
166 : }
167 : });
168 :
169 : // Killswitch that flips the run marker
170 : auto killswitch = std::thread([&]() {
171 : TLOG() << "Application will terminate in 5s...";
172 : std::this_thread::sleep_for(std::chrono::seconds(runsecs));
173 : marker.store(false);
174 : });
175 :
176 : // Join local threads
177 : TLOG() << "Flipping killswitch in order to stop...";
178 : if (killswitch.joinable()) {
179 : killswitch.join();
180 : }
181 :
182 : // Exit
183 : TLOG() << "Exiting.";
184 : return 0;
185 0 : }
|
