Line data Source code
1 : // DUNE DAQ modification notice:
2 : // This file has been modified from the original ATLAS oks source for the DUNE DAQ project.
3 : // Fork baseline commit: oks-08-03-04 (2022-04-14).
4 : // Renamed since fork: yes (from bin/oks_validate_repository.cpp to apps/oks_validate_repository.cxx).
5 :
6 : /**
7 : * \file oks_validate_repository.cpp
8 : *
9 : * This file is part of the OKS package.
10 : * Author: <Igor.Soloviev@cern.ch>
11 : */
12 :
13 : #include "oks/exceptions.hpp"
14 : #include "oks/kernel.hpp"
15 : #include "oks/pipeline.hpp"
16 :
17 : #include <boost/program_options.hpp>
18 :
19 : #include "ers/ers.hpp"
20 : #include "logging/Logging.hpp"
21 :
22 : #include <algorithm>
23 : #include <chrono>
24 : #include <filesystem>
25 : #include <iostream>
26 : #include <mutex>
27 : #include <vector>
28 :
29 : using namespace dunedaq::oks;
30 :
31 : enum __OksValidateRepositoryExitStatus__
32 : {
33 : __Success__ = 0,
34 : __BadCommandLine__,
35 : __UserAuthenticationFailure__,
36 : __NoRepository__,
37 : __ConsistencyError__,
38 : __IncludesCircularDependencyError__,
39 : __AccessManagerAuthorizationFailed__,
40 : __AccessManagerNoPermission__,
41 : __NoIncludedFile__,
42 : __ExceptionCaught__
43 : };
44 :
45 : std::string s_load_error;
46 :
47 : static void
48 0 : init_file_load_error(const std::string& file)
49 : {
50 0 : s_load_error = "repository validation failed for file \'";
51 0 : s_load_error += file;
52 0 : s_load_error += "\':\n";
53 0 : }
54 :
55 : struct OksValidateJob : public OksJob
56 : {
57 : public:
58 0 : OksValidateJob(OksKernel& kernel, const std::string& file_name)
59 0 : : m_kernel(kernel)
60 0 : , m_file_name(file_name)
61 : {
62 0 : ;
63 0 : }
64 :
65 0 : void run()
66 : {
67 0 : static std::mutex s_mutex;
68 :
69 0 : try {
70 0 : auto start_usage = std::chrono::steady_clock::now();
71 :
72 0 : m_kernel.set_silence_mode(true);
73 :
74 0 : m_kernel.load_file(m_file_name);
75 :
76 0 : if (!m_kernel.get_bind_classes_status().empty() || !m_kernel.get_bind_objects_status().empty()) {
77 0 : std::lock_guard lock(s_mutex);
78 :
79 0 : if (s_load_error.empty()) {
80 0 : init_file_load_error(m_file_name);
81 :
82 0 : if (!m_kernel.get_bind_classes_status().empty()) {
83 0 : s_load_error += "the schema contains dangling references to non-loaded classes:\n";
84 0 : s_load_error += m_kernel.get_bind_classes_status();
85 : }
86 :
87 0 : if (!m_kernel.get_bind_objects_status().empty()) {
88 0 : s_load_error += "the data contain dangling references to non-loaded objects:\n";
89 0 : s_load_error += m_kernel.get_bind_objects_status();
90 : }
91 : }
92 0 : }
93 :
94 0 : static std::mutex s_log_mutex;
95 0 : std::lock_guard scoped_lock(s_log_mutex);
96 :
97 0 : log_timestamp() << "validated file \"" << m_file_name << "\" in "
98 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
99 0 : start_usage)
100 0 : .count() /
101 0 : 1000.
102 0 : << " ms\n";
103 0 : } catch (std::exception& ex) {
104 0 : std::lock_guard lock(s_mutex);
105 :
106 0 : if (s_load_error.empty()) {
107 0 : init_file_load_error(m_file_name);
108 0 : s_load_error += ex.what();
109 :
110 0 : const std::string& user_repository(m_kernel.get_user_repository_root());
111 0 : const std::size_t user_repository_len(m_kernel.get_user_repository_root().length() + 1);
112 :
113 0 : std::size_t pos;
114 0 : while ((pos = s_load_error.find(user_repository)) != std::string::npos)
115 0 : s_load_error.replace(pos, user_repository_len, "");
116 : }
117 0 : }
118 0 : }
119 :
120 : private:
121 : OksKernel m_kernel;
122 : const std::string& m_file_name;
123 :
124 : // protect usage of copy constructor and assignment operator
125 :
126 : private:
127 : OksValidateJob(const OksValidateJob&);
128 : OksValidateJob& operator=(const OksValidateJob&);
129 : };
130 :
131 : struct FoundCircularDependency
132 : {
133 : unsigned int m_count;
134 : std::ostringstream m_text;
135 :
136 0 : FoundCircularDependency()
137 0 : : m_count(0)
138 : {
139 0 : ;
140 0 : }
141 : } s_circular_dependency_message;
142 :
143 : struct TestCircularDependency
144 : {
145 0 : TestCircularDependency(const std::string* file)
146 0 : {
147 0 : p_set_includes.insert(file);
148 0 : p_vector_includes.push_back(file);
149 0 : }
150 :
151 0 : bool push(const std::string* file)
152 : {
153 0 : auto it = p_set_includes.insert(file);
154 0 : if (it.second == false) {
155 0 : std::ostringstream s;
156 :
157 0 : bool report = false;
158 :
159 0 : for (const auto& x : p_vector_includes) {
160 0 : if (x == *it.first) {
161 0 : s_circular_dependency_message.m_text << "\nCircular dependency [" << ++s_circular_dependency_message.m_count
162 0 : << "]:";
163 : report = true;
164 : }
165 :
166 0 : if (report)
167 0 : s_circular_dependency_message.m_text << '\n' << " - \"" << *x << "\"";
168 : }
169 :
170 0 : return false;
171 0 : }
172 :
173 0 : p_vector_includes.push_back(file);
174 :
175 : return true;
176 : }
177 :
178 0 : void pop()
179 : {
180 0 : p_set_includes.erase(p_vector_includes.back());
181 0 : p_vector_includes.pop_back();
182 0 : }
183 :
184 : std::vector<const std::string*> p_vector_includes;
185 : std::set<const std::string*, OksFile::SortByName> p_set_includes;
186 : };
187 :
188 : std::set<std::string>&
189 0 : define_includes(const std::string& f,
190 : const std::set<std::string>& s,
191 : std::map<std::string, std::set<std::string>>& file_all_includes,
192 : std::map<std::string, std::set<std::string>>& file_explicit_includes,
193 : TestCircularDependency& cd_fuse)
194 : {
195 0 : std::set<std::string>& all_includes = file_all_includes[f];
196 :
197 0 : if (all_includes.empty()) {
198 0 : for (auto& x : s) {
199 0 : if (cd_fuse.push(&x)) {
200 0 : all_includes.insert(x);
201 :
202 0 : std::set<std::string>& includes =
203 0 : define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
204 0 : for (const auto& y : includes)
205 0 : all_includes.insert(y);
206 :
207 0 : cd_fuse.pop();
208 : }
209 : }
210 : }
211 :
212 0 : return all_includes;
213 : }
214 :
215 : int
216 0 : main(int argc, char** argv)
217 : {
218 0 : boost::program_options::options_description desc(
219 0 : "This program validates OKS git repository for commit by pre-receive hook");
220 :
221 0 : std::vector<std::string> created, updated, deleted;
222 0 : bool circular_dependency_between_includes_is_error = true;
223 0 : bool verbose = false;
224 0 : std::string user;
225 0 : std::size_t pipeline_size = 4;
226 :
227 0 : try {
228 0 : std::vector<std::string> app_types_list;
229 0 : std::vector<std::string> segments_list;
230 :
231 0 : desc.add_options()("add,a",
232 0 : boost::program_options::value<std::vector<std::string>>(&created)->multitoken(),
233 0 : "list of new OKS files and directories to be added to the repository")(
234 : "update,u",
235 0 : boost::program_options::value<std::vector<std::string>>(&updated)->multitoken(),
236 0 : "list of new OKS files and directories to be updated in the repository")(
237 : "remove,r",
238 0 : boost::program_options::value<std::vector<std::string>>(&deleted)->multitoken(),
239 0 : "list of new OKS files and directories to be removed from the repository")(
240 : "permissive-circular-dependencies-between-includes,C",
241 0 : "downgrade severity of detected circular dependencies between includes from errors to warnings")(
242 0 : "user,U", boost::program_options::value<std::string>(&user), "user id")(
243 : "threads-number,t",
244 0 : boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size),
245 0 : "number of threads used by validation pipeline")("verbose,v", "Print debug information")("help,h",
246 : "Print help message");
247 :
248 0 : boost::program_options::variables_map vm;
249 0 : boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
250 :
251 0 : if (vm.count("help")) {
252 0 : std::cout << desc << std::endl;
253 0 : return __Success__;
254 : }
255 :
256 0 : if (vm.count("permissive-circular-dependencies-between-includes"))
257 0 : circular_dependency_between_includes_is_error = false;
258 :
259 0 : if (vm.count("verbose"))
260 0 : verbose = true;
261 :
262 0 : boost::program_options::notify(vm);
263 :
264 0 : } catch (std::exception& ex) {
265 0 : std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
266 0 : return __BadCommandLine__;
267 0 : }
268 :
269 0 : try {
270 0 : OksKernel kernel;
271 :
272 0 : kernel.set_allow_duplicated_objects_mode(false);
273 0 : kernel.set_test_duplicated_objects_via_inheritance_mode(true);
274 :
275 0 : if (kernel.get_user_repository_root().empty()) {
276 0 : std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
277 : return __NoRepository__;
278 : }
279 :
280 0 : std::filesystem::current_path(kernel.get_user_repository_root());
281 :
282 0 : auto start_usage = std::chrono::steady_clock::now();
283 :
284 : // directories
285 :
286 0 : std::set<std::string> directories;
287 :
288 : // file: explicit includes
289 0 : std::map<std::string, std::set<std::string>> file_explicit_includes;
290 :
291 0 : for (auto& p : std::filesystem::recursive_directory_iterator("."))
292 0 : if (std::filesystem::is_directory(p))
293 0 : directories.insert(p.path().native().substr(2));
294 0 : else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 &&
295 0 : p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
296 0 : kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
297 :
298 0 : if (verbose)
299 0 : log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in "
300 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
301 0 : start_usage)
302 0 : .count() /
303 0 : 1000.
304 0 : << " ms\n";
305 :
306 0 : auto start_usage2 = std::chrono::steady_clock::now();
307 :
308 0 : std::set<std::string> all_includes;
309 :
310 : // check every include exists
311 0 : for (const auto& f : file_explicit_includes) {
312 0 : for (const auto& i : f.second) {
313 0 : if (file_explicit_includes.find(i) != file_explicit_includes.end()) {
314 0 : all_includes.insert(i);
315 : } else {
316 0 : std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
317 0 : return __NoIncludedFile__;
318 : }
319 : }
320 : }
321 :
322 0 : if (verbose)
323 0 : log_timestamp(Debug) << "check existence of includes in "
324 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
325 0 : start_usage2)
326 0 : .count() /
327 0 : 1000.
328 0 : << " ms\n";
329 :
330 0 : auto start_usage3 = std::chrono::steady_clock::now();
331 :
332 : // file: all includes
333 0 : std::map<std::string, std::set<std::string>> file_all_includes;
334 :
335 0 : for (auto& x : file_explicit_includes) {
336 0 : TestCircularDependency cd_fuse(&x.first);
337 0 : define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
338 0 : }
339 :
340 0 : auto stop_usage3 = std::chrono::steady_clock::now();
341 :
342 0 : if (verbose)
343 0 : log_timestamp(Debug)
344 0 : << "calculated inclusion graph in "
345 0 : << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage3).count() / 1000. << " ms\n";
346 :
347 0 : log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in "
348 0 : << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage).count() / 1000.
349 0 : << " ms" << std::endl;
350 :
351 0 : if (s_circular_dependency_message.m_count) {
352 0 : log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning))
353 0 : << "Detected " << s_circular_dependency_message.m_count
354 : << " circular dependencies between includes of the repository files:"
355 0 : << s_circular_dependency_message.m_text.str() << std::endl;
356 :
357 0 : if (circular_dependency_between_includes_is_error == true)
358 : return __IncludesCircularDependencyError__;
359 : }
360 :
361 0 : if (ers::debug_level() >= 2) {
362 0 : std::ostringstream text;
363 0 : text << "ALL INCLUDES:\n";
364 :
365 0 : for (const auto& x : file_all_includes) {
366 0 : text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
367 :
368 0 : for (const auto& y : x.second)
369 0 : text << " - \"" << y << "\"\n";
370 : }
371 :
372 0 : TLOG_DEBUG(2) << text.str();
373 0 : }
374 :
375 0 : OksPipeline pipeline(pipeline_size);
376 :
377 : // do not run check for README file
378 0 : auto ignore_files = [](std::string& x) {
379 0 : static const std::string readme_file("README.md");
380 0 : return x != readme_file;
381 : };
382 :
383 : // all modified file paths
384 0 : std::set<std::string> modified;
385 0 : std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
386 0 : std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
387 :
388 : // validate independently every created or updated file
389 0 : for (const auto& x : modified)
390 0 : pipeline.addJob(new OksValidateJob(kernel, x));
391 :
392 0 : std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
393 :
394 0 : for (const auto& f : file_explicit_includes)
395 0 : if (all_includes.find(f.first) == all_includes.end()) {
396 0 : if (modified.empty() == false) {
397 0 : if (modified.find(f.first) == modified.end()) {
398 0 : const auto& file_includes = file_all_includes[f.first];
399 :
400 0 : bool found = false;
401 :
402 0 : for (const auto& x : modified)
403 0 : if (file_includes.find(x) != file_includes.end()) {
404 0 : found = true;
405 0 : TLOG_DEBUG(1) << "file \"" << f.first << "\" contains modified include \"" << x << '\"';
406 0 : break;
407 : }
408 :
409 0 : if (found == false) {
410 0 : TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
411 0 : continue;
412 0 : }
413 : } else {
414 0 : TLOG_DEBUG(1) << "list of modified files contains file \"" << f.first << '\"';
415 : }
416 : }
417 :
418 0 : if (modified.find(f.first) == modified.end())
419 0 : pipeline.addJob(new OksValidateJob(kernel, f.first));
420 : }
421 :
422 0 : pipeline.waitForCompletion();
423 :
424 0 : if (!s_load_error.empty()) {
425 0 : log_timestamp(Error) << s_load_error << std::endl;
426 0 : return __ConsistencyError__;
427 : }
428 0 : } catch (exception& ex) {
429 0 : log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
430 0 : return __ExceptionCaught__;
431 0 : } catch (std::exception& e) {
432 0 : log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
433 0 : return __ExceptionCaught__;
434 0 : } catch (...) {
435 0 : log_timestamp(Error) << "Caught unknown exception" << std::endl;
436 0 : return __ExceptionCaught__;
437 0 : }
438 :
439 0 : return __Success__;
440 0 : }
|
