Line data Source code
1 : // DUNE DAQ modification notice:
2 : // This file has been modified from the original ATLAS oks source for the DUNE DAQ project.
3 : // Fork baseline commit: oks-08-03-04 (2022-04-14).
4 : // Renamed since fork: yes (from bin/oks_validate_repository.cpp to apps/oks_validate_repository.cxx).
5 :
6 : /**
7 : * \file oks_validate_repository.cpp
8 : *
9 : * This file is part of the OKS package.
10 : * Author: <Igor.Soloviev@cern.ch>
11 : */
12 :
13 :
14 : #include "oks/kernel.hpp"
15 : #include "oks/pipeline.hpp"
16 : #include "oks/exceptions.hpp"
17 :
18 : #include <boost/program_options.hpp>
19 :
20 : #include "ers/ers.hpp"
21 : #include "logging/Logging.hpp"
22 :
23 : #include <algorithm>
24 : #include <chrono>
25 : #include <vector>
26 : #include <iostream>
27 : #include <filesystem>
28 : #include <mutex>
29 :
30 : using namespace dunedaq::oks;
31 :
32 : enum __OksValidateRepositoryExitStatus__ {
33 : __Success__ = 0,
34 : __BadCommandLine__,
35 : __UserAuthenticationFailure__,
36 : __NoRepository__,
37 : __ConsistencyError__,
38 : __IncludesCircularDependencyError__,
39 : __AccessManagerAuthorizationFailed__,
40 : __AccessManagerNoPermission__,
41 : __NoIncludedFile__,
42 : __ExceptionCaught__
43 : };
44 :
45 :
46 : std::string s_load_error;
47 :
48 : static void
49 0 : init_file_load_error(const std::string& file)
50 : {
51 0 : s_load_error = "repository validation failed for file \'";
52 0 : s_load_error += file;
53 0 : s_load_error += "\':\n";
54 0 : }
55 :
56 : struct OksValidateJob : public OksJob
57 : {
58 : public:
59 :
60 0 : OksValidateJob(OksKernel& kernel, const std::string& file_name) :
61 0 : m_kernel(kernel), m_file_name(file_name)
62 : {
63 0 : ;
64 0 : }
65 :
66 : void
67 0 : run()
68 : {
69 0 : static std::mutex s_mutex;
70 :
71 0 : try
72 : {
73 0 : auto start_usage = std::chrono::steady_clock::now();
74 :
75 0 : m_kernel.set_silence_mode(true);
76 :
77 0 : m_kernel.load_file(m_file_name);
78 :
79 0 : if (!m_kernel.get_bind_classes_status().empty() || !m_kernel.get_bind_objects_status().empty())
80 : {
81 0 : std::lock_guard lock(s_mutex);
82 :
83 0 : if (s_load_error.empty())
84 : {
85 0 : init_file_load_error(m_file_name);
86 :
87 0 : if (!m_kernel.get_bind_classes_status().empty())
88 : {
89 0 : s_load_error += "the schema contains dangling references to non-loaded classes:\n";
90 0 : s_load_error += m_kernel.get_bind_classes_status();
91 : }
92 :
93 0 : if (!m_kernel.get_bind_objects_status().empty())
94 : {
95 0 : s_load_error += "the data contain dangling references to non-loaded objects:\n";
96 0 : s_load_error += m_kernel.get_bind_objects_status();
97 : }
98 : }
99 0 : }
100 :
101 0 : static std::mutex s_log_mutex;
102 0 : std::lock_guard scoped_lock(s_log_mutex);
103 :
104 0 : log_timestamp() << "validated file \"" << m_file_name << "\" in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage).count() / 1000. << " ms\n";
105 0 : }
106 0 : catch (std::exception& ex)
107 : {
108 0 : std::lock_guard lock(s_mutex);
109 :
110 0 : if (s_load_error.empty())
111 : {
112 0 : init_file_load_error(m_file_name);
113 0 : s_load_error += ex.what();
114 :
115 0 : const std::string& user_repository(m_kernel.get_user_repository_root());
116 0 : const std::size_t user_repository_len(m_kernel.get_user_repository_root().length() + 1);
117 :
118 0 : std::size_t pos;
119 0 : while ((pos = s_load_error.find(user_repository)) != std::string::npos)
120 0 : s_load_error.replace(pos, user_repository_len, "");
121 : }
122 0 : }
123 0 : }
124 :
125 : private:
126 :
127 : OksKernel m_kernel;
128 : const std::string& m_file_name;
129 :
130 : // protect usage of copy constructor and assignment operator
131 :
132 : private:
133 :
134 : OksValidateJob(const OksValidateJob&);
135 : OksValidateJob&
136 : operator=(const OksValidateJob&);
137 :
138 : };
139 :
140 :
141 : struct FoundCircularDependency
142 : {
143 : unsigned int m_count;
144 : std::ostringstream m_text;
145 :
146 0 : FoundCircularDependency() :
147 0 : m_count(0)
148 : {
149 0 : ;
150 0 : }
151 : } s_circular_dependency_message;
152 :
153 :
154 : struct TestCircularDependency
155 : {
156 0 : TestCircularDependency(const std::string * file)
157 0 : {
158 0 : p_set_includes.insert(file);
159 0 : p_vector_includes.push_back(file);
160 0 : }
161 :
162 : bool
163 0 : push(const std::string * file)
164 : {
165 0 : auto it = p_set_includes.insert(file);
166 0 : if (it.second == false)
167 : {
168 0 : std::ostringstream s;
169 :
170 0 : bool report = false;
171 :
172 0 : for (const auto& x : p_vector_includes)
173 : {
174 0 : if (x == *it.first)
175 : {
176 0 : s_circular_dependency_message.m_text << "\nCircular dependency [" << ++s_circular_dependency_message.m_count << "]:";
177 : report = true;
178 : }
179 :
180 0 : if (report)
181 0 : s_circular_dependency_message.m_text << '\n' << " - \"" << *x << "\"";
182 : }
183 :
184 0 : return false;
185 0 : }
186 :
187 0 : p_vector_includes.push_back(file);
188 :
189 : return true;
190 : }
191 :
192 : void
193 0 : pop()
194 : {
195 0 : p_set_includes.erase(p_vector_includes.back());
196 0 : p_vector_includes.pop_back();
197 0 : }
198 :
199 : std::vector<const std::string *> p_vector_includes;
200 : std::set<const std::string *, OksFile::SortByName> p_set_includes;
201 : };
202 :
203 :
204 : std::set<std::string>&
205 0 : define_includes(const std::string& f, const std::set<std::string>& s, std::map<std::string, std::set<std::string>>& file_all_includes, std::map<std::string, std::set<std::string>>& file_explicit_includes, TestCircularDependency& cd_fuse)
206 : {
207 0 : std::set<std::string>& all_includes = file_all_includes[f];
208 :
209 0 : if(all_includes.empty())
210 : {
211 0 : for(auto& x : s)
212 : {
213 0 : if(cd_fuse.push(&x))
214 : {
215 0 : all_includes.insert(x);
216 :
217 0 : std::set<std::string>& includes = define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
218 0 : for(const auto& y : includes)
219 0 : all_includes.insert(y);
220 :
221 0 : cd_fuse.pop();
222 : }
223 : }
224 : }
225 :
226 0 : return all_includes;
227 : }
228 :
229 :
230 : int
231 0 : main(int argc, char **argv)
232 : {
233 0 : boost::program_options::options_description desc("This program validates OKS git repository for commit by pre-receive hook");
234 :
235 0 : std::vector<std::string> created, updated, deleted;
236 0 : bool circular_dependency_between_includes_is_error = true;
237 0 : bool verbose = false;
238 0 : std::string user;
239 0 : std::size_t pipeline_size = 4;
240 :
241 0 : try
242 : {
243 0 : std::vector<std::string> app_types_list;
244 0 : std::vector<std::string> segments_list;
245 :
246 0 : desc.add_options()
247 0 : ("add,a", boost::program_options::value<std::vector<std::string> >(&created)->multitoken(), "list of new OKS files and directories to be added to the repository")
248 0 : ("update,u", boost::program_options::value<std::vector<std::string> >(&updated)->multitoken(), "list of new OKS files and directories to be updated in the repository")
249 0 : ("remove,r", boost::program_options::value<std::vector<std::string> >(&deleted)->multitoken(), "list of new OKS files and directories to be removed from the repository")
250 0 : ("permissive-circular-dependencies-between-includes,C", "downgrade severity of detected circular dependencies between includes from errors to warnings")
251 0 : ("user,U", boost::program_options::value<std::string>(&user), "user id")
252 0 : ("threads-number,t", boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size), "number of threads used by validation pipeline")
253 0 : ("verbose,v", "Print debug information")
254 0 : ("help,h", "Print help message");
255 :
256 0 : boost::program_options::variables_map vm;
257 0 : boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
258 :
259 0 : if (vm.count("help"))
260 : {
261 0 : std::cout << desc << std::endl;
262 0 : return __Success__;
263 : }
264 :
265 0 : if (vm.count("permissive-circular-dependencies-between-includes"))
266 0 : circular_dependency_between_includes_is_error = false;
267 :
268 0 : if (vm.count("verbose"))
269 0 : verbose = true;
270 :
271 0 : boost::program_options::notify(vm);
272 :
273 0 : }
274 0 : catch (std::exception& ex)
275 : {
276 0 : std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
277 0 : return __BadCommandLine__;
278 0 : }
279 :
280 :
281 0 : try
282 : {
283 0 : OksKernel kernel;
284 :
285 0 : kernel.set_allow_duplicated_objects_mode(false);
286 0 : kernel.set_test_duplicated_objects_via_inheritance_mode(true);
287 :
288 0 : if(kernel.get_user_repository_root().empty())
289 : {
290 0 : std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
291 : return __NoRepository__;
292 : }
293 :
294 0 : std::filesystem::current_path(kernel.get_user_repository_root());
295 :
296 0 : auto start_usage = std::chrono::steady_clock::now();
297 :
298 : // directories
299 :
300 0 : std::set<std::string> directories;
301 :
302 :
303 : // file: explicit includes
304 0 : std::map<std::string, std::set<std::string>> file_explicit_includes;
305 :
306 0 : for (auto& p : std::filesystem::recursive_directory_iterator("."))
307 0 : if (std::filesystem::is_directory(p))
308 0 : directories.insert(p.path().native().substr(2));
309 0 : else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 && p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
310 0 : kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
311 :
312 0 : if (verbose)
313 0 : log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage).count() / 1000. << " ms\n";
314 :
315 :
316 0 : auto start_usage2 = std::chrono::steady_clock::now();
317 :
318 0 : std::set<std::string> all_includes;
319 :
320 : // check every include exists
321 0 : for (const auto& f : file_explicit_includes)
322 : {
323 0 : for (const auto& i : f.second)
324 : {
325 0 : if(file_explicit_includes.find(i) != file_explicit_includes.end())
326 : {
327 0 : all_includes.insert(i);
328 : }
329 : else
330 : {
331 0 : std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
332 0 : return __NoIncludedFile__;
333 : }
334 : }
335 : }
336 :
337 0 : if (verbose)
338 0 : log_timestamp(Debug) << "check existence of includes in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage2).count() / 1000. << " ms\n";
339 :
340 :
341 0 : auto start_usage3 = std::chrono::steady_clock::now();
342 :
343 : // file: all includes
344 0 : std::map<std::string, std::set<std::string>> file_all_includes;
345 :
346 0 : for(auto& x : file_explicit_includes)
347 : {
348 0 : TestCircularDependency cd_fuse(&x.first);
349 0 : define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
350 0 : }
351 :
352 0 : auto stop_usage3 = std::chrono::steady_clock::now();
353 :
354 0 : if (verbose)
355 0 : log_timestamp(Debug) << "calculated inclusion graph in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage3).count() / 1000. << " ms\n";
356 :
357 0 : log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage).count() / 1000. << " ms" << std::endl;
358 :
359 :
360 0 : if (s_circular_dependency_message.m_count)
361 : {
362 0 : log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning)) << "Detected " << s_circular_dependency_message.m_count << " circular dependencies between includes of the repository files:" << s_circular_dependency_message.m_text.str() << std::endl;
363 :
364 0 : if (circular_dependency_between_includes_is_error == true)
365 : return __IncludesCircularDependencyError__;
366 : }
367 :
368 0 : if (ers::debug_level() >= 2)
369 : {
370 0 : std::ostringstream text;
371 0 : text << "ALL INCLUDES:\n";
372 :
373 0 : for (const auto& x : file_all_includes)
374 : {
375 0 : text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
376 :
377 0 : for (const auto& y : x.second)
378 0 : text << " - \"" << y << "\"\n";
379 : }
380 :
381 0 : TLOG_DEBUG(2) << text.str();
382 0 : }
383 :
384 0 : OksPipeline pipeline(pipeline_size);
385 :
386 : // do not run check for README file
387 0 : auto ignore_files = [](std::string& x)
388 : {
389 0 : static const std::string readme_file("README.md");
390 0 : return x != readme_file;
391 : };
392 :
393 : // all modified file paths
394 0 : std::set<std::string> modified;
395 0 : std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
396 0 : std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
397 :
398 : // validate independently every created or updated file
399 0 : for (const auto& x : modified)
400 0 : pipeline.addJob(new OksValidateJob(kernel, x));
401 :
402 0 : std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
403 :
404 0 : for (const auto& f : file_explicit_includes)
405 0 : if (all_includes.find(f.first) == all_includes.end())
406 : {
407 0 : if (modified.empty() == false)
408 : {
409 0 : if (modified.find(f.first) == modified.end())
410 : {
411 0 : const auto& file_includes = file_all_includes[f.first];
412 :
413 0 : bool found = false;
414 :
415 0 : for (const auto& x : modified)
416 0 : if (file_includes.find(x) != file_includes.end())
417 : {
418 0 : found = true;
419 0 : TLOG_DEBUG(1) << "file \"" << f.first << "\" contains modified include \"" << x << '\"';
420 0 : break;
421 : }
422 :
423 0 : if(found == false)
424 : {
425 0 : TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
426 0 : continue;
427 0 : }
428 : }
429 : else
430 : {
431 0 : TLOG_DEBUG(1) << "list of modified files contains file \"" << f.first << '\"';
432 : }
433 : }
434 :
435 0 : if (modified.find(f.first) == modified.end())
436 0 : pipeline.addJob(new OksValidateJob(kernel, f.first));
437 : }
438 :
439 0 : pipeline.waitForCompletion();
440 :
441 0 : if (!s_load_error.empty())
442 : {
443 0 : log_timestamp(Error) << s_load_error << std::endl;
444 0 : return __ConsistencyError__;
445 : }
446 0 : }
447 0 : catch (exception & ex)
448 : {
449 0 : log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
450 0 : return __ExceptionCaught__;
451 0 : }
452 0 : catch (std::exception & e)
453 : {
454 0 : log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
455 0 : return __ExceptionCaught__;
456 0 : }
457 0 : catch (...)
458 : {
459 0 : log_timestamp(Error) << "Caught unknown exception" << std::endl;
460 0 : return __ExceptionCaught__;
461 0 : }
462 :
463 0 : return __Success__;
464 0 : }
|
