Line data Source code
1 : /**
2 : * \file oks_validate_repository.cpp
3 : *
4 : * This file is part of the OKS package.
5 : * Author: <Igor.Soloviev@cern.ch>
6 : */
7 :
8 :
9 : #include "oks/kernel.hpp"
10 : #include "oks/pipeline.hpp"
11 : #include "oks/exceptions.hpp"
12 :
13 : #include <boost/program_options.hpp>
14 :
15 : #include "ers/ers.hpp"
16 : #include "logging/Logging.hpp"
17 :
18 : #include <algorithm>
19 : #include <chrono>
20 : #include <vector>
21 : #include <iostream>
22 : #include <filesystem>
23 : #include <mutex>
24 :
25 : using namespace dunedaq::oks;
26 :
27 : enum __OksValidateRepositoryExitStatus__ {
28 : __Success__ = 0,
29 : __BadCommandLine__,
30 : __UserAuthenticationFailure__,
31 : __NoRepository__,
32 : __ConsistencyError__,
33 : __IncludesCircularDependencyError__,
34 : __AccessManagerAuthorizationFailed__,
35 : __AccessManagerNoPermission__,
36 : __NoIncludedFile__,
37 : __ExceptionCaught__
38 : };
39 :
40 :
41 : std::string s_load_error;
42 :
43 : static void
44 0 : init_file_load_error(const std::string& file)
45 : {
46 0 : s_load_error = "repository validation failed for file \'";
47 0 : s_load_error += file;
48 0 : s_load_error += "\':\n";
49 0 : }
50 :
51 : struct OksValidateJob : public OksJob
52 : {
53 : public:
54 :
55 0 : OksValidateJob(OksKernel& kernel, const std::string& file_name) :
56 0 : m_kernel(kernel), m_file_name(file_name)
57 : {
58 0 : ;
59 0 : }
60 :
61 : void
62 0 : run()
63 : {
64 0 : static std::mutex s_mutex;
65 :
66 0 : try
67 : {
68 0 : auto start_usage = std::chrono::steady_clock::now();
69 :
70 0 : m_kernel.set_silence_mode(true);
71 :
72 0 : m_kernel.load_file(m_file_name);
73 :
74 0 : if (!m_kernel.get_bind_classes_status().empty() || !m_kernel.get_bind_objects_status().empty())
75 : {
76 0 : std::lock_guard lock(s_mutex);
77 :
78 0 : if (s_load_error.empty())
79 : {
80 0 : init_file_load_error(m_file_name);
81 :
82 0 : if (!m_kernel.get_bind_classes_status().empty())
83 : {
84 0 : s_load_error += "the schema contains dangling references to non-loaded classes:\n";
85 0 : s_load_error += m_kernel.get_bind_classes_status();
86 : }
87 :
88 0 : if (!m_kernel.get_bind_objects_status().empty())
89 : {
90 0 : s_load_error += "the data contain dangling references to non-loaded objects:\n";
91 0 : s_load_error += m_kernel.get_bind_objects_status();
92 : }
93 : }
94 0 : }
95 :
96 0 : static std::mutex s_log_mutex;
97 0 : std::lock_guard scoped_lock(s_log_mutex);
98 :
99 0 : log_timestamp() << "validated file \"" << m_file_name << "\" in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage).count() / 1000. << " ms\n";
100 0 : }
101 0 : catch (std::exception& ex)
102 : {
103 0 : std::lock_guard lock(s_mutex);
104 :
105 0 : if (s_load_error.empty())
106 : {
107 0 : init_file_load_error(m_file_name);
108 0 : s_load_error += ex.what();
109 :
110 0 : const std::string& user_repository(m_kernel.get_user_repository_root());
111 0 : const std::size_t user_repository_len(m_kernel.get_user_repository_root().length() + 1);
112 :
113 0 : std::size_t pos;
114 0 : while ((pos = s_load_error.find(user_repository)) != std::string::npos)
115 0 : s_load_error.replace(pos, user_repository_len, "");
116 : }
117 0 : }
118 0 : }
119 :
120 : private:
121 :
122 : OksKernel m_kernel;
123 : const std::string& m_file_name;
124 :
125 : // protect usage of copy constructor and assignment operator
126 :
127 : private:
128 :
129 : OksValidateJob(const OksValidateJob&);
130 : OksValidateJob&
131 : operator=(const OksValidateJob&);
132 :
133 : };
134 :
135 :
136 : struct FoundCircularDependency
137 : {
138 : unsigned int m_count;
139 : std::ostringstream m_text;
140 :
141 0 : FoundCircularDependency() :
142 0 : m_count(0)
143 : {
144 0 : ;
145 0 : }
146 : } s_circular_dependency_message;
147 :
148 :
149 : struct TestCircularDependency
150 : {
151 0 : TestCircularDependency(const std::string * file)
152 0 : {
153 0 : p_set_includes.insert(file);
154 0 : p_vector_includes.push_back(file);
155 0 : }
156 :
157 : bool
158 0 : push(const std::string * file)
159 : {
160 0 : auto it = p_set_includes.insert(file);
161 0 : if (it.second == false)
162 : {
163 0 : std::ostringstream s;
164 :
165 0 : bool report = false;
166 :
167 0 : for (const auto& x : p_vector_includes)
168 : {
169 0 : if (x == *it.first)
170 : {
171 0 : s_circular_dependency_message.m_text << "\nCircular dependency [" << ++s_circular_dependency_message.m_count << "]:";
172 : report = true;
173 : }
174 :
175 0 : if (report)
176 0 : s_circular_dependency_message.m_text << '\n' << " - \"" << *x << "\"";
177 : }
178 :
179 0 : return false;
180 0 : }
181 :
182 0 : p_vector_includes.push_back(file);
183 :
184 : return true;
185 : }
186 :
187 : void
188 0 : pop()
189 : {
190 0 : p_set_includes.erase(p_vector_includes.back());
191 0 : p_vector_includes.pop_back();
192 0 : }
193 :
194 : std::vector<const std::string *> p_vector_includes;
195 : std::set<const std::string *, OksFile::SortByName> p_set_includes;
196 : };
197 :
198 :
199 : std::set<std::string>&
200 0 : define_includes(const std::string& f, const std::set<std::string>& s, std::map<std::string, std::set<std::string>>& file_all_includes, std::map<std::string, std::set<std::string>>& file_explicit_includes, TestCircularDependency& cd_fuse)
201 : {
202 0 : std::set<std::string>& all_includes = file_all_includes[f];
203 :
204 0 : if(all_includes.empty())
205 : {
206 0 : for(auto& x : s)
207 : {
208 0 : if(cd_fuse.push(&x))
209 : {
210 0 : all_includes.insert(x);
211 :
212 0 : std::set<std::string>& includes = define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
213 0 : for(const auto& y : includes)
214 0 : all_includes.insert(y);
215 :
216 0 : cd_fuse.pop();
217 : }
218 : }
219 : }
220 :
221 0 : return all_includes;
222 : }
223 :
224 :
225 : int
226 0 : main(int argc, char **argv)
227 : {
228 0 : boost::program_options::options_description desc("This program validates OKS git repository for commit by pre-receive hook");
229 :
230 0 : std::vector<std::string> created, updated, deleted;
231 0 : bool circular_dependency_between_includes_is_error = true;
232 0 : bool verbose = false;
233 0 : std::string user;
234 0 : std::size_t pipeline_size = 4;
235 :
236 0 : try
237 : {
238 0 : std::vector<std::string> app_types_list;
239 0 : std::vector<std::string> segments_list;
240 :
241 0 : desc.add_options()
242 0 : ("add,a", boost::program_options::value<std::vector<std::string> >(&created)->multitoken(), "list of new OKS files and directories to be added to the repository")
243 0 : ("update,u", boost::program_options::value<std::vector<std::string> >(&updated)->multitoken(), "list of new OKS files and directories to be updated in the repository")
244 0 : ("remove,r", boost::program_options::value<std::vector<std::string> >(&deleted)->multitoken(), "list of new OKS files and directories to be removed from the repository")
245 0 : ("permissive-circular-dependencies-between-includes,C", "downgrade severity of detected circular dependencies between includes from errors to warnings")
246 0 : ("user,U", boost::program_options::value<std::string>(&user), "user id")
247 0 : ("threads-number,t", boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size), "number of threads used by validation pipeline")
248 0 : ("verbose,v", "Print debug information")
249 0 : ("help,h", "Print help message");
250 :
251 0 : boost::program_options::variables_map vm;
252 0 : boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
253 :
254 0 : if (vm.count("help"))
255 : {
256 0 : std::cout << desc << std::endl;
257 0 : return __Success__;
258 : }
259 :
260 0 : if (vm.count("permissive-circular-dependencies-between-includes"))
261 0 : circular_dependency_between_includes_is_error = false;
262 :
263 0 : if (vm.count("verbose"))
264 0 : verbose = true;
265 :
266 0 : boost::program_options::notify(vm);
267 :
268 0 : }
269 0 : catch (std::exception& ex)
270 : {
271 0 : std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
272 0 : return __BadCommandLine__;
273 0 : }
274 :
275 :
276 0 : try
277 : {
278 0 : OksKernel kernel;
279 :
280 0 : kernel.set_allow_duplicated_objects_mode(false);
281 0 : kernel.set_test_duplicated_objects_via_inheritance_mode(true);
282 :
283 0 : if(kernel.get_user_repository_root().empty())
284 : {
285 0 : std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
286 : return __NoRepository__;
287 : }
288 :
289 0 : std::filesystem::current_path(kernel.get_user_repository_root());
290 :
291 0 : auto start_usage = std::chrono::steady_clock::now();
292 :
293 : // directories
294 :
295 0 : std::set<std::string> directories;
296 :
297 :
298 : // file: explicit includes
299 0 : std::map<std::string, std::set<std::string>> file_explicit_includes;
300 :
301 0 : for (auto& p : std::filesystem::recursive_directory_iterator("."))
302 0 : if (std::filesystem::is_directory(p))
303 0 : directories.insert(p.path().native().substr(2));
304 0 : else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 && p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
305 0 : kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
306 :
307 0 : if (verbose)
308 0 : log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage).count() / 1000. << " ms\n";
309 :
310 :
311 0 : auto start_usage2 = std::chrono::steady_clock::now();
312 :
313 0 : std::set<std::string> all_includes;
314 :
315 : // check every include exists
316 0 : for (const auto& f : file_explicit_includes)
317 : {
318 0 : for (const auto& i : f.second)
319 : {
320 0 : if(file_explicit_includes.find(i) != file_explicit_includes.end())
321 : {
322 0 : all_includes.insert(i);
323 : }
324 : else
325 : {
326 0 : std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
327 0 : return __NoIncludedFile__;
328 : }
329 : }
330 : }
331 :
332 0 : if (verbose)
333 0 : log_timestamp(Debug) << "check existence of includes in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage2).count() / 1000. << " ms\n";
334 :
335 :
336 0 : auto start_usage3 = std::chrono::steady_clock::now();
337 :
338 : // file: all includes
339 0 : std::map<std::string, std::set<std::string>> file_all_includes;
340 :
341 0 : for(auto& x : file_explicit_includes)
342 : {
343 0 : TestCircularDependency cd_fuse(&x.first);
344 0 : define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
345 0 : }
346 :
347 0 : auto stop_usage3 = std::chrono::steady_clock::now();
348 :
349 0 : if (verbose)
350 0 : log_timestamp(Debug) << "calculated inclusion graph in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage3).count() / 1000. << " ms\n";
351 :
352 0 : log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage).count() / 1000. << " ms" << std::endl;
353 :
354 :
355 0 : if (s_circular_dependency_message.m_count)
356 : {
357 0 : log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning)) << "Detected " << s_circular_dependency_message.m_count << " circular dependencies between includes of the repository files:" << s_circular_dependency_message.m_text.str() << std::endl;
358 :
359 0 : if (circular_dependency_between_includes_is_error == true)
360 : return __IncludesCircularDependencyError__;
361 : }
362 :
363 0 : if (ers::debug_level() >= 2)
364 : {
365 0 : std::ostringstream text;
366 0 : text << "ALL INCLUDES:\n";
367 :
368 0 : for (const auto& x : file_all_includes)
369 : {
370 0 : text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
371 :
372 0 : for (const auto& y : x.second)
373 0 : text << " - \"" << y << "\"\n";
374 : }
375 :
376 0 : TLOG_DEBUG(2) << text.str();
377 0 : }
378 :
379 0 : OksPipeline pipeline(pipeline_size);
380 :
381 : // do not run check for README file
382 0 : auto ignore_files = [](std::string& x)
383 : {
384 0 : static const std::string readme_file("README.md");
385 0 : return x != readme_file;
386 : };
387 :
388 : // all modified file paths
389 0 : std::set<std::string> modified;
390 0 : std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
391 0 : std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
392 :
393 : // validate independently every created or updated file
394 0 : for (const auto& x : modified)
395 0 : pipeline.addJob(new OksValidateJob(kernel, x));
396 :
397 0 : std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
398 :
399 0 : for (const auto& f : file_explicit_includes)
400 0 : if (all_includes.find(f.first) == all_includes.end())
401 : {
402 0 : if (modified.empty() == false)
403 : {
404 0 : if (modified.find(f.first) == modified.end())
405 : {
406 0 : const auto& file_includes = file_all_includes[f.first];
407 :
408 0 : bool found = false;
409 :
410 0 : for (const auto& x : modified)
411 0 : if (file_includes.find(x) != file_includes.end())
412 : {
413 0 : found = true;
414 0 : TLOG_DEBUG(1) << "file \"" << f.first << "\" contains modified include \"" << x << '\"';
415 0 : break;
416 : }
417 :
418 0 : if(found == false)
419 : {
420 0 : TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
421 0 : continue;
422 0 : }
423 : }
424 : else
425 : {
426 0 : TLOG_DEBUG(1) << "list of modified files contains file \"" << f.first << '\"';
427 : }
428 : }
429 :
430 0 : if (modified.find(f.first) == modified.end())
431 0 : pipeline.addJob(new OksValidateJob(kernel, f.first));
432 : }
433 :
434 0 : pipeline.waitForCompletion();
435 :
436 0 : if (!s_load_error.empty())
437 : {
438 0 : log_timestamp(Error) << s_load_error << std::endl;
439 0 : return __ConsistencyError__;
440 : }
441 0 : }
442 0 : catch (exception & ex)
443 : {
444 0 : log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
445 0 : return __ExceptionCaught__;
446 0 : }
447 0 : catch (std::exception & e)
448 : {
449 0 : log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
450 0 : return __ExceptionCaught__;
451 0 : }
452 0 : catch (...)
453 : {
454 0 : log_timestamp(Error) << "Caught unknown exception" << std::endl;
455 0 : return __ExceptionCaught__;
456 0 : }
457 :
458 0 : return __Success__;
459 0 : }
|
