oks_validate_repository.cxx File Reference

#include "oks/exceptions.hpp"
#include "oks/kernel.hpp"
#include "oks/pipeline.hpp"
#include <boost/program_options.hpp>
#include "ers/ers.hpp"
#include "logging/Logging.hpp"
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <iostream>
#include <mutex>
#include <vector>

Include dependency graph for oks_validate_repository.cxx:

Go to the source code of this file.

Classes
struct	OksValidateJob
struct	FoundCircularDependency
struct	TestCircularDependency

Enumerations
enum	__OksValidateRepositoryExitStatus__ {   __Success__ = 0 , __BadCommandLine__ , __UserAuthenticationFailure__ , __NoRepository__ ,   __ConsistencyError__ , __IncludesCircularDependencyError__ , __AccessManagerAuthorizationFailed__ , __AccessManagerNoPermission__ ,   __NoIncludedFile__ , __ExceptionCaught__ }

Functions
static void	init_file_load_error (const std::string &file)
std::set< std::string > &	define_includes (const std::string &f, const std::set< std::string > &s, std::map< std::string, std::set< std::string > > &file_all_includes, std::map< std::string, std::set< std::string > > &file_explicit_includes, TestCircularDependency &cd_fuse)
int	main (int argc, char **argv)

Variables
std::string	s_load_error
struct FoundCircularDependency	s_circular_dependency_message

Enumeration Type Documentation

__OksValidateRepositoryExitStatus__

enum __OksValidateRepositoryExitStatus__

Enumerator
__Success__
__BadCommandLine__
__UserAuthenticationFailure__
__NoRepository__
__ConsistencyError__
__IncludesCircularDependencyError__
__AccessManagerAuthorizationFailed__
__AccessManagerNoPermission__
__NoIncludedFile__
__ExceptionCaught__

Definition at line 31 of file oks_validate_repository.cxx.

{
  __Success__ = 0,
  __BadCommandLine__,
  __UserAuthenticationFailure__,
  __NoRepository__,
  __ConsistencyError__,
  __IncludesCircularDependencyError__,
  __AccessManagerAuthorizationFailed__,
  __AccessManagerNoPermission__,
  __NoIncludedFile__,
  __ExceptionCaught__
};

Function Documentation

define_includes()

std::set< std::string > & define_includes	(	const std::string &	f,
		const std::set< std::string > &	s,
		std::map< std::string, std::set< std::string > > &	file_all_includes,
		std::map< std::string, std::set< std::string > > &	file_explicit_includes,
		TestCircularDependency &	cd_fuse )

Definition at line 189 of file oks_validate_repository.cxx.

{
  std::set<std::string>& all_includes = file_all_includes[f];
 
  if (all_includes.empty()) {
    for (auto& x : s) {
      if (cd_fuse.push(&x)) {
        all_includes.insert(x);
 
        std::set<std::string>& includes =
          define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
        for (const auto& y : includes)
          all_includes.insert(y);
 
        cd_fuse.pop();
      }
    }
  }
 
  return all_includes;
}

init_file_load_error()

void init_file_load_error ( const std::string & file )

static

Definition at line 48 of file oks_validate_repository.cxx.

{
  s_load_error = "repository validation failed for file \'";
  s_load_error += file;
  s_load_error += "\':\n";
}

main()

int main	(	int	argc,
		char **	argv )

Definition at line 216 of file oks_validate_repository.cxx.

{
  boost::program_options::options_description desc(
    "This program validates OKS git repository for commit by pre-receive hook");
 
  std::vector<std::string> created, updated, deleted;
  bool circular_dependency_between_includes_is_error = true;
  bool verbose = false;
  std::string user;
  std::size_t pipeline_size = 4;
 
  try {
    std::vector<std::string> app_types_list;
    std::vector<std::string> segments_list;
 
    desc.add_options()("add,a",
                       boost::program_options::value<std::vector<std::string>>(&created)->multitoken(),
                       "list of new OKS files and directories to be added to the repository")(
      "update,u",
      boost::program_options::value<std::vector<std::string>>(&updated)->multitoken(),
      "list of new OKS files and directories to be updated in the repository")(
      "remove,r",
      boost::program_options::value<std::vector<std::string>>(&deleted)->multitoken(),
      "list of new OKS files and directories to be removed from the repository")(
      "permissive-circular-dependencies-between-includes,C",
      "downgrade severity of detected circular dependencies between includes from errors to warnings")(
      "user,U", boost::program_options::value<std::string>(&user), "user id")(
      "threads-number,t",
      boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size),
      "number of threads used by validation pipeline")("verbose,v", "Print debug information")("help,h",
                                                                                               "Print help message");
 
    boost::program_options::variables_map vm;
    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
 
    if (vm.count("help")) {
      std::cout << desc << std::endl;
      return __Success__;
    }
 
    if (vm.count("permissive-circular-dependencies-between-includes"))
      circular_dependency_between_includes_is_error = false;
 
    if (vm.count("verbose"))
      verbose = true;
 
    boost::program_options::notify(vm);
 
  } catch (std::exception& ex) {
    std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
    return __BadCommandLine__;
  }
 
  try {
    OksKernel kernel;
 
    kernel.set_allow_duplicated_objects_mode(false);
    kernel.set_test_duplicated_objects_via_inheritance_mode(true);
 
    if (kernel.get_user_repository_root().empty()) {
      std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
      return __NoRepository__;
    }
 
    std::filesystem::current_path(kernel.get_user_repository_root());
 
    auto start_usage = std::chrono::steady_clock::now();
 
    // directories
 
    std::set<std::string> directories;
 
    // file: explicit includes
    std::map<std::string, std::set<std::string>> file_explicit_includes;
 
    for (auto& p : std::filesystem::recursive_directory_iterator("."))
      if (std::filesystem::is_directory(p))
        directories.insert(p.path().native().substr(2));
      else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 &&
               p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
        kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
 
    if (verbose)
      log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in "
                           << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
                                                                                    start_usage)
                                  .count() /
                                1000.
                           << " ms\n";
 
    auto start_usage2 = std::chrono::steady_clock::now();
 
    std::set<std::string> all_includes;
 
    // check every include exists
    for (const auto& f : file_explicit_includes) {
      for (const auto& i : f.second) {
        if (file_explicit_includes.find(i) != file_explicit_includes.end()) {
          all_includes.insert(i);
        } else {
          std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
          return __NoIncludedFile__;
        }
      }
    }
 
    if (verbose)
      log_timestamp(Debug) << "check existence of includes in "
                           << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
                                                                                    start_usage2)
                                  .count() /
                                1000.
                           << " ms\n";
 
    auto start_usage3 = std::chrono::steady_clock::now();
 
    // file: all includes
    std::map<std::string, std::set<std::string>> file_all_includes;
 
    for (auto& x : file_explicit_includes) {
      TestCircularDependency cd_fuse(&x.first);
      define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
    }
 
    auto stop_usage3 = std::chrono::steady_clock::now();
 
    if (verbose)
      log_timestamp(Debug)
        << "calculated inclusion graph in "
        << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage3).count() / 1000. << " ms\n";
 
    log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in "
                    << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage).count() / 1000.
                    << " ms" << std::endl;
 
    if (s_circular_dependency_message.m_count) {
      log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning))
        << "Detected " << s_circular_dependency_message.m_count
        << " circular dependencies between includes of the repository files:"
        << s_circular_dependency_message.m_text.str() << std::endl;
 
      if (circular_dependency_between_includes_is_error == true)
        return __IncludesCircularDependencyError__;
    }
 
    if (ers::debug_level() >= 2) {
      std::ostringstream text;
      text << "ALL INCLUDES:\n";
 
      for (const auto& x : file_all_includes) {
        text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
 
        for (const auto& y : x.second)
          text << " - \"" << y << "\"\n";
      }
 
      TLOG_DEBUG(2) << text.str();
    }
 
    OksPipeline pipeline(pipeline_size);
 
    // do not run check for README file
    auto ignore_files = [](std::string& x) {
      static const std::string readme_file("README.md");
      return x != readme_file;
    };
 
    // all modified file paths
    std::set<std::string> modified;
    std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
    std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
 
    // validate independently every created or updated file
    for (const auto& x : modified)
      pipeline.addJob(new OksValidateJob(kernel, x));
 
    std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
 
    for (const auto& f : file_explicit_includes)
      if (all_includes.find(f.first) == all_includes.end()) {
        if (modified.empty() == false) {
          if (modified.find(f.first) == modified.end()) {
            const auto& file_includes = file_all_includes[f.first];
 
            bool found = false;
 
            for (const auto& x : modified)
              if (file_includes.find(x) != file_includes.end()) {
                found = true;
                TLOG_DEBUG(1) << "file \"" << f.first << "\" contains modified include \"" << x << '\"';
                break;
              }
 
            if (found == false) {
              TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
              continue;
            }
          } else {
            TLOG_DEBUG(1) << "list of modified files contains file \"" << f.first << '\"';
          }
        }
 
        if (modified.find(f.first) == modified.end())
          pipeline.addJob(new OksValidateJob(kernel, f.first));
      }
 
    pipeline.waitForCompletion();
 
    if (!s_load_error.empty()) {
      log_timestamp(Error) << s_load_error << std::endl;
      return __ConsistencyError__;
    }
  } catch (exception& ex) {
    log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
    return __ExceptionCaught__;
  } catch (std::exception& e) {
    log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
    return __ExceptionCaught__;
  } catch (...) {
    log_timestamp(Error) << "Caught unknown exception" << std::endl;
    return __ExceptionCaught__;
  }
 
  return __Success__;
}

Variable Documentation

s_circular_dependency_message

struct FoundCircularDependency s_circular_dependency_message

s_load_error

std::string s_load_error

Definition at line 45 of file oks_validate_repository.cxx.