oks_validate_repository.cxx File Reference

#include "oks/kernel.hpp"
#include "oks/pipeline.hpp"
#include "oks/exceptions.hpp"
#include <boost/program_options.hpp>
#include "ers/ers.hpp"
#include "logging/Logging.hpp"
#include <algorithm>
#include <chrono>
#include <vector>
#include <iostream>
#include <filesystem>
#include <mutex>

Include dependency graph for oks_validate_repository.cxx:

Go to the source code of this file.

Classes
struct	OksValidateJob
struct	FoundCircularDependency
struct	TestCircularDependency

Enumerations
enum	__OksValidateRepositoryExitStatus__ {   __Success__ = 0 , __BadCommandLine__ , __UserAuthenticationFailure__ , __NoRepository__ ,   __ConsistencyError__ , __IncludesCircularDependencyError__ , __AccessManagerAuthorizationFailed__ , __AccessManagerNoPermission__ ,   __NoIncludedFile__ , __ExceptionCaught__ }

Functions
static void	init_file_load_error (const std::string &file)
std::set< std::string > &	define_includes (const std::string &f, const std::set< std::string > &s, std::map< std::string, std::set< std::string > > &file_all_includes, std::map< std::string, std::set< std::string > > &file_explicit_includes, TestCircularDependency &cd_fuse)
int	main (int argc, char **argv)

Variables
std::string	s_load_error
struct FoundCircularDependency	s_circular_dependency_message

Enumeration Type Documentation

__OksValidateRepositoryExitStatus__

enum __OksValidateRepositoryExitStatus__

Enumerator
__Success__
__BadCommandLine__
__UserAuthenticationFailure__
__NoRepository__
__ConsistencyError__
__IncludesCircularDependencyError__
__AccessManagerAuthorizationFailed__
__AccessManagerNoPermission__
__NoIncludedFile__
__ExceptionCaught__

Definition at line 27 of file oks_validate_repository.cxx.

                                         {
  __Success__ = 0,
  __BadCommandLine__,
  __UserAuthenticationFailure__,
  __NoRepository__,
  __ConsistencyError__,
  __IncludesCircularDependencyError__,
  __AccessManagerAuthorizationFailed__,
  __AccessManagerNoPermission__,
  __NoIncludedFile__,
  __ExceptionCaught__
};

Function Documentation

define_includes()

std::set< std::string > & define_includes	(	const std::string &	f,
		const std::set< std::string > &	s,
		std::map< std::string, std::set< std::string > > &	file_all_includes,
		std::map< std::string, std::set< std::string > > &	file_explicit_includes,
		TestCircularDependency &	cd_fuse )

Definition at line 200 of file oks_validate_repository.cxx.

{
  std::set<std::string>& all_includes = file_all_includes[f];
 
  if(all_includes.empty())
    {
      for(auto& x : s)
        {
          if(cd_fuse.push(&x))
            {
              all_includes.insert(x);
 
              std::set<std::string>& includes = define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
              for(const auto& y : includes)
                all_includes.insert(y);
 
              cd_fuse.pop();
            }
        }
    }
 
  return all_includes;
}

init_file_load_error()

static void init_file_load_error ( const std::string & file )

static

Definition at line 44 of file oks_validate_repository.cxx.

{
  s_load_error = "repository validation failed for file \'";
  s_load_error += file;
  s_load_error += "\':\n";
}

main()

int main	(	int	argc,
		char **	argv )

Definition at line 226 of file oks_validate_repository.cxx.

{
  boost::program_options::options_description desc("This program validates OKS git repository for commit by pre-receive hook");
 
  std::vector<std::string> created, updated, deleted;
  bool circular_dependency_between_includes_is_error = true;
  bool verbose = false;
  std::string user;
  std::size_t pipeline_size = 4;
 
  try
    {
      std::vector<std::string> app_types_list;
      std::vector<std::string> segments_list;
 
      desc.add_options()
        ("add,a", boost::program_options::value<std::vector<std::string> >(&created)->multitoken(), "list of new OKS files and directories to be added to the repository")
        ("update,u", boost::program_options::value<std::vector<std::string> >(&updated)->multitoken(), "list of new OKS files and directories to be updated in the repository")
        ("remove,r", boost::program_options::value<std::vector<std::string> >(&deleted)->multitoken(), "list of new OKS files and directories to be removed from the repository")
        ("permissive-circular-dependencies-between-includes,C", "downgrade severity of detected circular dependencies between includes from errors to warnings")
        ("user,U", boost::program_options::value<std::string>(&user), "user id")
        ("threads-number,t", boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size), "number of threads used by validation pipeline")
        ("verbose,v", "Print debug information")
        ("help,h", "Print help message");
 
      boost::program_options::variables_map vm;
      boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
 
      if (vm.count("help"))
        {
          std::cout << desc << std::endl;
          return __Success__;
        }
 
      if (vm.count("permissive-circular-dependencies-between-includes"))
        circular_dependency_between_includes_is_error = false;
 
      if (vm.count("verbose"))
        verbose = true;
 
      boost::program_options::notify(vm);
 
    }
  catch (std::exception& ex)
    {
      std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
      return __BadCommandLine__;
    }
 
 
  try
    {
      OksKernel kernel;
 
      kernel.set_allow_duplicated_objects_mode(false);
      kernel.set_test_duplicated_objects_via_inheritance_mode(true);
 
      if(kernel.get_user_repository_root().empty())
        {
          std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
          return __NoRepository__;
        }
 
      std::filesystem::current_path(kernel.get_user_repository_root());
 
      auto start_usage = std::chrono::steady_clock::now();
 
      // directories
 
      std::set<std::string> directories;
 
 
      // file: explicit includes
      std::map<std::string, std::set<std::string>> file_explicit_includes;
 
      for (auto& p : std::filesystem::recursive_directory_iterator("."))
        if (std::filesystem::is_directory(p))
          directories.insert(p.path().native().substr(2));
        else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 && p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
          kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
 
      if (verbose)
        log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage).count() / 1000. << " ms\n";
 
 
      auto start_usage2 = std::chrono::steady_clock::now();
 
      std::set<std::string> all_includes;
 
      // check every include exists
      for (const auto& f : file_explicit_includes)
        {
          for (const auto& i : f.second)
            {
              if(file_explicit_includes.find(i) != file_explicit_includes.end())
                {
                  all_includes.insert(i);
                }
              else
                {
                  std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
                  return __NoIncludedFile__;
                }
            }
        }
 
      if (verbose)
        log_timestamp(Debug) << "check existence of includes in " << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()-start_usage2).count() / 1000. << " ms\n";
 
 
      auto start_usage3 = std::chrono::steady_clock::now();
 
      // file: all includes
      std::map<std::string, std::set<std::string>> file_all_includes;
 
      for(auto& x : file_explicit_includes)
        {
          TestCircularDependency cd_fuse(&x.first);
          define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
        }
 
      auto stop_usage3 = std::chrono::steady_clock::now();
 
      if (verbose)
        log_timestamp(Debug) << "calculated inclusion graph in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage3).count() / 1000. << " ms\n";
 
      log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in " << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3-start_usage).count() / 1000. << " ms" << std::endl;
 
 
      if (s_circular_dependency_message.m_count)
        {
          log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning)) << "Detected " << s_circular_dependency_message.m_count << " circular dependencies between includes of the repository files:" << s_circular_dependency_message.m_text.str() << std::endl;
 
          if (circular_dependency_between_includes_is_error == true)
            return __IncludesCircularDependencyError__;
        }
 
      if (ers::debug_level() >= 2)
        {
          std::ostringstream text;
          text << "ALL INCLUDES:\n";
 
          for (const auto& x : file_all_includes)
            {
              text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
 
              for (const auto& y : x.second)
                text << " - \"" << y << "\"\n";
            }
 
          TLOG_DEBUG(2) << text.str();
        }
 
      OksPipeline pipeline(pipeline_size);
 
      // do not run check for README file
      auto ignore_files = [](std::string& x)
      {
        static const std::string readme_file("README.md");
        return x != readme_file;
      };
 
      // all modified file paths
      std::set<std::string> modified;
      std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
      std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
 
      // validate independently every created or updated file
      for (const auto& x : modified)
        pipeline.addJob(new OksValidateJob(kernel, x));
 
      std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
 
      for (const auto& f : file_explicit_includes)
        if (all_includes.find(f.first) == all_includes.end())
          {
            if (modified.empty() == false)
              {
                if (modified.find(f.first) == modified.end())
                  {
                    const auto& file_includes = file_all_includes[f.first];
 
                    bool found = false;
 
                    for (const auto& x : modified)
                      if (file_includes.find(x) != file_includes.end())
                        {
                          found = true;
                          TLOG_DEBUG(1) <<  "file \"" << f.first << "\" contains modified include \"" << x << '\"';
                          break;
                        }
 
                    if(found == false)
                      {
                        TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
                        continue;
                      }
                  }
                else
                  {
                    TLOG_DEBUG(1) <<  "list of modified files contains file \"" << f.first << '\"';
                  }
              }
 
            if (modified.find(f.first) == modified.end())
              pipeline.addJob(new OksValidateJob(kernel, f.first));
          }
 
      pipeline.waitForCompletion();
 
      if (!s_load_error.empty())
        {
          log_timestamp(Error) << s_load_error << std::endl;
          return __ConsistencyError__;
        }
    }
  catch (exception & ex)
    {
      log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
      return __ExceptionCaught__;
    }
  catch (std::exception & e)
    {
      log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
      return __ExceptionCaught__;
    }
  catch (...)
    {
      log_timestamp(Error) << "Caught unknown exception" << std::endl;
      return __ExceptionCaught__;
    }
 
  return __Success__;
}

Variable Documentation

s_circular_dependency_message

struct FoundCircularDependency s_circular_dependency_message

s_load_error

std::string s_load_error

Definition at line 41 of file oks_validate_repository.cxx.