oks_validate_repository.cxx

Go to the documentation of this file.

// DUNE DAQ modification notice:
// This file has been modified from the original ATLAS oks source for the DUNE DAQ project.
// Fork baseline commit: oks-08-03-04 (2022-04-14).
// Renamed since fork: yes (from bin/oks_validate_repository.cpp to apps/oks_validate_repository.cxx).

 
#include "oks/exceptions.hpp"
#include "oks/kernel.hpp"
#include "oks/pipeline.hpp"
 
#include <boost/program_options.hpp>
 
#include "ers/ers.hpp"
#include "logging/Logging.hpp"
 
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <iostream>
#include <mutex>
#include <vector>
 
using namespace dunedaq::oks;
 
enum __OksValidateRepositoryExitStatus__
{
  __Success__ = 0,
  __BadCommandLine__,
  __UserAuthenticationFailure__,
  __NoRepository__,
  __ConsistencyError__,
  __IncludesCircularDependencyError__,
  __AccessManagerAuthorizationFailed__,
  __AccessManagerNoPermission__,
  __NoIncludedFile__,
  __ExceptionCaught__
};
 
std::string s_load_error;
 
static void
init_file_load_error(const std::string& file)
{
  s_load_error = "repository validation failed for file \'";
  s_load_error += file;
  s_load_error += "\':\n";
}
 
struct OksValidateJob : public OksJob
{
public:
  OksValidateJob(OksKernel& kernel, const std::string& file_name)
    : m_kernel(kernel)
    , m_file_name(file_name)
  {
    ;
  }
 
  void run()
  {
    static std::mutex s_mutex;
 
    try {
      auto start_usage = std::chrono::steady_clock::now();
 
      m_kernel.set_silence_mode(true);
 
      m_kernel.load_file(m_file_name);
 
      if (!m_kernel.get_bind_classes_status().empty() || !m_kernel.get_bind_objects_status().empty()) {
        std::lock_guard lock(s_mutex);
 
        if (s_load_error.empty()) {
          init_file_load_error(m_file_name);
 
          if (!m_kernel.get_bind_classes_status().empty()) {
            s_load_error += "the schema contains dangling references to non-loaded classes:\n";
            s_load_error += m_kernel.get_bind_classes_status();
          }
 
          if (!m_kernel.get_bind_objects_status().empty()) {
            s_load_error += "the data contain dangling references to non-loaded objects:\n";
            s_load_error += m_kernel.get_bind_objects_status();
          }
        }
      }
 
      static std::mutex s_log_mutex;
      std::lock_guard scoped_lock(s_log_mutex);
 
      log_timestamp() << "validated file \"" << m_file_name << "\" in "
                      << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
                                                                               start_usage)
                             .count() /
                           1000.
                      << " ms\n";
    } catch (std::exception& ex) {
      std::lock_guard lock(s_mutex);
 
      if (s_load_error.empty()) {
        init_file_load_error(m_file_name);
        s_load_error += ex.what();
 
        const std::string& user_repository(m_kernel.get_user_repository_root());
        const std::size_t user_repository_len(m_kernel.get_user_repository_root().length() + 1);
 
        std::size_t pos;
        while ((pos = s_load_error.find(user_repository)) != std::string::npos)
          s_load_error.replace(pos, user_repository_len, "");
      }
    }
  }
 
private:
  OksKernel m_kernel;
  const std::string& m_file_name;
 
  // protect usage of copy constructor and assignment operator
 
private:
  OksValidateJob(const OksValidateJob&);
  OksValidateJob& operator=(const OksValidateJob&);
};
 
struct FoundCircularDependency
{
  unsigned int m_count;
  std::ostringstream m_text;
 
  FoundCircularDependency()
    : m_count(0)
  {
    ;
  }
} s_circular_dependency_message;
 
struct TestCircularDependency
{
  TestCircularDependency(const std::string* file)
  {
    p_set_includes.insert(file);
    p_vector_includes.push_back(file);
  }
 
  bool push(const std::string* file)
  {
    auto it = p_set_includes.insert(file);
    if (it.second == false) {
      std::ostringstream s;
 
      bool report = false;
 
      for (const auto& x : p_vector_includes) {
        if (x == *it.first) {
          s_circular_dependency_message.m_text << "\nCircular dependency [" << ++s_circular_dependency_message.m_count
                                               << "]:";
          report = true;
        }
 
        if (report)
          s_circular_dependency_message.m_text << '\n' << " - \"" << *x << "\"";
      }
 
      return false;
    }
 
    p_vector_includes.push_back(file);
 
    return true;
  }
 
  void pop()
  {
    p_set_includes.erase(p_vector_includes.back());
    p_vector_includes.pop_back();
  }
 
  std::vector<const std::string*> p_vector_includes;
  std::set<const std::string*, OksFile::SortByName> p_set_includes;
};
 
std::set<std::string>&
define_includes(const std::string& f,
                const std::set<std::string>& s,
                std::map<std::string, std::set<std::string>>& file_all_includes,
                std::map<std::string, std::set<std::string>>& file_explicit_includes,
                TestCircularDependency& cd_fuse)
{
  std::set<std::string>& all_includes = file_all_includes[f];
 
  if (all_includes.empty()) {
    for (auto& x : s) {
      if (cd_fuse.push(&x)) {
        all_includes.insert(x);
 
        std::set<std::string>& includes =
          define_includes(x, file_explicit_includes[x], file_all_includes, file_explicit_includes, cd_fuse);
        for (const auto& y : includes)
          all_includes.insert(y);
 
        cd_fuse.pop();
      }
    }
  }
 
  return all_includes;
}
 
int
main(int argc, char** argv)
{
  boost::program_options::options_description desc(
    "This program validates OKS git repository for commit by pre-receive hook");
 
  std::vector<std::string> created, updated, deleted;
  bool circular_dependency_between_includes_is_error = true;
  bool verbose = false;
  std::string user;
  std::size_t pipeline_size = 4;
 
  try {
    std::vector<std::string> app_types_list;
    std::vector<std::string> segments_list;
 
    desc.add_options()("add,a",
                       boost::program_options::value<std::vector<std::string>>(&created)->multitoken(),
                       "list of new OKS files and directories to be added to the repository")(
      "update,u",
      boost::program_options::value<std::vector<std::string>>(&updated)->multitoken(),
      "list of new OKS files and directories to be updated in the repository")(
      "remove,r",
      boost::program_options::value<std::vector<std::string>>(&deleted)->multitoken(),
      "list of new OKS files and directories to be removed from the repository")(
      "permissive-circular-dependencies-between-includes,C",
      "downgrade severity of detected circular dependencies between includes from errors to warnings")(
      "user,U", boost::program_options::value<std::string>(&user), "user id")(
      "threads-number,t",
      boost::program_options::value<std::size_t>(&pipeline_size)->default_value(pipeline_size),
      "number of threads used by validation pipeline")("verbose,v", "Print debug information")("help,h",
                                                                                               "Print help message");
 
    boost::program_options::variables_map vm;
    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm);
 
    if (vm.count("help")) {
      std::cout << desc << std::endl;
      return __Success__;
    }
 
    if (vm.count("permissive-circular-dependencies-between-includes"))
      circular_dependency_between_includes_is_error = false;
 
    if (vm.count("verbose"))
      verbose = true;
 
    boost::program_options::notify(vm);
 
  } catch (std::exception& ex) {
    std::cerr << "Command line parsing errors occurred:\n" << ex.what() << std::endl;
    return __BadCommandLine__;
  }
 
  try {
    OksKernel kernel;
 
    kernel.set_allow_duplicated_objects_mode(false);
    kernel.set_test_duplicated_objects_via_inheritance_mode(true);
 
    if (kernel.get_user_repository_root().empty()) {
      std::cerr << "There is no OKS repository set (check TDAQ_DB_REPOSITORY)" << std::endl;
      return __NoRepository__;
    }
 
    std::filesystem::current_path(kernel.get_user_repository_root());
 
    auto start_usage = std::chrono::steady_clock::now();
 
    // directories
 
    std::set<std::string> directories;
 
    // file: explicit includes
    std::map<std::string, std::set<std::string>> file_explicit_includes;
 
    for (auto& p : std::filesystem::recursive_directory_iterator("."))
      if (std::filesystem::is_directory(p))
        directories.insert(p.path().native().substr(2));
      else if (std::filesystem::is_regular_file(p) && p.path().native().find("./.git") != 0 &&
               p.path().native().find("./admin") != 0 && p.path().native().find("./README.md") != 0)
        kernel.get_includes(p.path().native(), file_explicit_includes[p.path().native().substr(2)], true);
 
    if (verbose)
      log_timestamp(Debug) << "scan " << file_explicit_includes.size() << " repository files in "
                           << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
                                                                                    start_usage)
                                  .count() /
                                1000.
                           << " ms\n";
 
    auto start_usage2 = std::chrono::steady_clock::now();
 
    std::set<std::string> all_includes;
 
    // check every include exists
    for (const auto& f : file_explicit_includes) {
      for (const auto& i : f.second) {
        if (file_explicit_includes.find(i) != file_explicit_includes.end()) {
          all_includes.insert(i);
        } else {
          std::cerr << "Cannot find file \"" << i << "\" included by \"" << f.first << "\"" << std::endl;
          return __NoIncludedFile__;
        }
      }
    }
 
    if (verbose)
      log_timestamp(Debug) << "check existence of includes in "
                           << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
                                                                                    start_usage2)
                                  .count() /
                                1000.
                           << " ms\n";
 
    auto start_usage3 = std::chrono::steady_clock::now();
 
    // file: all includes
    std::map<std::string, std::set<std::string>> file_all_includes;
 
    for (auto& x : file_explicit_includes) {
      TestCircularDependency cd_fuse(&x.first);
      define_includes(x.first, x.second, file_all_includes, file_explicit_includes, cd_fuse);
    }
 
    auto stop_usage3 = std::chrono::steady_clock::now();
 
    if (verbose)
      log_timestamp(Debug)
        << "calculated inclusion graph in "
        << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage3).count() / 1000. << " ms\n";
 
    log_timestamp() << "process " << file_explicit_includes.size() << " repository files and their includes in "
                    << std::chrono::duration_cast<std::chrono::microseconds>(stop_usage3 - start_usage).count() / 1000.
                    << " ms" << std::endl;
 
    if (s_circular_dependency_message.m_count) {
      log_timestamp((circular_dependency_between_includes_is_error == true ? Error : Warning))
        << "Detected " << s_circular_dependency_message.m_count
        << " circular dependencies between includes of the repository files:"
        << s_circular_dependency_message.m_text.str() << std::endl;
 
      if (circular_dependency_between_includes_is_error == true)
        return __IncludesCircularDependencyError__;
    }
 
    if (ers::debug_level() >= 2) {
      std::ostringstream text;
      text << "ALL INCLUDES:\n";
 
      for (const auto& x : file_all_includes) {
        text << "FILE \"" << x.first << "\" has " << x.second.size() << " includes:\n";
 
        for (const auto& y : x.second)
          text << " - \"" << y << "\"\n";
      }
 
      TLOG_DEBUG(2) << text.str();
    }
 
    OksPipeline pipeline(pipeline_size);
 
    // do not run check for README file
    auto ignore_files = [](std::string& x) {
      static const std::string readme_file("README.md");
      return x != readme_file;
    };
 
    // all modified file paths
    std::set<std::string> modified;
    std::copy_if(created.begin(), created.end(), std::inserter(modified, modified.end()), ignore_files);
    std::copy_if(updated.begin(), updated.end(), std::inserter(modified, modified.end()), ignore_files);
 
    // validate independently every created or updated file
    for (const auto& x : modified)
      pipeline.addJob(new OksValidateJob(kernel, x));
 
    std::copy_if(deleted.begin(), deleted.end(), std::inserter(modified, modified.end()), ignore_files);
 
    for (const auto& f : file_explicit_includes)
      if (all_includes.find(f.first) == all_includes.end()) {
        if (modified.empty() == false) {
          if (modified.find(f.first) == modified.end()) {
            const auto& file_includes = file_all_includes[f.first];
 
            bool found = false;
 
            for (const auto& x : modified)
              if (file_includes.find(x) != file_includes.end()) {
                found = true;
                TLOG_DEBUG(1) << "file \"" << f.first << "\" contains modified include \"" << x << '\"';
                break;
              }
 
            if (found == false) {
              TLOG_DEBUG(1) << "skip file \"" << f.first << '\"';
              continue;
            }
          } else {
            TLOG_DEBUG(1) << "list of modified files contains file \"" << f.first << '\"';
          }
        }
 
        if (modified.find(f.first) == modified.end())
          pipeline.addJob(new OksValidateJob(kernel, f.first));
      }
 
    pipeline.waitForCompletion();
 
    if (!s_load_error.empty()) {
      log_timestamp(Error) << s_load_error << std::endl;
      return __ConsistencyError__;
    }
  } catch (exception& ex) {
    log_timestamp(Error) << "Caught oks exception:\n" << ex << std::endl;
    return __ExceptionCaught__;
  } catch (std::exception& e) {
    log_timestamp(Error) << "Caught standard C++ exception:\n" << e.what() << std::endl;
    return __ExceptionCaught__;
  } catch (...) {
    log_timestamp(Error) << "Caught unknown exception" << std::endl;
    return __ExceptionCaught__;
  }
 
  return __Success__;
}