kernel.hpp

Go to the documentation of this file.

// DUNE DAQ modification notice:
// This file has been modified from the original ATLAS oks source for the DUNE DAQ project.
// Fork baseline commit: oks-08-03-04 (2022-04-14).
// Renamed since fork: yes (from oks/kernel.h to include/oks/kernel.hpp).

 
#ifndef OKS_KERNEL_H
#define OKS_KERNEL_H
 
#include <sys/time.h>
 
#include <ctime>
#include <list>
#include <mutex>
#include <set>
#include <shared_mutex>
#include <string>
#include <vector>
 
#include <boost/shared_ptr.hpp>
 
#include "oks/class.hpp"
#include "oks/defs.hpp"
#include "oks/exceptions.hpp"
#include "oks/file.hpp"
#include "oks/object.hpp"
#include "oks/xml.hpp"
 
namespace dunedaq {
namespace oks {
//
// Forward declarations
//
 
class OksMethod;
class OksProfiler;
class OksString;
class OksPipeline;


 
class FailedLoadFile : public exception
{
 
public:
  FailedLoadFile(const std::string& item, const std::string& name, const exception& reason) noexcept
    : exception(fill(item, name, reason.what()), reason.level() + 1)
  {
  }

  FailedLoadFile(const std::string& item, const std::string& name, const std::string& reason) noexcept
    : exception(fill(item, name, reason), 0)
  {
  }
 
  virtual ~FailedLoadFile() noexcept {}
 
private:
  static std::string fill(const std::string& item, const std::string& name, const std::string& reason) noexcept;
};

 
class FailedReloadFile : public exception
{
 
public:
  FailedReloadFile(const std::string& names, const exception& reason) noexcept
    : exception(fill(names, reason.what()), reason.level() + 1)
  {
  }

  FailedReloadFile(const std::string& names, const std::string& reason) noexcept
    : exception(fill(names, reason), 0)
  {
  }
 
  virtual ~FailedReloadFile() noexcept {}
 
private:
  static std::string fill(const std::string& names, const std::string& reason) noexcept;
};

 
class RepositoryOperationFailed : public exception
{
 
public:
  RepositoryOperationFailed(const char* op, const exception& reason) noexcept
    : exception(fill(op, reason.what()), reason.level() + 1)
  {
  }

  RepositoryOperationFailed(const char* op, const std::string& reason) noexcept
    : exception(fill(op, reason), 0)
  {
  }
 
  virtual ~RepositoryOperationFailed() noexcept {}
 
private:
  static std::string fill(const char* op, const std::string& reason) noexcept;
};

 
class CanNotOpenFile : public exception
{
 
public:
  CanNotOpenFile(const char* prefix, const std::string& name, const std::string& reason) noexcept
    : exception(fill(prefix, name, reason), 0)
  {
  }
 
  virtual ~CanNotOpenFile() noexcept {}
 
private:
  static std::string fill(const char* prefix, const std::string& name, const std::string& reason) noexcept;
};

 
class CanNotCreateFile : public exception
{
 
public:
  CanNotCreateFile(const char* prefix, const char* item, const std::string& name, const exception& reason) noexcept
    : exception(fill(prefix, item, name, reason.what()), reason.level() + 1)
  {
  }
  CanNotCreateFile(const char* prefix, const char* item, const std::string& name, const std::string& reason) noexcept
    : exception(fill(prefix, item, name, reason), 0)
  {
  }
 
  virtual ~CanNotCreateFile() noexcept {}
 
private:
  static std::string fill(const char* prefix,
                          const char* item,
                          const std::string& name,
                          const std::string& reason) noexcept;
};

 
class CanNotCreateRepositoryDir : public exception
{
 
public:
  CanNotCreateRepositoryDir(const char* prefix, const std::string& name) noexcept
    : exception(fill(prefix, name), 0)
  {
  }
 
  virtual ~CanNotCreateRepositoryDir() noexcept {}
 
private:
  static std::string fill(const char* prefix, const std::string& name) noexcept;
};

 
class CanNotWriteToFile : public exception
{
 
public:
  CanNotWriteToFile(const char* prefix, const char* item, const std::string& name, const exception& reason) noexcept
    : exception(fill(prefix, item, name, reason.what()), reason.level() + 1)
  {
  }

  CanNotWriteToFile(const char* prefix, const char* item, const std::string& name, const std::string& reason) noexcept
    : exception(fill(prefix, item, name, reason), 0)
  {
  }
 
  virtual ~CanNotWriteToFile() noexcept {}
 
private:
  static std::string fill(const char* prefix,
                          const char* item,
                          const std::string& name,
                          const std::string& reason) noexcept;
};

 
class CanNotBackupFile : public exception
{
 
public:
  CanNotBackupFile(const std::string& name, const exception& reason) noexcept
    : exception(fill(name, reason.what()), reason.level() + 1)
  {
  }
 
  virtual ~CanNotBackupFile() noexcept {}
 
private:
  static std::string fill(const std::string& name, const std::string& reason) noexcept;
};

 
class CanNotSetActiveFile : public exception
{
 
public:
  CanNotSetActiveFile(const char* item, const std::string& name, const exception& reason) noexcept
    : exception(fill(item, name, reason.what()), reason.level() + 1)
  {
  }

  CanNotSetActiveFile(const char* item, const std::string& name, const std::string& reason) noexcept
    : exception(fill(item, name, reason), 0)
  {
  }
 
  virtual ~CanNotSetActiveFile() noexcept {}
 
private:
  static std::string fill(const char* item, const std::string& name, const std::string& reason) noexcept;
};

 
class CanNotSetFile : public exception
{
 
public:
  CanNotSetFile(const OksClass* c, const OksObject* o, const OksFile& file, const exception& reason) noexcept
    : exception(fill(c, o, file, reason.what()), reason.level() + 1)
  {
  }

  CanNotSetFile(const OksClass* c, const OksObject* o, const OksFile& file, const std::string& reason) noexcept
    : exception(fill(c, o, file, reason), 0)
  {
  }
 
  virtual ~CanNotSetFile() noexcept {}
 
private:
  static std::string fill(const OksClass* c,
                          const OksObject* o,
                          const OksFile& file,
                          const std::string& reason) noexcept;
};

 
class CannotAddClass : public exception
{
 
public:
  CannotAddClass(const OksClass& c, const exception& reason) noexcept
    : exception(fill(c, reason.what()), reason.level() + 1)
  {
  }

  CannotAddClass(const OksClass& c, const std::string& reason) noexcept
    : exception(fill(c, reason), 0)
  {
  }
 
  virtual ~CannotAddClass() noexcept {}
 
private:
  static std::string fill(const OksClass& c, const std::string& reason) noexcept;
};

 
class AuthenticationFailure : public exception
{
 
public:
  AuthenticationFailure(const std::string& reason) noexcept
    : exception(reason, 0)
  {
  }
 
  virtual ~AuthenticationFailure() noexcept {}
};

 
class CannotResolvePath : public exception
{
 
public:
  CannotResolvePath(const std::string& path, int error_code) noexcept
    : exception(fill(path, error_code), 0)
  {
  }
 
  virtual ~CannotResolvePath() noexcept {}
 
private:
  static std::string fill(const std::string& path, int error_code) noexcept;
};
 
struct LoadErrors
{
public:
  void add_error(const OksFile& file, std::exception& ex);
  bool is_empty()
  {
    std::lock_guard lock(p_mutex);
    return m_errors.empty();
  }
  void clear()
  {
    std::lock_guard lock(p_mutex);
    m_errors.clear();
  }
  std::string get_text();
 
private:
  std::list<std::string> m_errors;
  std::string m_error_string;
  std::mutex p_mutex;
 
  void add_parents(std::string& text, const OksFile* file, std::set<const OksFile*>& parents);
};
 
double
get_time_interval(const timeval* t1, const timeval* t2);
 
enum __LogSeverity__
{
  Error = 0,
  Warning,
  Log,
  Debug
};
 
std::ostream&
log_timestamp(__LogSeverity__ severity = Log);

 
struct OksNameTable
{
 
  OksNameTable() { reset(); }
 
  OksString* get();
  void reset() { count = -1; }
 
private:
  static const char symbols[];
  long count;
};

 
struct OksAliasTable
{
 
  ~OksAliasTable();
 
  // struct Value stores pointer to class (if loaded) or pointer to its name
 
  struct Value
  {
 
    struct SortByName
    {
      bool operator()(const OksString* s1, const OksString* s2) const { return *s1 < *s2; }
    };
 
    typedef std::map<const OksString*, Value*, SortByName> Map;
 
    Value(OksString* n, const OksClass* p)
      : class_name(n)
      , class_ptr(p)
    {
      ;
    }
 
    OksString* class_name;
    const OksClass* class_ptr;
  };
 
  inline void add_key(OksString*);                          // class is not loaded
  inline void add_value(OksString* vn, const OksClass* vp); // class was loaded
  inline const Value* get(const OksString* key) const;
  const Value* get(const char* key) const
  {
    OksString str(key);
    return get(&str);
  }
 
private:
  Value::Map p_aliases;
  OksNameTable p_nameTable;
};

 
inline void
OksAliasTable::add_key(OksString* key)
{
  p_aliases[key] = new Value(p_nameTable.get(), 0);
}
 
inline void
OksAliasTable::add_value(OksString* vn, const OksClass* vp)
{
  p_aliases[p_nameTable.get()] = new Value(vn, vp);
}
 
inline const OksAliasTable::Value*
OksAliasTable::get(const OksString* key) const
{
  Value::Map::const_iterator i = p_aliases.find(key);
 
  if (i != p_aliases.end())
    return i->second;
  else
    return 0;
}
 
struct OksRepositoryVersion
{
  std::string m_commit_hash;
  std::string m_user;
  std::time_t m_date;
  std::string m_comment;
  std::vector<std::string> m_files;
 
  void clear()
  {
    m_commit_hash.clear();
    m_user.clear();
    m_date = 0;
    m_comment.clear();
    m_files.clear();
  }
};

 
class OksKernel
{
  friend std::ostream& operator<<(std::ostream&, OksKernel&);
  friend class OksFile;
  friend class OksClass;
  friend class OksObject;
  friend struct OksLoadObjectsJob;
  friend struct OksData;
 
  template<typename T>
  using map_str_t = std::map<std::string, T>;
 
public:
 
  OksKernel(bool silence_mode = false,
            bool verbose_mode = false,
            bool profiling_mode = false,
            bool allow_repository = true,
            const char* version = nullptr,
            std::string branch_name = "");

 
  OksKernel(const OksKernel& src, bool copy_repository = false);
 
  ~OksKernel();
 
  OksProfiler* GetOksProfiler() const { return profiler; }

 
  static const char* GetVersion();

 
  static std::string& get_host_name();

 
  static std::string& get_domain_name();

 
  static std::string& get_user_name();
 
  /*   /\** */
  /*    *  \brief Validate user credentials (used by rdb and oksconflibs). */
  /*    * */
  /*    *  PAM authentication is used. */
  /*    * */
  /*    *  \param user    - user name */
  /*    *  \param passwd  - user password */
  /*    * */
  /*    *  \throw Throw std::exception in case of problems. */
  /*    *\/ */
 
  /* static void validate_credentials(const char * user, const char * passwd); */

 
  bool get_verbose_mode() const { return p_verbose; }

 
  void set_verbose_mode(const bool b) { p_verbose = b; }

 
  bool get_silence_mode() const { return p_silence; }

 
  void set_silence_mode(const bool b) { p_silence = b; }

 
  bool get_profiling_mode() const { return p_profiling; }

 
  void set_profiling_mode(const bool b);

 
  bool get_allow_duplicated_classes_mode() const { return p_allow_duplicated_classes; }

 
  bool get_allow_duplicated_objects_mode() const { return p_allow_duplicated_objects; }

 
  void set_test_duplicated_objects_via_inheritance_mode(const bool b) { p_test_duplicated_objects_via_inheritance = b; }

 
  bool get_test_duplicated_objects_via_inheritance_mode() const { return p_test_duplicated_objects_via_inheritance; }

 
  void set_allow_duplicated_classes_mode(const bool b) { p_allow_duplicated_classes = b; }

 
  void set_allow_duplicated_objects_mode(const bool b) { p_allow_duplicated_objects = b; }

 
  static bool get_skip_string_range() { return p_skip_string_range; }

 
  static void set_skip_string_range(const bool b) { p_skip_string_range = b; }

 
  std::shared_mutex& get_mutex() { return p_kernel_mutex; }
 
  //
  //  Methods to work with OKS database files
  //

 
  OksFile* find_schema_file(const std::string& s) const;

 
  OksFile* find_data_file(const std::string& s) const;

 
  std::list<OksClass*>* create_list_of_schema_classes(OksFile*) const;

 
  std::list<OksObject*>* create_list_of_data_objects(OksFile*) const;

 
  OksFile* create_file_info(const std::string& short_file_name, const std::string& file_name);

 
  static bool check_read_only(OksFile* f);

 
  std::string get_file_path(const std::string& path, const OksFile* parent_file = 0, bool strict_paths = true) const;

 
  static const std::string& get_repository_root();
 
  const std::string& get_repository_version() { return p_repository_version; }
 
  bool is_user_repository_created() const { return p_user_repository_root_created; }

 
  static const std::string& get_repository_mapping_dir();

 
  const std::string& get_user_repository_root() const;

 
  void set_user_repository_root(const std::string& path, const std::string& version = "");

 
  static std::string get_tmp_file(const std::string& file_name);

 
  void get_includes(const std::string& file_name, std::set<std::string>& includes, bool use_repository_name = false);

 
  void k_close_dangling_includes();

 
  OksFile* load_file(const std::string& name, bool bind = true);

 
  OksFile* load_schema(const std::string& name, const OksFile* parent = 0);

 
  OksFile* new_schema(const std::string& name);

 
  void save_schema(OksFile* file_h, bool force = false, OksFile* true_file_h = 0);

 
  void save_schema(OksFile* file_h, bool force, const OksClass::Map& classes);

 
  void backup_schema(OksFile* pf, const char* suffix = ".bak");

 
  void save_as_schema(const std::string& name, OksFile* file_h);

 
  void save_all_schema();

 
  void close_schema(OksFile* file_h);

 
  void close_all_schema();

 
  void set_active_schema(OksFile* file_h);

 
  void k_set_active_schema(OksFile* file_h);

 
  OksFile* get_active_schema() const { return p_active_schema; }

 
  const OksFile::Map& schema_files() const { return p_schema_files; }

 
  void create_lists_of_updated_schema_files(std::list<OksFile*>** updated, std::list<OksFile*>** removed) const;

 
  void get_updated_repository_files(std::set<std::string>& updated,
                                    std::set<std::string>& added,
                                    std::set<std::string>& removed);

 
  OksFile* load_data(const std::string& name, bool bind = true);

 
  void reload_data(std::set<OksFile*>& files, bool allow_schema_extension = true);

 
  OksFile* new_data(const std::string& name, const std::string& logical_name = "", const std::string& type = "");

 
  void save_data(OksFile* file_h,
                 bool ignore_bad_objects = false,
                 OksFile* true_file_h = nullptr,
                 bool force_defaults = false);

 
  void save_data(OksFile* file_h, const OksObject::FSet& objects);

 
  void backup_data(OksFile* pf, const char* suffix = ".bak");

 
  void save_as_data(const std::string& new_name, OksFile* file_h);

 
  void save_all_data(bool force_defaults = false);

 
  void close_data(OksFile* file_h, bool unbind_objects = true);

 
  void close_all_data();

 
  void set_active_data(OksFile* file_h);

 
  void k_set_active_data(OksFile*);

 
  OksFile* get_active_data() const { return p_active_data; }

 
  const OksFile::Map& data_files() const { return p_data_files; }

 
  void create_lists_of_updated_data_files(std::list<OksFile*>** updated, std::list<OksFile*>** removed) const;

 
  void get_modified_files(std::set<OksFile*>& mfs, std::set<OksFile*>& rfs, const std::string& version);

 
  const std::list<std::string>& get_repository_dirs() const { return p_repository_dirs; }

 
  void commit_repository(const std::string& comments, const std::string& credentials = "");

 
  void tag_repository(const std::string& tag);

 
  std::time_t get_repository_checkout_ts() const { return p_repository_checkout_ts; }
 
  enum RepositoryUpdateType
  {
    DiscardChanges,
    MergeChanges,
    NoChanges
  };

 
  void update_repository(const std::string& hash_val, RepositoryUpdateType update_type)
  {
    update_repository("hash", hash_val, update_type);
  }

 
  void update_repository(const std::string& param, const std::string& val, RepositoryUpdateType update_type);

 
  std::list<std::string> get_repository_versions_diff(const std::string& sha1, const std::string& sha2);

 
  std::list<std::string> get_repository_unmerged_files() { return get_repository_versions_diff("", ""); }

 
  std::vector<OksRepositoryVersion> get_repository_versions(bool skip_irrelevant, const std::string& command_line);

 
  std::vector<OksRepositoryVersion> get_repository_versions_by_hash(bool skip_irrelevant = true,
                                                                    const std::string& sha1 = "",
                                                                    const std::string& sha2 = "");

 
  std::vector<OksRepositoryVersion> get_repository_versions_by_date(bool skip_irrelevant = true,
                                                                    const std::string& since = "",
                                                                    const std::string& until = "");

 
  std::string read_repository_version();

 
  static const char* get_cwd();

 
  static void reset_cwd() { s_cwd = 0; }

 
  static void set_use_strict_repository_paths(bool flag) { p_use_strict_repository_paths = flag; }

 
  std::string insert_repository_dir(const std::string& dir, bool push_back = true);

 
  void remove_repository_dir(const std::string& dir);

 
  const OksClass::Map& classes() const { return p_classes; }

 
  size_t number_of_classes() const { return p_classes.size(); }

 
  const OksObject::Set& objects() const { return p_objects; }

 
  size_t number_of_objects() const { return p_objects.size(); }

 
  OksClass* find_class(const std::string& class_name) const { return find_class(class_name.c_str()); }

 
  OksClass* find_class(const char* class_name) const;

 
  void get_all_classes(const std::vector<std::string>& names_in, ClassSet& classes_out) const;

 
  void registrate_all_classes(bool skip_registered = false);

 
  bool is_dangling(OksClass* class_ptr) const;

 
  bool is_dangling(OksObject* obj_ptr) const;

 
  void subscribe_create_class(void (*f)(OksClass*));

 
  void subscribe_change_class(void (*f)(OksClass*, OksClass::ChangeType, const void*));

 
  void subscribe_delete_class(void (*f)(OksClass*));

 
  void subscribe_create_object(OksObject::notify_obj cb_f, void* parameter);

 
  void subscribe_change_object(OksObject::notify_obj, void*);

 
  void subscribe_delete_object(OksObject::notify_obj, void*);

 
  void bind_objects();

 
  const std::string& get_bind_objects_status() const noexcept { return p_bind_objects_status; }

 
  const std::string& get_bind_classes_status() const noexcept { return p_bind_classes_status; }

 
  void unset_repository_created();
 
private:
  bool p_silence;
  bool p_verbose;
  bool p_profiling;
  bool p_allow_repository;
  bool p_allow_duplicated_classes;
  bool p_allow_duplicated_objects;
  bool p_test_duplicated_objects_via_inheritance;
 
  static bool p_skip_string_range;
  static bool p_use_strict_repository_paths;
 
  static std::string p_repository_root;
  static std::string p_repository_mapping_dir;
  std::string p_user_repository_root;
  bool p_user_repository_root_inited;
  bool p_user_repository_root_created;
  std::string p_repository_version;
  std::time_t p_repository_checkout_ts;
  std::time_t p_repository_update_ts;
 
  // kernel mutexes
 
  mutable std::shared_mutex p_kernel_mutex;
  mutable std::mutex p_objects_mutex;
  std::mutex p_objects_refs_mutex;
  std::shared_mutex p_schema_mutex;
  static std::mutex p_parallel_out_mutex;
 
  // schema and data files members
 
  OksFile::Map p_schema_files;
  OksFile::Map p_data_files;
 
  OksFile* p_active_schema;
  OksFile* p_active_data;
 
  bool p_close_all;
 
  // repository directories
 
  std::list<std::string> p_repository_dirs;
 
  static char* s_cwd;
 
  OksClass::Map p_classes;
 
  static unsigned long p_count;
 
  OksProfiler* profiler;
 
  OksObject::Set p_objects;
 
  static int p_threads_pool_size;
 
  std::string p_bind_objects_status;
  mutable std::string p_bind_classes_status;
 
  LoadErrors p_load_errors;
 
  std::map<const OksFile*, OksFile*> p_preload_file_info;
  std::vector<OksFile*> p_preload_added_files;
 
  // to be used by the kernel only
 
  std::string create_user_repository_dir();
  void remove_user_repository_dir();
 
  void k_copy_repository(const std::string& source, const std::string& destination);

 
  void k_checkout_repository(const std::string& param, const std::string& val, const std::string& branch);
 
  OksFile* find_file(const std::string& s, const OksFile::Map& files) const;
  void k_rename_repository_file(OksFile* file_h, const std::string& new_name);
 
  OksFile* k_load_file(const std::string& name, bool bind, const OksFile* parent, OksPipeline* pipeline);
 
  void k_load_includes(const OksFile&, OksPipeline*);
  bool k_preload_includes(OksFile* file_h, std::set<OksFile*>& new_files, bool allow_schema_extension);
  void clear_preload_file_info();
  void restore_preload_file_info();
 
  OksFile* k_load_schema(const std::string&, const OksFile*);
  void k_load_schema(OksFile* fp, std::shared_ptr<OksXmlInputStream> xmls, const OksFile* parent_h);
  void k_close_schema(OksFile*);
  void k_save_schema(OksFile*, bool force = false, OksFile* = 0, const OksClass::Map* = 0);
  void k_rename_schema(OksFile*, const std::string& short_name, const std::string& long_name);
 
  OksFile* k_load_data(const std::string&, bool, const OksFile*, OksPipeline*);
  void k_load_data(OksFile* fp,
                   char format,
                   std::shared_ptr<OksXmlInputStream> xmls,
                   long file_length,
                   bool bind,
                   const OksFile* parent_h,
                   OksPipeline*);
  void k_close_data(OksFile*, bool);
  void k_save_data(OksFile*,
                   bool = false,
                   OksFile* = nullptr,
                   const OksObject::FSet* = nullptr,
                   bool force_defaults = false);
  void k_rename_data(OksFile*, const std::string& short_name, const std::string& long_name);
 
  void k_bind_objects();
  void k_check_bind_classes_status() const noexcept;

 
  void unbind_all_rels(const OksObject::FSet& rm_objs, OksObject::FSet& updated) const;
 
  void k_add(OksClass*);
  void k_remove(OksClass*);
 
  void define(OksObject* o)
  {
    std::lock_guard lock(p_objects_mutex);
    p_objects.insert(o);
  }
  void undefine(OksObject* o)
  {
    if (!p_objects.empty()) {
      std::lock_guard lock(p_objects_mutex);
      p_objects.erase(o);
    }
  }
 
  void add_data_file(OksFile*);
  void add_schema_file(OksFile*);
  void remove_data_file(OksFile*);
  void remove_schema_file(OksFile*);
 
  bool test_parent(OksFile* file, OksFile::IMap::iterator& i);
 
  OksObject::notify_obj p_create_object_notify_fn;
  void* p_create_object_notify_param;
 
  OksObject::notify_obj p_change_object_notify_fn;
  void* p_change_object_notify_param;
 
  OksObject::notify_obj p_delete_object_notify_fn;
  void* p_delete_object_notify_param;
};

 
inline void
OksKernel::subscribe_create_object(OksObject::notify_obj f, void* p)
{
  p_create_object_notify_fn = f;
  p_create_object_notify_param = p;
}
 
inline void
OksKernel::subscribe_change_object(OksObject::notify_obj f, void* p)
{
  p_change_object_notify_fn = f;
  p_change_object_notify_param = p;
}
 
inline void
OksKernel::subscribe_delete_object(OksObject::notify_obj f, void* p)
{
  p_delete_object_notify_fn = f;
  p_delete_object_notify_param = p;
}
 
inline void
OksKernel::subscribe_create_class(void (*f)(OksClass*))
{
  OksClass::create_notify_fn = f;
}
 
inline void
OksKernel::subscribe_change_class(void (*f)(OksClass*, OksClass::ChangeType, const void*))
{
  OksClass::change_notify_fn = f;
}
 
inline void
OksKernel::subscribe_delete_class(void (*f)(OksClass*))
{
  OksClass::delete_notify_fn = f;
}
 
} // namespace oks
} // namespace dunedaq
 
#endif