kernel.hpp

Go to the documentation of this file.

 
#ifndef OKS_KERNEL_H
#define OKS_KERNEL_H
 
#include <sys/time.h>
 
#include <ctime>
#include <string>
#include <list>
#include <mutex>
#include <shared_mutex>
#include <set>
#include <vector>
 
#include <boost/shared_ptr.hpp>
 
#include "oks/defs.hpp"
#include "oks/xml.hpp"
#include "oks/file.hpp"
#include "oks/class.hpp"
#include "oks/object.hpp"
#include "oks/exceptions.hpp"
 
 
namespace dunedaq {
namespace oks {
  //
  // Forward declarations
  //
 
class   OksMethod;
class   OksProfiler;
class   OksString;
class   OksPipeline;
 
 
 
  class FailedLoadFile : public exception {
 
    public:
 
      FailedLoadFile(const std::string& item, const std::string& name, const exception& reason) noexcept : exception (fill(item, name, reason.what()), reason.level() + 1) { }
 
      FailedLoadFile(const std::string& item, const std::string& name, const std::string& reason) noexcept : exception (fill(item, name, reason), 0) { }
 
      virtual ~FailedLoadFile() noexcept { }
 
 
    private:
 
      static std::string fill(const std::string& item, const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class FailedReloadFile : public exception {
 
    public:
 
      FailedReloadFile(const std::string& names, const exception& reason) noexcept : exception (fill(names, reason.what()), reason.level() + 1) { }
 
      FailedReloadFile(const std::string& names, const std::string& reason) noexcept : exception (fill(names, reason), 0) { }
 
      virtual ~FailedReloadFile() noexcept { }
 
 
    private:
 
      static std::string fill(const std::string& names, const std::string& reason) noexcept;
 
  };
 
 
  class RepositoryOperationFailed : public exception {
 
    public:
 
      RepositoryOperationFailed(const char * op, const exception& reason) noexcept : exception (fill(op, reason.what()), reason.level() + 1) { }
 
      RepositoryOperationFailed(const char * op, const std::string& reason) noexcept : exception (fill(op, reason), 0) { }
 
      virtual ~RepositoryOperationFailed() noexcept { }
 
 
    private:
 
      static std::string fill(const char * op, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotOpenFile : public exception {
 
    public:
 
      CanNotOpenFile(const char * prefix, const std::string& name, const std::string& reason) noexcept : exception (fill(prefix, name, reason), 0) { }
 
      virtual ~CanNotOpenFile() noexcept { }
 
 
    private:
 
      static std::string fill(const char * prefix, const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotCreateFile : public exception {
 
    public:
 
      CanNotCreateFile(const char * prefix, const char * item, const std::string& name, const exception& reason) noexcept : exception (fill(prefix, item, name, reason.what()), reason.level() + 1) { }
      CanNotCreateFile(const char * prefix, const char * item, const std::string& name, const std::string& reason) noexcept : exception (fill(prefix, item, name, reason), 0) { }
 
      virtual ~CanNotCreateFile() noexcept { }
 
 
    private:
 
      static std::string fill(const char * prefix, const char * item, const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotCreateRepositoryDir : public exception {
 
    public:
 
      CanNotCreateRepositoryDir(const char * prefix, const std::string& name) noexcept : exception (fill(prefix, name), 0) { }
 
      virtual ~CanNotCreateRepositoryDir() noexcept { }
 
 
    private:
 
      static std::string fill(const char * prefix, const std::string& name) noexcept;
 
  };
 
  class CanNotWriteToFile : public exception {
 
    public:
 
      CanNotWriteToFile(const char * prefix, const char * item, const std::string& name, const exception& reason) noexcept : exception (fill(prefix, item, name, reason.what()), reason.level() + 1) {}
 
      CanNotWriteToFile(const char * prefix, const char * item, const std::string& name, const std::string& reason) noexcept : exception (fill(prefix, item, name, reason), 0) {}
 
      virtual ~CanNotWriteToFile() noexcept { }
 
 
    private:
 
      static std::string fill(const char * prefix, const char * item, const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotBackupFile : public exception {
 
    public:
 
      CanNotBackupFile(const std::string& name, const exception& reason) noexcept : exception (fill(name, reason.what()), reason.level() + 1) {}
 
      virtual ~CanNotBackupFile() noexcept { }
 
 
    private:
 
      static std::string fill(const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotSetActiveFile : public exception {
 
    public:
 
      CanNotSetActiveFile(const char * item, const std::string& name, const exception& reason) noexcept : exception (fill(item, name, reason.what()), reason.level() + 1) {}
 
      CanNotSetActiveFile(const char * item, const std::string& name, const std::string& reason) noexcept : exception (fill(item, name, reason), 0) {}
 
      virtual ~CanNotSetActiveFile() noexcept { }
 
 
    private:
 
      static std::string fill(const char * item, const std::string& name, const std::string& reason) noexcept;
 
  };
 
 
  class CanNotSetFile : public exception {
 
    public:
 
      CanNotSetFile(const OksClass * c, const OksObject * o, const OksFile& file, const exception& reason) noexcept : exception (fill(c, o, file, reason.what()), reason.level() + 1) {}
 
      CanNotSetFile(const OksClass * c, const OksObject * o, const OksFile& file, const std::string& reason) noexcept : exception (fill(c, o, file, reason), 0) {}
 
      virtual ~CanNotSetFile() noexcept { }
 
 
    private:
 
      static std::string fill(const OksClass * c, const OksObject * o, const OksFile& file, const std::string& reason) noexcept;
 
  };
 
 
  class CannotAddClass : public exception {
 
    public:
 
      CannotAddClass(const OksClass &c, const exception& reason) noexcept : exception (fill(c, reason.what()), reason.level() + 1) {}
 
      CannotAddClass(const OksClass &c, const std::string& reason) noexcept : exception (fill(c, reason), 0) {}
 
      virtual ~CannotAddClass() noexcept { }
 
    private:
 
      static std::string fill(const OksClass &c, const std::string& reason) noexcept;
 
  };
  
  
  class AuthenticationFailure : public exception {
 
    public:
 
      AuthenticationFailure(const std::string& reason) noexcept : exception (reason, 0) {}
 
      virtual ~AuthenticationFailure() noexcept { }
 
  };
 
 
  class CannotResolvePath : public exception {
 
    public:
 
      CannotResolvePath(const std::string& path, int error_code) noexcept : exception (fill(path, error_code), 0) {}
 
      virtual ~CannotResolvePath() noexcept { }
 
    private:
 
      static std::string fill(const std::string& path, int error_code) noexcept;
 
  };
 
  struct LoadErrors
  {
    public:
 
      void add_error(const OksFile& file, std::exception& ex);
      bool is_empty() { std::lock_guard lock(p_mutex); return m_errors.empty(); }
      void clear() { std::lock_guard lock(p_mutex); m_errors.clear(); }
      std::string get_text();
 
    private:
 
      std::list<std::string> m_errors;
      std::string m_error_string;
      std::mutex p_mutex;
 
      void add_parents(std::string& text, const OksFile * file, std::set<const OksFile *>& parents);
 
  };
 
  double get_time_interval(const timeval * t1, const timeval * t2);
 
  enum __LogSeverity__
  {
    Error = 0,
    Warning,
    Log,
    Debug
  };
 
  std::ostream&
  log_timestamp(__LogSeverity__ severity = Log);
 
 
 
struct OksNameTable {
 
    OksNameTable() {reset();}
 
    OksString * get();
    void reset() {count = -1;}
 
 
  private:
 
    static const char   symbols[];
    long        count;
};
 
 
struct OksAliasTable {
 
    ~OksAliasTable  ();
 
 
      // struct Value stores pointer to class (if loaded) or pointer to its name
 
    struct Value {
 
      struct SortByName {
        bool operator() (const OksString * s1, const OksString * s2) const {
          return *s1 < *s2;
        }
      };
 
      typedef std::map<const OksString *, Value *, SortByName> Map;
 
 
      Value     (OksString * n, const OksClass * p) : class_name (n), class_ptr(p) {;}
 
      OksString *   class_name;
      const OksClass *  class_ptr;
    };
 
    inline void add_key(OksString *);               // class is not loaded
    inline void add_value(OksString * vn, const OksClass * vp); // class was loaded
    inline const Value * get(const OksString * key) const;
    const Value * get(const char * key) const { OksString str(key); return get(&str); }
 
 
  private:
 
    Value::Map      p_aliases;
    OksNameTable    p_nameTable;
 
};
 
 
inline void
OksAliasTable::add_key(OksString * key)
{
  p_aliases[key] = new Value(p_nameTable.get(), 0);
}
 
inline void
OksAliasTable::add_value(OksString * vn, const OksClass * vp)
{
  p_aliases[p_nameTable.get()] = new Value(vn, vp);
}
 
inline const OksAliasTable::Value *
OksAliasTable::get(const OksString * key) const
{
  Value::Map::const_iterator i = p_aliases.find(key);
 
  if(i != p_aliases.end()) return i->second;
  else return 0;
}
 
struct OksRepositoryVersion
{
  std::string m_commit_hash;
  std::string m_user;
  std::time_t m_date;
  std::string m_comment;
  std::vector<std::string> m_files;
 
  void clear()
  {
    m_commit_hash.clear();
    m_user.clear();
    m_date = 0;
    m_comment.clear();
    m_files.clear();
  }
};
 
class OksKernel
{
  friend std::ostream& operator<<(std::ostream&, OksKernel&);
  friend class OksFile;
  friend class OksClass;
  friend class OksObject;
  friend struct OksLoadObjectsJob;
  friend struct OksData;
 
  template<typename T>
  using map_str_t = std::map<std::string, T>;
  
  public:
 
    OksKernel(bool silence_mode = false, bool verbose_mode = false, bool profiling_mode = false, bool allow_repository = true, const char * version = nullptr, std::string branch_name = "");
 
 
    OksKernel(const OksKernel& src, bool copy_repository = false);
 
    ~OksKernel();
 
    OksProfiler * GetOksProfiler() const {return profiler;}
 
 
    static const char * GetVersion();
 
 
    static std::string& get_host_name();
 
 
    static std::string& get_domain_name();
 
 
    static std::string& get_user_name();
 
 
    /*   /\** */
    /*    *  \brief Validate user credentials (used by rdb and oksconflibs). */
    /*    * */
    /*    *  PAM authentication is used. */
    /*    * */
    /*    *  \param user    - user name */
    /*    *  \param passwd  - user password */
    /*    * */
    /*    *  \throw Throw std::exception in case of problems. */
    /*    *\/ */
 
    /* static void validate_credentials(const char * user, const char * passwd); */
 
 
    bool get_verbose_mode() const {return p_verbose;}
 
 
    void set_verbose_mode(const bool b) {p_verbose = b;}
 
 
    bool get_silence_mode() const {return p_silence;}
 
 
    void set_silence_mode(const bool b) {p_silence = b;}
 
 
    bool get_profiling_mode() const {return p_profiling;}
 
 
    void set_profiling_mode(const bool b);
 
 
    bool get_allow_duplicated_classes_mode() const {return p_allow_duplicated_classes;}
 
 
 
    bool get_allow_duplicated_objects_mode() const {return p_allow_duplicated_objects;}
 
 
    void set_test_duplicated_objects_via_inheritance_mode(const bool b) {p_test_duplicated_objects_via_inheritance = b;}
 
 
    bool get_test_duplicated_objects_via_inheritance_mode() const {return p_test_duplicated_objects_via_inheritance;}
 
 
    void set_allow_duplicated_classes_mode(const bool b) {p_allow_duplicated_classes = b;}
 
 
    void set_allow_duplicated_objects_mode(const bool b) {p_allow_duplicated_objects = b;}
 
 
    static bool get_skip_string_range() {return p_skip_string_range;}
 
 
    static void set_skip_string_range(const bool b) {p_skip_string_range = b;}
 
    std::shared_mutex& get_mutex() {return p_kernel_mutex;}
 
 
     //
     //  Methods to work with OKS database files
     //
 
    OksFile * find_schema_file(const std::string & s) const;
 
 
    OksFile * find_data_file(const std::string & s) const;
 
 
    std::list<OksClass *> * create_list_of_schema_classes(OksFile *) const;
 
 
    std::list<OksObject *> * create_list_of_data_objects(OksFile *) const;
 
 
    OksFile * create_file_info(const std::string& short_file_name, const std::string& file_name);
 
 
    static bool check_read_only(OksFile * f);
 
 
    std::string get_file_path(const std::string& path, const OksFile * parent_file = 0, bool strict_paths = true) const;
 
 
    static const std::string& get_repository_root();
 
    const std::string&
    get_repository_version()
    {
      return p_repository_version;
    }
 
    bool
    is_user_repository_created() const
    {
      return p_user_repository_root_created;
    }
 
    static const std::string& get_repository_mapping_dir();
 
 
    const std::string& get_user_repository_root() const;
 
 
    void set_user_repository_root(const std::string& path, const std::string& version = "");
 
 
    static std::string get_tmp_file(const std::string& file_name);
 
 
    void get_includes(const std::string& file_name, std::set< std::string >& includes, bool use_repository_name = false);
    
    
    void k_close_dangling_includes();
 
 
    OksFile * load_file(const std::string& name, bool bind = true);
 
 
    OksFile * load_schema(const std::string& name, const OksFile * parent = 0);
 
 
    OksFile * new_schema(const std::string& name);
 
 
    void save_schema(OksFile * file_h, bool force = false, OksFile * true_file_h = 0);
 
 
    void save_schema(OksFile * file_h, bool force, const OksClass::Map& classes);
 
 
    void backup_schema(OksFile * pf, const char * suffix = ".bak");
 
 
    void save_as_schema(const std::string& name, OksFile * file_h);
 
 
    void save_all_schema();
 
 
    void close_schema(OksFile * file_h);
 
 
    void close_all_schema();
 
 
    void set_active_schema(OksFile * file_h);
 
 
    void k_set_active_schema(OksFile * file_h);
 
 
    OksFile * get_active_schema() const {return p_active_schema;}
 
 
    const OksFile::Map & schema_files() const {return p_schema_files;}
 
 
    void create_lists_of_updated_schema_files(std::list<OksFile *> ** updated, std::list<OksFile *> ** removed) const;
 
 
    void get_updated_repository_files(std::set<std::string>& updated, std::set<std::string>& added, std::set<std::string>& removed);
 
 
    OksFile * load_data(const std::string& name, bool bind = true);
 
 
    void reload_data(std::set<OksFile *>& files, bool allow_schema_extension = true);
 
 
    OksFile * new_data(const std::string& name, const std::string& logical_name = "", const std::string& type = "");
 
 
    void save_data(OksFile * file_h, bool ignore_bad_objects = false, OksFile * true_file_h = nullptr, bool force_defaults = false);
 
 
    void save_data(OksFile * file_h, const OksObject::FSet& objects);
 
 
    void backup_data(OksFile * pf, const char * suffix = ".bak");
 
 
    void save_as_data(const std::string& new_name, OksFile * file_h);
 
 
    void save_all_data(bool force_defaults=false);
 
 
    void close_data(OksFile * file_h, bool unbind_objects = true);
 
 
    void close_all_data();
 
 
    void set_active_data(OksFile * file_h);
 
 
    void k_set_active_data(OksFile *);
 
 
    OksFile * get_active_data() const {return p_active_data;}
 
 
    const OksFile::Map & data_files() const {return p_data_files;}
 
 
    void create_lists_of_updated_data_files(std::list<OksFile *> ** updated, std::list<OksFile *> ** removed) const;
 
 
    void
    get_modified_files(std::set<OksFile *>& mfs, std::set<OksFile *>& rfs, const std::string& version);
 
 
    const std::list<std::string>& get_repository_dirs() const {return p_repository_dirs;}
 
 
    void
    commit_repository(const std::string& comments, const std::string& credentials = "");
 
 
    void
    tag_repository(const std::string& tag);
 
 
    std::time_t
    get_repository_checkout_ts() const
    {
      return p_repository_checkout_ts;
    }
 
 
    enum RepositoryUpdateType
      {
        DiscardChanges, MergeChanges, NoChanges
      };
 
 
    void
    update_repository(const std::string& hash_val, RepositoryUpdateType update_type)
    {
      update_repository("hash", hash_val, update_type);
    }
 
 
    void
    update_repository(const std::string& param, const std::string& val, RepositoryUpdateType update_type);
 
 
    std::list<std::string>
    get_repository_versions_diff(const std::string& sha1, const std::string& sha2);
 
 
    std::list<std::string>
    get_repository_unmerged_files()
    {
      return get_repository_versions_diff("", "");
    }
 
 
    std::vector<OksRepositoryVersion>
    get_repository_versions(bool skip_irrelevant, const std::string& command_line);
 
 
    std::vector<OksRepositoryVersion>
    get_repository_versions_by_hash(bool skip_irrelevant = true, const std::string& sha1 = "", const std::string& sha2 = "");
 
 
    std::vector<OksRepositoryVersion>
    get_repository_versions_by_date(bool skip_irrelevant = true, const std::string& since = "", const std::string& until = "");
 
 
    std::string
    read_repository_version();
 
 
 
    static const char * get_cwd();
 
    static void reset_cwd() { s_cwd = 0;}
 
 
    static void
    set_use_strict_repository_paths(bool flag)
    {
      p_use_strict_repository_paths = flag;
    }
 
 
    std::string insert_repository_dir(const std::string& dir, bool push_back = true);
 
 
    void remove_repository_dir(const std::string& dir);
 
 
    const OksClass::Map & classes() const {return p_classes;}
 
 
    size_t number_of_classes() const {return p_classes.size();}
 
 
    const OksObject::Set & objects() const {return p_objects;}
 
 
    size_t number_of_objects() const {return p_objects.size();}
 
 
    OksClass * find_class(const std::string& class_name) const { return find_class(class_name.c_str()); }
 
 
    OksClass * find_class(const char * class_name) const;
 
 
    void get_all_classes(const std::vector<std::string>& names_in, ClassSet& classes_out) const;
 
 
    void registrate_all_classes(bool skip_registered = false);
 
 
    bool is_dangling(OksClass * class_ptr) const;
 
 
    bool is_dangling(OksObject * obj_ptr) const;
 
 
    void subscribe_create_class(void (*f)(OksClass *));
 
 
    void subscribe_change_class(void (*f)(OksClass *, OksClass::ChangeType, const void *));
 
 
    void subscribe_delete_class(void (*f)(OksClass *));
 
 
    void subscribe_create_object(OksObject::notify_obj cb_f, void * parameter);
 
 
    void subscribe_change_object(OksObject::notify_obj, void *);
 
 
    void subscribe_delete_object(OksObject::notify_obj, void *);
 
 
    void bind_objects();
 
 
    const std::string& get_bind_objects_status() const noexcept { return p_bind_objects_status; }
 
 
    const std::string& get_bind_classes_status() const noexcept { return p_bind_classes_status; }
 
 
    void
    unset_repository_created();
 
 
  private:
    bool p_silence;
    bool p_verbose;
    bool p_profiling;
    bool p_allow_repository;
    bool p_allow_duplicated_classes;
    bool p_allow_duplicated_objects;
    bool p_test_duplicated_objects_via_inheritance;
 
    static bool p_skip_string_range;
    static bool p_use_strict_repository_paths;
 
    static std::string p_repository_root;
    static std::string p_repository_mapping_dir;
    std::string p_user_repository_root;
    bool p_user_repository_root_inited;
    bool p_user_repository_root_created;
    std::string p_repository_version;
    std::time_t p_repository_checkout_ts;
    std::time_t p_repository_update_ts;
 
 
      // kernel mutexes
 
    mutable std::shared_mutex p_kernel_mutex;
    mutable std::mutex p_objects_mutex;
    std::mutex p_objects_refs_mutex;
    std::shared_mutex p_schema_mutex;
    static std::mutex p_parallel_out_mutex;
 
      // schema and data files members
 
    OksFile::Map p_schema_files;
    OksFile::Map p_data_files;
 
    OksFile * p_active_schema;
    OksFile * p_active_data;
    
    bool p_close_all;
 
 
      // repository directories
 
    std::list<std::string> p_repository_dirs;
 
    static char * s_cwd;
 
 
    OksClass::Map p_classes;
 
    static unsigned long p_count;
 
    OksProfiler * profiler;
 
    OksObject::Set p_objects;
 
    static int p_threads_pool_size;
 
    std::string p_bind_objects_status;
    mutable std::string p_bind_classes_status;
 
    LoadErrors p_load_errors;
 
    std::map<const OksFile *, OksFile *> p_preload_file_info;
    std::vector<OksFile *>  p_preload_added_files;
 
 
      // to be used by the kernel only
 
    std::string create_user_repository_dir();
    void remove_user_repository_dir();
 
    void k_copy_repository(const std::string& source, const std::string& destination);
 
 
    void k_checkout_repository(const std::string& param, const std::string& val, const std::string& branch);
 
 
    OksFile * find_file(const std::string &s, const OksFile::Map& files) const;
    void k_rename_repository_file(OksFile * file_h, const std::string& new_name);
 
    OksFile * k_load_file(const std::string& name, bool bind, const OksFile * parent, OksPipeline * pipeline);
 
    void k_load_includes(const OksFile&, OksPipeline *);
    bool k_preload_includes(OksFile * file_h, std::set<OksFile *>& new_files, bool allow_schema_extension);
    void clear_preload_file_info();
    void restore_preload_file_info();
 
    OksFile * k_load_schema(const std::string&, const OksFile *);
    void k_load_schema(OksFile * fp, std::shared_ptr<OksXmlInputStream> xmls, const OksFile * parent_h);
    void k_close_schema(OksFile *);
    void k_save_schema(OksFile *, bool force = false, OksFile * = 0, const OksClass::Map * = 0);
    void k_rename_schema(OksFile *, const std::string& short_name, const std::string& long_name);
 
    OksFile * k_load_data(const std::string&, bool, const OksFile *, OksPipeline *);
    void k_load_data(OksFile * fp, char format, std::shared_ptr<OksXmlInputStream> xmls, long file_length, bool bind, const OksFile * parent_h, OksPipeline *);
    void k_close_data(OksFile *, bool);
    void k_save_data(OksFile *, bool = false, OksFile * = nullptr, const OksObject::FSet * = nullptr, bool force_defaults = false);
    void k_rename_data(OksFile *, const std::string& short_name, const std::string& long_name);
 
    void k_bind_objects();
    void k_check_bind_classes_status() const noexcept;
 
 
    void unbind_all_rels(const OksObject::FSet& rm_objs, OksObject::FSet& updated) const;
 
    void k_add(OksClass*);
    void k_remove(OksClass*);
 
    void define(OksObject *o) { std::lock_guard lock(p_objects_mutex); p_objects.insert(o);  }
    void undefine(OksObject *o) { if(!p_objects.empty()) { std::lock_guard lock(p_objects_mutex); p_objects.erase(o); } }
 
    void add_data_file(OksFile *);
    void add_schema_file(OksFile *);
    void remove_data_file(OksFile *);
    void remove_schema_file(OksFile *);
 
    bool test_parent(OksFile * file, OksFile::IMap::iterator& i);
 
    OksObject::notify_obj p_create_object_notify_fn;
    void *                p_create_object_notify_param;
 
    OksObject::notify_obj p_change_object_notify_fn;
    void *                p_change_object_notify_param;
 
    OksObject::notify_obj p_delete_object_notify_fn;
    void *                p_delete_object_notify_param;
 
};
 
 
inline void
OksKernel::subscribe_create_object(OksObject::notify_obj f, void * p)
{
  p_create_object_notify_fn = f;
  p_create_object_notify_param = p;
}
 
 
inline void
OksKernel::subscribe_change_object(OksObject::notify_obj f, void * p)
{
  p_change_object_notify_fn = f;
  p_change_object_notify_param = p;
}
 
 
inline void
OksKernel::subscribe_delete_object(OksObject::notify_obj f, void * p)
{
  p_delete_object_notify_fn = f;
  p_delete_object_notify_param = p;
}
 
 
inline void
OksKernel::subscribe_create_class(void (*f)(OksClass *))
{
  OksClass::create_notify_fn = f;
}
 
 
inline void
OksKernel::subscribe_change_class(void (*f)(OksClass *, OksClass::ChangeType, const void *))
{
  OksClass::change_notify_fn = f;
}
 
 
inline void
OksKernel::subscribe_delete_class(void (*f)(OksClass *))
{
  OksClass::delete_notify_fn = f;
}
 
} // namespace oks
} // namespace dunedaq
 
#endif