OksConfiguration.cpp

Go to the documentation of this file.

//
// DUNE DAQ modification notice:
// This file has been modified from the original ATLAS oksconfig source for the DUNE DAQ project.
// Fork baseline commit: oksconfig-03-02-00 (2021-04-20).
// Renamed since fork: no.
//
 
#include <errno.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
 
#include <algorithm>
#include <memory>
 
#include "ers/ers.hpp"
 
#include "oks/file.hpp"
#include "oks/kernel.hpp"
#include "oks/object.hpp"
#include "oks/query.hpp"
#include "oks/relationship.hpp"
#include "oksutils/oks/access.hpp"
 
#include "conffwk/Change.hpp"
#include "conffwk/ConfigObject.hpp"
#include "conffwk/Configuration.hpp"
#include "conffwk/Schema.hpp"
 
#include "oksconflibs/OksConfigObject.hpp"
#include "oksconflibs/OksConfiguration.hpp"
 
namespace dunedaq {
ERS_DECLARE_ISSUE(oksconflibs, Exception, , )
}
using namespace dunedaq;
using namespace dunedaq::oks;
using namespace dunedaq::oksconflibs;
// to be used as plug-in
 
extern "C" conffwk::ConfigurationImpl*
_oksconflibs_creator_(const std::string& spec)
{
  try {
    std::unique_ptr<OksConfiguration> impl(new OksConfiguration());
    if (!spec.empty()) {
      impl->open_db(spec);
    }
    return impl.release();
  } catch (dunedaq::conffwk::Exception& ex) {
    throw dunedaq::conffwk::Generic(ERS_HERE, "oksconflibs initialization error", ex);
  } catch (...) {
    throw dunedaq::conffwk::Generic(ERS_HERE,
                                    "oksconflibs initialization error:\n***** caught unknown exception *****");
  }
}
 
struct dunedaq::oksconflibs::OksConfigurationCheckDB
{
 
  OksConfiguration* m_db;
  bool m_run;
 
  OksConfigurationCheckDB(OksConfiguration* db)
    : m_db(db)
    , m_run(true)
  {
    ;
  }
 
  ~OksConfigurationCheckDB()
  {
    TLOG_DEBUG(3) << "Call destructor of OksConfigurationCheckDB object";
    m_db = nullptr;
  }
 
  void operator()()
  {
    TLOG_DEBUG(2) << "Call user notification";
 
    while (m_run) {
      sleep(1);
      try {
        m_db->check_db();
      } catch (dunedaq::conffwk::Generic& ex) {
        m_db->m_check_db_thread = nullptr;
        m_db->m_check_db_obj = nullptr;
 
        if (getenv("OKSCONFLIBS_NO_RELOAD_ABORT") != nullptr) {
          ers::fatal(dunedaq::conffwk::Generic(ERS_HERE, "database reload has failed, unsubscribing...", ex));
          return;
        } else {
          ers::fatal(dunedaq::conffwk::Generic(ERS_HERE, "database reload has failed, aborting...", ex));
          abort();
        }
      }
    }
 
    TLOG_DEBUG(4) << "Destroy OksConfigurationCheckDB object = " << (void*)this;
 
    delete this;
 
    TLOG_DEBUG(2) << "Exit user notification";
  }
};
 
bool
is_repo_name(const std::string& name)
{
  if (!OksKernel::get_repository_root().empty()) {
    std::string s(name);
 
    Oks::substitute_variables(s);
 
    bool is_absolute_path = (s[0] == '/');
 
    if (!is_absolute_path)
      s = std::string(OksKernel::get_cwd()) + '/' + s;
 
    // if file system path exists
    if (Oks::real_path(s, true)) {
      if (!OksKernel::get_repository_mapping_dir().empty() && s.find(OksKernel::get_repository_mapping_dir()) == 0)
        return true;
 
      static std::string user_repo_dir;
      static std::once_flag flag;
 
      std::call_once(flag, []() {
        if (const char* s = getenv("TDAQ_DB_USER_REPOSITORY")) {
          user_repo_dir = s;
 
          try {
            Oks::real_path(user_repo_dir, false);
          } catch (oks::exception& ex) {
            TLOG_DEBUG(0) << "Failed to read TDAQ_DB_USER_REPOSITORY = \'" << s << "\':\n\tcaused by: " << ex.what();
          }
        }
      });
 
      if (!user_repo_dir.empty() && s.find(user_repo_dir) == 0)
        return true;
 
      return false;
    } else
      return true;
  }
 
  return false;
}
 
void
OksConfiguration::open_db(const std::string& spec_params)
{
  // separate parameters first
 
  std::string::size_type idx = spec_params.find_first_of('&');
 
  std::string data, params;
 
  if (idx == std::string::npos) {
    data = spec_params;
  } else {
    data = spec_params.substr(0, idx);
    params = spec_params.substr(idx + 1);
  }
 
  std::string token;
 
  // guess about oks kernel mode based on the file names
 
  {
    Oks::Tokenizer t(data, ":");
 
    while (!(token = t.next()).empty()) {
      if (is_repo_name(token) == false)
        m_oks_kernel_no_repo = true;
    }
  }
 
  // create OKS kernel if it was not created
 
  if (m_kernel == nullptr) {
    if (!params.empty()) {
      Oks::Tokenizer t(params, ";");
 
      while (!(token = t.next()).empty()) {
        if (token == "norepo")
          m_oks_kernel_no_repo = true;
      }
    }
 
    m_kernel = new OksKernel(m_oks_kernel_silence, false, false, !m_oks_kernel_no_repo);
  }
 
  Oks::Tokenizer t(data, ":");
 
  while (!(token = t.next()).empty()) {
    try {
      m_kernel->load_file(token);
    } catch (oks::exception& e) {
      std::ostringstream text;
      text << "cannot load file \'" << token << "\':\n" << e.what();
      close_db();
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    } catch (...) {
      std::ostringstream text;
      text << "cannot load file \'" << token << '\'';
      close_db();
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    }
  }
}
 
void
OksConfiguration::close_database(bool call_unsubscribe)
{
  clean(); // clean implementation cache
 
  if (m_kernel)
    try {
      if (m_check_db_obj) {
        if (call_unsubscribe)
          unsubscribe();
        else
          m_check_db_obj->m_run = false;
      }
 
      m_kernel->subscribe_create_object(0, 0);
      m_kernel->subscribe_change_object(0, 0);
      m_kernel->subscribe_delete_object(0, 0);
 
      m_kernel->close_all_data();
      m_kernel->close_all_schema();
 
      delete m_kernel;
      m_kernel = 0;
    } catch (oksconflibs::Exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "failed to close database", ex));
    } catch (std::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "failed to close database", ex));
    }
}
 
void
OksConfiguration::create(const std::string& db_name, const std::list<std::string>& includes)
{
  // create OKS kernel if it was not created
 
  if (m_kernel == nullptr) {
    if (is_repo_name(db_name) == false)
      m_oks_kernel_no_repo = true;
 
    m_kernel = new OksKernel(m_oks_kernel_silence, false, false, !m_oks_kernel_no_repo);
  }
 
  // create new data file
 
  try {
    OksFile* h = m_kernel->new_data(db_name);
    m_created_files.insert(h);
 
    // add includes
 
    for (std::list<std::string>::const_iterator i = includes.begin(); i != includes.end(); ++i) {
      try {
        h->add_include_file(*i);
      } catch (oks::exception& e) {
        std::ostringstream text;
        text << "cannot add and load include file \'" << *i << "\' to \'" << db_name << "\':\n" << e.what();
        close_db();
        throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
      } catch (...) {
        std::ostringstream text;
        text << "cannot add and load include file \'" << *i << "\' to \'" << db_name << '\'';
        close_db();
        throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
      }
    }
 
    m_kernel->bind_objects();
  } catch (oks::exception& ex) {
    close_db();
    std::ostringstream text;
    text << "cannot create new data file \'" << db_name << "\': " << ex.what();
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
  } catch (dunedaq::conffwk::Generic& ex) {
    close_db();
    std::ostringstream text;
    text << "cannot create new data file \'" << db_name << "\'";
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str(), ex));
  } catch (...) {
    close_db();
    std::ostringstream text;
    text << "cannot create new data file \'" << db_name << '\'';
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
  }
}
 
static std::string
mk_no_file_error_text(const std::string& db)
{
  std::ostringstream text;
  text << "cannot find file \'" << db << '\'';
  return text.str();
}
 
static std::string
mk_add_include_error_text(const std::string& db, const std::string& include, const char* error = nullptr)
{
  std::ostringstream text;
  text << "cannot add and load include file \'" << include << "\' to \'" << db << '\'';
  if (error)
    text << ":\n" << error;
  return text.str();
}
 
void
OksConfiguration::get_updated_dbs(std::list<std::string>& dbs) const
{
  for (const auto& i : m_kernel->data_files()) {
    if (i.second->is_updated())
      dbs.push_back(i.second->get_full_file_name());
  }
}
 
void
OksConfiguration::set_commit_credentials(const std::string& user, const std::string& password)
{
  m_user = user;
  m_password = password;
}
 
static void
throw_update_exception(const OksFile* file_h, const char* action, const char* reason)
{
  std::ostringstream text;
  text << "cannot " << action << " file \'" << file_h->get_full_file_name() << "\' since " << reason;
  throw dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str());
}
 
void
OksConfiguration::commit(const std::string& log_message)
{
  try {
    m_kernel->set_active_data(nullptr);
  } catch (oks::exception& ex) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, ex.what()));
  }
 
  const bool commit2repo(!m_kernel->get_user_repository_root().empty() && m_kernel->is_user_repository_created());
  unsigned int count(0);
 
  for (const auto& i : m_kernel->data_files()) {
    OksFile* file_h = i.second;
 
    if (file_h->is_updated()) {
      const OksFile::FileStatus file_status = file_h->get_status_of_file();
 
      if (file_status == OksFile::FileModified)
        throw_update_exception(file_h, "save", "it was externally modified");
      else if (file_status == OksFile::FileRemoved)
        throw_update_exception(file_h, "save", "it was externally removed");
 
      try {
        if (!commit2repo && !log_message.empty())
          file_h->add_comment(log_message, m_kernel->get_user_name());
 
        m_kernel->save_data(file_h);
 
        count++;
      } catch (oks::exception& ex) {
        std::ostringstream text;
        text << "cannot save updated data file \'" << *(i.first) << "\':\n" << ex.what();
        throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
      }
    }
  }
 
  if (commit2repo && count) {
    try {
      m_kernel->commit_repository(log_message);
    } catch (oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, ex.what()));
    }
  }
 
  m_created_files.clear();
}
 
namespace dunedaq {
namespace oksconflibs {
class ResubscribeGuard
{
 
public:
  ResubscribeGuard(OksConfiguration& db)
    : m_db(db)
  {
    if (m_db.m_check_db_obj) {
      m_db.unsubscribe();
      m_restart = true;
    } else {
      m_restart = false;
    }
 
    m_db.m_created.clear();
    m_db.m_modified.clear();
    m_db.m_removed.clear();
  }
 
  ~ResubscribeGuard()
  {
    if (m_restart) {
      m_db.subscribe();
    }
  }
 
private:
  OksConfiguration& m_db;
  bool m_restart;
};
 
void
OksConfiguration::abort()
{
  // unsubscribe changes; if there are any, re-subscribe on return from this method
 
  ResubscribeGuard __rg__(*this);
 
  // destroy any newly created files
  for (const auto& i : m_created_files) {
    const std::string file_name(i->get_full_file_name());
 
    TLOG_DEBUG(1) << "destroy created file \'" << file_name << "\')";
    m_kernel->close_data(i);
 
    if (int result = unlink(file_name.c_str())) {
      std::ostringstream text;
      text << "abort changes failed since cannot erase created file \'" << file_name << "\'; unlink failed with code "
           << result << ": " << dunedaq::oks::strerror(errno);
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    }
  }
 
  m_created_files.clear();
 
  // add to "updated" any modified file
  std::set<OksFile*> updated;
 
  for (const auto& i : m_kernel->data_files()) {
    if (i.second->is_updated()) {
      updated.insert(i.second);
      TLOG_DEBUG(2) << "file \'" << i.second->get_full_file_name() << "\' was updated, to be reloaded...";
    }
  }
 
  if (!m_kernel->get_user_repository_root().empty()) {
    std::set<OksFile*> mfs, rfs;
    try {
      m_kernel->get_modified_files(mfs, rfs, "origin/master");
    } catch (const oks::exception& ex) {
      std::ostringstream text;
      text << "failed to get differences between revisions " << m_kernel->get_repository_version()
           << " and origin/master: " << ex.what() << '\n';
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str(), ex));
    }
  }
 
  if (!updated.empty()) {
    try {
      if (!m_kernel->get_user_repository_root().empty())
        m_kernel->update_repository("origin/master", OksKernel::RepositoryUpdateType::DiscardChanges);
 
      m_kernel->reload_data(updated);
    } catch (oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot reload updated data files", ex));
    } catch (...) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot reload updated data files"));
    }
  }
}
 
static std::vector<dunedaq::conffwk::Version>
oks2config(const std::vector<OksRepositoryVersion>& in)
{
  std::vector<dunedaq::conffwk::Version> out;
  out.reserve(in.size());
 
  for (auto& x : in)
    out.emplace_back(x.m_commit_hash, x.m_user, x.m_date, x.m_comment, x.m_files);
 
  return out;
}
 
std::vector<dunedaq::conffwk::Version>
OksConfiguration::get_changes()
{
  if (!m_kernel->get_repository_root().empty()) {
    try {
      return oks2config(m_kernel->get_repository_versions_by_hash(true, m_kernel->get_repository_version()));
    } catch (const oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot get new versions", ex));
    }
  } else {
    std::vector<dunedaq::conffwk::Version> out;
 
    std::set<OksFile*> oks_files;
    m_kernel->get_modified_files(oks_files, oks_files, "");
 
    if (!oks_files.empty()) {
      std::vector<std::string> files;
 
      for (const auto& x : oks_files)
        files.push_back(x->get_well_formed_name());
 
      out.emplace_back("", "", 0, "", files);
    }
 
    return out;
  }
}
 
std::vector<dunedaq::conffwk::Version>
OksConfiguration::get_versions(const std::string& since,
                               const std::string& until,
                               dunedaq::conffwk::Version::QueryType type,
                               bool skip_irrelevant)
{
  if (!m_kernel->get_repository_root().empty()) {
    try {
      return oks2config(type == dunedaq::conffwk::Version::query_by_date
                          ? m_kernel->get_repository_versions_by_date(skip_irrelevant, since, until)
                          : m_kernel->get_repository_versions_by_hash(skip_irrelevant, since, until));
    } catch (const oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot get repository versions", ex));
    }
  } else {
    throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot get versions, repository is not set"));
  }
}
 
}
}
 
dunedaq::oksconflibs::OksConfigObject*
dunedaq::oksconflibs::OksConfiguration::new_object(oks::OksObject* obj) noexcept
{
  return insert_object<dunedaq::oksconflibs::OksConfigObject>(obj, obj->GetId(), obj->GetClass()->get_name());
}
 
bool
OksConfiguration::test_object(const std::string& class_name,
                              const std::string& name,
                              unsigned long /*rlevel*/,
                              const std::vector<std::string>* /*rclasses*/)
{
  // Looking for the class. If it does not exist, we return false.
  OksClass* cl = m_kernel->find_class(class_name);
  if (!cl)
    return false;
 
  // The object exist for this class?
  if (cl->get_object(name))
    return true;
 
  // Trying all subclasses
  if (const OksClass::FList* subclasses = cl->all_sub_classes()) {
    for (OksClass::FList::const_iterator it = subclasses->begin(); it != subclasses->end(); ++it) {
      if ((*it)->get_object(name))
        return true;
    }
  }
  // If we get here, we found the class, but not the object.
  return false;
}
 
void
dunedaq::oksconflibs::OksConfiguration::get(const std::string& class_name,
                                            const std::string& name,
                                            dunedaq::conffwk::ConfigObject& object,
                                            unsigned long /*rlevel*/,
                                            const std::vector<std::string>* /* rclasses */)
{
  if (OksClass* cl = m_kernel->find_class(class_name)) {
    OksObject* obj = cl->get_object(name);
    if (obj == nullptr) {
      // try all subclasses
      if (const OksClass::FList* subclasses = cl->all_sub_classes()) {
        for (const auto& it : *subclasses) {
          obj = it->get_object(name);
          if (obj != nullptr)
            break;
        }
      }
      if (obj == nullptr) {
        const std::string id(name + '@' + class_name);
        throw dunedaq::conffwk::NotFound(ERS_HERE, "object", id.c_str());
      }
    }
 
    object = new_object(obj);
  } else {
    throw dunedaq::conffwk::NotFound(ERS_HERE, "class", class_name.c_str());
  }
}
 
void
OksConfiguration::get(const std::string& class_name,
                      std::vector<conffwk::ConfigObject>& objects,
                      const std::string& query,
                      unsigned long /*rlevel*/,
                      const std::vector<std::string>* /* rclasses */)
{
  if (OksClass* cl = m_kernel->find_class(class_name)) {
    objects.clear();
 
    OksObject::List* objs = 0;
 
    if (query.empty()) {
      objs = cl->create_list_of_all_objects();
    } else {
      try {
        std::unique_ptr<OksQuery> qe(new OksQuery(cl, query.c_str()));
        if (qe->good() == false) {
          std::ostringstream text;
          text << "bad query syntax \"" << query << "\" in scope of class \"" << class_name << '\"';
          throw dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str());
        }
        objs = cl->execute_query(qe.get()); // FIXME: check .get()
      } catch (oks::exception& ex) {
        std::ostringstream text;
        text << "failed to execute query:\n" << ex.what();
        throw dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str());
      }
    }
 
    if (objs) {
      for (OksObject::List::iterator i = objs->begin(); i != objs->end(); ++i) {
        objects.push_back(new_object(*i));
      }
      delete objs;
    }
  } else {
    throw dunedaq::conffwk::NotFound(ERS_HERE, "class", class_name.c_str());
  }
}
 
void
OksConfiguration::get(const conffwk::ConfigObject& obj_from,
                      const std::string& query,
                      std::vector<conffwk::ConfigObject>& objects,
                      unsigned long /*rlevel*/,
                      const std::vector<std::string>* /* rclasses */)
{
  if (obj_from.is_null()) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, "parameter \'obj_from\' is (null)"));
  }
 
  OksObject* o = (static_cast<const OksConfigObject*>(obj_from.implementation()))->m_obj;
 
  try {
    oks::QueryPath q(query, *m_kernel);
    if (OksObject::List* objs = o->find_path(q)) {
      for (OksObject::List::iterator i = objs->begin(); i != objs->end(); ++i) {
        objects.push_back(new_object(*i));
      }
      delete objs;
    }
  } catch (oks::bad_query_syntax& e) {
    std::ostringstream text;
    text << "bad path-query \"" << query << "\" to object " << o;
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
  }
}
 
dunedaq::conffwk::class_t*
OksConfiguration::get(const std::string& class_name, bool direct_only)
{
  OksClass* c = m_kernel->find_class(class_name);
 
  if (c == 0) {
    throw dunedaq::conffwk::NotFound(ERS_HERE, "class", class_name.c_str());
  }
 
  dunedaq::conffwk::class_t* d = new dunedaq::conffwk::class_t(
    c->get_name(), c->get_description(), c->get_file()->get_full_file_name(), c->get_is_abstract());
 
  if (direct_only) {
    if (const std::list<std::string*>* classes = c->direct_super_classes()) {
      for (const auto& i : *classes) {
        const_cast<std::vector<std::string>&>(d->p_superclasses).push_back(*i);
      }
    }
  } else {
    if (const OksClass::FList* classes = c->all_super_classes()) {
      for (const auto& i : *classes) {
        const_cast<std::vector<std::string>&>(d->p_superclasses).push_back(i->get_name());
      }
    }
  }
 
  if (const OksClass::FList* subclasses = c->all_sub_classes()) {
    for (const auto& i : *subclasses) {
      if (direct_only == false || i->has_direct_super_class(c->get_name()) == true) {
        const_cast<std::vector<std::string>&>(d->p_subclasses).push_back(i->get_name());
      }
    }
  }
 
  if (const std::list<OksAttribute*>* attrs = (direct_only ? c->direct_attributes() : c->all_attributes())) {
    for (const auto& i : *attrs) {
      OksAttribute::Format format = i->get_format();
      OksData::Type type = i->get_data_type();
 
      const_cast<std::vector<dunedaq::conffwk::attribute_t>&>(d->p_attributes)
        .push_back(dunedaq::conffwk::attribute_t(
          i->get_name(),
          (type == OksData::string_type    ? dunedaq::conffwk::string_type
           : type == OksData::enum_type    ? dunedaq::conffwk::enum_type
           : type == OksData::bool_type    ? dunedaq::conffwk::bool_type
           : type == OksData::s8_int_type  ? dunedaq::conffwk::s8_type
           : type == OksData::u8_int_type  ? dunedaq::conffwk::u8_type
           : type == OksData::s16_int_type ? dunedaq::conffwk::s16_type
           : type == OksData::u16_int_type ? dunedaq::conffwk::u16_type
           : type == OksData::s32_int_type ? dunedaq::conffwk::s32_type
           : type == OksData::u32_int_type ? dunedaq::conffwk::u32_type
           : type == OksData::s64_int_type ? dunedaq::conffwk::s64_type
           : type == OksData::u64_int_type ? dunedaq::conffwk::u64_type
           : type == OksData::float_type   ? dunedaq::conffwk::float_type
           : type == OksData::double_type  ? dunedaq::conffwk::double_type
           : type == OksData::date_type    ? dunedaq::conffwk::date_type
           : type == OksData::time_type    ? dunedaq::conffwk::time_type
                                           : dunedaq::conffwk::class_type),
          i->get_range(),
          (i->is_integer() == false ? dunedaq::conffwk::na_int_format
                                    : (format == OksAttribute::Dec   ? dunedaq::conffwk::dec_int_format
                                       : format == OksAttribute::Hex ? dunedaq::conffwk::hex_int_format
                                                                     : dunedaq::conffwk::oct_int_format)),
          i->get_is_no_null(),
          i->get_is_multi_values(),
          i->get_init_value(),
          i->get_description()));
    }
  }
 
  if (const std::list<OksRelationship*>* rels = (direct_only ? c->direct_relationships() : c->all_relationships())) {
    for (const auto& i : *rels) {
      const_cast<std::vector<dunedaq::conffwk::relationship_t>&>(d->p_relationships)
        .push_back(dunedaq::conffwk::relationship_t(i->get_name(),
                                                    i->get_type(),
                                                    (i->get_low_cardinality_constraint() == OksRelationship::Zero),
                                                    (i->get_high_cardinality_constraint() == OksRelationship::Many),
                                                    i->get_is_composite(),
                                                    i->get_description()));
    }
  }
 
  return d;
}
 
void
OksConfiguration::get_superclasses(conffwk::fmap<conffwk::fset>& schema)
{
  schema.clear();
 
  if (!m_kernel) {
    return; // throw ( dunedaq::conffwk::Generic( ERS_HERE, "database is not loaded" ) );
  }
 
  schema.reserve(m_kernel->classes().size() * 3);
 
  for (OksClass::Map::const_iterator i = m_kernel->classes().begin(); i != m_kernel->classes().end(); ++i) {
    if (const OksClass::FList* scl = i->second->all_super_classes()) {
      const std::string* name = &conffwk::DalFactory::instance().get_known_class_name_ref(i->second->get_name());
 
      auto& subclasses = schema[name];
 
      if (!scl->empty()) {
        subclasses.reserve(scl->size() * 3);
        for (OksClass::FList::const_iterator j = scl->begin(); j != scl->end(); ++j) {
          subclasses.insert(&conffwk::DalFactory::instance().get_known_class_name_ref((*j)->get_name()));
        }
      }
    }
  }
}
 
void
OksConfiguration::create(OksFile* at,
                         const std::string& class_name,
                         const std::string& id,
                         conffwk::ConfigObject& object)
{
  try {
    m_kernel->set_active_data(at);
  } catch (oks::exception& ex) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, ex.what()));
  }
 
  OksClass* c = m_kernel->find_class(class_name);
 
  if (c == nullptr) {
    std::ostringstream text;
    text << "cannot find class \"" << class_name << '\"';
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
  } else if (id.empty() == false) {
    if (OksObject* obj = c->get_object(id)) {
      std::ostringstream text;
      text << "object \"" << id << '@' << class_name << "\" already exists in file \""
           << obj->get_file()->get_full_file_name() << '\"';
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    }
  }
 
  if (m_kernel->get_test_duplicated_objects_via_inheritance_mode() == false) {
    m_kernel->set_test_duplicated_objects_via_inheritance_mode(true);
    m_kernel->registrate_all_classes();
  }
 
  try {
    object = new_object(new OksObject(c, id.c_str()));
  } catch (oks::exception& ex) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, "cannot create configuration object", ex));
  }
}
 
void
OksConfiguration::create(const std::string& at,
                         const std::string& class_name,
                         const std::string& id,
                         conffwk::ConfigObject& object)
{
  if (at.empty() == true) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, "parameter \'at\' (i.e. filename) cannot be empty"));
  }
 
  if (OksFile* h = m_kernel->find_data_file(at)) {
    return create(h, class_name, id, object);
  } else {
    std::ostringstream text;
    text << "file \"" << at << "\" is not loaded";
    throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
  }
}
 
void
OksConfiguration::create(const conffwk::ConfigObject& at,
                         const std::string& class_name,
                         const std::string& id,
                         conffwk::ConfigObject& object)
{
  create((static_cast<const OksConfigObject*>(at.implementation()))->m_obj->get_file(), class_name, id, object);
}
 
void
OksConfiguration::create_notify(OksObject* o, void* p) noexcept
{
  reinterpret_cast<OksConfiguration*>(p)->m_created.push_back(o);
}
 
void
OksConfiguration::change_notify(OksObject* o, void* p) noexcept
{
  reinterpret_cast<OksConfiguration*>(p)->m_modified.insert(o);
}
 
void
OksConfiguration::delete_notify(OksObject* o, void* p) noexcept
{
  reinterpret_cast<OksConfiguration*>(p)->m_removed[o->GetClass()->get_name()].insert(o->GetId());
}
 
struct InheritanceData
{
 
  InheritanceData(const OksKernel&);
 
  const OksConfiguration::SMap& data() const { return m_data; }
 
  OksConfiguration::SMap m_data;
};
 
InheritanceData::InheritanceData(const OksKernel& kernel)
{
  for (OksClass::Map::const_iterator i = kernel.classes().begin(); i != kernel.classes().end(); ++i)
    if (const OksClass::FList* slist = i->second->all_super_classes()) {
      std::set<std::string>& data(m_data[i->second->get_name()]);
      for (const auto& j : *slist)
        data.insert(j->get_name());
    }
}

 
static void
check(std::vector<conffwk::ConfigurationChange*>& changes,
      const InheritanceData& inheritance,
      const std::set<std::string>& class_names,
      const OksConfiguration::SMap& objects,
      const std::string& obj_class,
      const std::string& obj_id,
      const char action)
{
 
  bool any = (class_names.empty() && objects.empty());
 
  // check subscription on objects
  //
  // omi iterator in non-null, if on an object of class 'obj_class'
  // there is explicit subscription
 
  OksConfiguration::SMap::const_iterator omi = objects.end();
 
  if (action != '+') {
    omi = objects.find(obj_class);
    if (omi != objects.end()) {
      std::set<std::string>::const_iterator i = omi->second.find(obj_id);
 
      if (i != omi->second.end()) {
        conffwk::ConfigurationChange::add(changes, obj_class, obj_id, action);
      }
    }
  }
 
  // process the class
 
  if (action == '+' || omi == objects.end()) {
    if (any || class_names.find(obj_class) != class_names.end()) {
      conffwk::ConfigurationChange::add(changes, obj_class, obj_id, action);
    }
  }
 
  // process subclasses
 
  OksConfiguration::SMap::const_iterator it = inheritance.data().find(obj_class);
  if (it != inheritance.data().end()) {
    for (std::set<std::string>::const_iterator j = it->second.begin(); j != it->second.end(); ++j) {
      omi = (action != '+' ? objects.find(*j) : objects.end());
 
      // 1. add objects if criteria is 'any'
      // 2. add object of class which has subscription and has no explicit object subscription
      // 3. add explicitly subscribed object
 
      if (any || (omi == objects.end() && class_names.find(*j) != class_names.end()) ||
          (omi != objects.end() && omi->second.find(obj_id) != omi->second.end())) {
        conffwk::ConfigurationChange::add(changes, *j, obj_id, action);
      }
    }
  }
}
 
class DestroyGuard1
{
 
public:
  DestroyGuard1(OksKernel& kernel)
    : m_kernel(kernel)
  {
    ;
  }
 
  ~DestroyGuard1() { m_kernel.subscribe_delete_object(0, 0); }
 
private:
  OksKernel& m_kernel;
};
 
class DestroyGuard2
{
 
public:
  DestroyGuard2(OksConfiguration::SMap& removed)
    : m_removed(removed)
  {
    ;
  }
 
  ~DestroyGuard2() { m_removed.clear(); }
 
private:
  OksConfiguration::SMap& m_removed;
};
 
void
OksConfiguration::destroy(conffwk::ConfigObject& obj)
{
  OksObject* o((static_cast<const OksConfigObject*>(obj.implementation()))->m_obj);
 
  DestroyGuard2 dg2(m_removed); // => on exit from method, call m_removed.clear();
 
  InheritanceData inheritance(*m_kernel);
 
  try {
    m_kernel->subscribe_delete_object(delete_notify, reinterpret_cast<void*>(this));
    DestroyGuard1 dg1(*m_kernel); // => on exit from try block, call m_kernel->subscribe_delete_object(0, 0);
    OksObject::destroy(o);
  } catch (oks::exception& ex) {
    throw(dunedaq::conffwk::Generic(ERS_HERE, ex.what()));
  }
 
  if (m_conf) {
    std::vector<conffwk::ConfigurationChange*> changes;
 
    for (SMap::iterator i = m_removed.begin(); i != m_removed.end(); ++i) {
      for (std::set<std::string>::const_iterator j = i->second.begin(); j != i->second.end(); ++j) {
        check(changes, inheritance, m_class_names, m_objects, i->first, *j, '-');
      }
    }
 
    if (!changes.empty()) {
      m_conf->update_cache(changes);
      conffwk::ConfigurationChange::clear(changes);
    }
  }
}
 
bool
OksConfiguration::is_writable(const std::string& db_name)
{
  if (OksFile* h = m_kernel->find_data_file(db_name)) {
    try {
      if (!m_kernel->get_user_repository_root().empty()) {
        return oksutils::access::is_writable(*h, OksKernel::get_user_name());
      } else {
        return !OksKernel::check_read_only(h);
      }
    } catch (const std::exception& ex) {
      std::ostringstream text;
      text << "cannot check access status of file \'" << db_name << "\': " << ex.what();
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    }
  } else {
    throw(dunedaq::conffwk::Generic(ERS_HERE, mk_no_file_error_text(db_name).c_str()));
  }
}
 
static std::string
mk_rm_include_error_text(const std::string& db, const std::string& include, const char* error = 0)
{
  std::ostringstream text;
  text << "cannot remove include file \'" << include << "\' from \'" << db << '\'';
  if (error)
    text << ":\n" << error;
  return text.str();
}
 
void
OksConfiguration::add_include(const std::string& db_name, const std::string& include)
{
  if (OksFile* h = m_kernel->find_data_file(db_name)) {
    try {
      h->add_include_file(include);
    } catch (dunedaq::conffwk::Generic& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_add_include_error_text(db_name, include).c_str()), ex);
    } catch (oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_add_include_error_text(db_name, include, ex.what()).c_str()));
    } catch (...) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_add_include_error_text(db_name, include).c_str()));
    }
  } else {
    throw(dunedaq::conffwk::Generic(ERS_HERE, mk_no_file_error_text(db_name).c_str()));
  }
}
 
void
OksConfiguration::remove_include(const std::string& db_name, const std::string& include)
{
  InheritanceData inheritance(*m_kernel);
 
  DestroyGuard2 dg2(m_removed); // => on exit from method, call m_removed.clear();
 
  if (OksFile* h = m_kernel->find_data_file(db_name)) {
    try {
      m_kernel->subscribe_delete_object(delete_notify, reinterpret_cast<void*>(this));
      DestroyGuard1 dg1(*m_kernel); // => on exit from try block, call m_kernel->subscribe_delete_object(0, 0);
      h->remove_include_file(include);
    } catch (dunedaq::conffwk::Generic& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_rm_include_error_text(db_name, include).c_str()), ex);
    } catch (oks::exception& ex) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_rm_include_error_text(db_name, include, ex.what()).c_str()));
    } catch (...) {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_rm_include_error_text(db_name, include).c_str()));
    }
  } else {
    throw(dunedaq::conffwk::Generic(ERS_HERE, mk_no_file_error_text(db_name).c_str()));
  }
 
  std::vector<conffwk::ConfigurationChange*> changes;
 
  for (SMap::iterator i = m_removed.begin(); i != m_removed.end(); ++i) {
    for (std::set<std::string>::const_iterator j = i->second.begin(); j != i->second.end(); ++j) {
      check(changes, inheritance, m_class_names, m_objects, i->first, *j, '-');
    }
  }
 
  if (!changes.empty()) {
    m_conf->update_cache(changes);
    conffwk::ConfigurationChange::clear(changes);
  }
}
 
void
OksConfiguration::get_includes(const std::string& db_name, std::list<std::string>& includes) const
{
  if (db_name.empty()) {
    for (auto& i : m_kernel->data_files()) {
      if (i.second->get_parent() == nullptr)
        includes.push_back(*i.first);
    }
  } else {
    if (OksFile* h = m_kernel->find_data_file(db_name)) {
      for (auto& i : h->get_include_files()) {
        includes.push_back(i);
      }
    } else {
      throw(dunedaq::conffwk::Generic(ERS_HERE, mk_no_file_error_text(db_name).c_str()));
    }
  }
}
 
void
OksConfiguration::check_db()
{
  static const std::string version("origin/master");
 
  if (m_fn) {
    std::set<OksFile*> ufs, rfs;
 
    try {
      m_kernel->get_modified_files(ufs, rfs, version);
      m_repo_error_count = 0;
    } catch (const oks::RepositoryOperationFailed& ex) {
      std::ostringstream text;
      text << "cannot get modified repository files (attempt " << ++m_repo_error_count << "): " << ex.what();
      ers::error(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    } catch (const oks::exception& ex) {
      std::ostringstream text;
      text << "cannot get modified files: " << ex.what();
      throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
    }
 
    for (const auto& x : ufs) {
      if (x->get_oks_format() == "schema") {
        std::ostringstream text;
        text << "reload of schema files is not supported (\'" << x->get_well_formed_name() << "\')";
        throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
      }
    }
 
    if (ufs.empty() == false) {
      InheritanceData inheritance(*m_kernel);
 
      m_kernel->subscribe_create_object(create_notify, reinterpret_cast<void*>(this));
      m_kernel->subscribe_change_object(change_notify, reinterpret_cast<void*>(this));
      m_kernel->subscribe_delete_object(delete_notify, reinterpret_cast<void*>(this));
 
      (*m_pre_fn)(m_conf);
 
      try {
        if (!m_kernel->get_user_repository_root().empty())
          m_kernel->update_repository(version, OksKernel::RepositoryUpdateType::DiscardChanges);
 
        m_kernel->reload_data(ufs, false);
      } catch (oks::exception& ex) {
        close_database(false);
        m_fn = 0;
        std::ostringstream text;
        text << "failed to reload updated files:\n" << ex.what();
        throw(dunedaq::conffwk::Generic(ERS_HERE, text.str().c_str()));
      } catch (...) {
        close_database(false);
        m_fn = 0;
        throw(dunedaq::conffwk::Generic(ERS_HERE, "failed to reload updated files"));
      }
 
      m_kernel->subscribe_create_object(0, 0);
      m_kernel->subscribe_change_object(0, 0);
      m_kernel->subscribe_delete_object(0, 0);
 
      std::vector<conffwk::ConfigurationChange*> changes;
 
      {
        for (auto& i : m_created) {
          check(changes, inheritance, m_class_names, m_objects, i->GetClass()->get_name(), i->GetId(), '+');
 
          std::set<OksObject*>::iterator j = m_modified.find(i);
          if (j != m_modified.end()) {
            TLOG_DEBUG(1) << "created object " << i << " appears in \'modified\' set, removing...";
            m_modified.erase(j);
          }
        }
      }
 
      {
        for (auto& i : m_modified) {
          if (m_kernel->is_dangling(i)) {
            TLOG_DEBUG(1) << "object " << (void*)(i) << " is dangling, ignore...";
          } else {
            check(changes, inheritance, m_class_names, m_objects, i->GetClass()->get_name(), i->GetId(), '~');
          }
        }
      }
 
      {
        for (auto& i : m_removed) {
          for (auto& j : i.second) {
            check(changes, inheritance, m_class_names, m_objects, i.first, j, '-');
          }
        }
      }
 
      if (!changes.empty()) {
        (*m_fn)(changes, m_conf);
        conffwk::ConfigurationChange::clear(changes);
      }
 
      m_created.clear();
      m_modified.clear();
      m_removed.clear();
    }
  } else {
    throw(dunedaq::conffwk::Generic(ERS_HERE, "no subscription has been done"));
  }
}
 
//
// Subscribe on changes by class name
//
 
void
OksConfiguration::subscribe(const std::set<std::string>& class_names,
                            const SMap& objs,
                            ConfigurationImpl::notify cb,
                            ConfigurationImpl::pre_notify pre_cb)
{
  m_fn = cb;
  m_pre_fn = pre_cb;
  m_class_names = class_names;
  m_objects = objs;
 
  subscribe();
}
 
// Start subscription thread
 
void
OksConfiguration::subscribe()
{
  if (m_check_db_obj == 0) {
    TLOG_DEBUG(2) << "starting CheckDB thread ...";
    m_check_db_obj = new OksConfigurationCheckDB(this);
    m_check_db_thread = new std::thread(std::ref(*m_check_db_obj));
  }
}
 
//
// Unsubscribe
//
 
void
OksConfiguration::unsubscribe()
{
  if (m_check_db_obj) {
    TLOG_DEBUG(2) << "stopping CheckDB thread ...";
    m_check_db_obj->m_run = false;
    m_check_db_thread->join(); // wait thread termination
    delete m_check_db_thread;
    m_check_db_thread = 0;
    m_check_db_obj = 0;
    TLOG_DEBUG(2) << "the CheckDB thread has been terminated";
  }
}
 
void
OksConfiguration::print_profiling_info() noexcept
{
  std::cout << "OksConfiguration profiler report:\n"
               "  number of loaded schema files: "
            << (m_kernel ? m_kernel->schema_files().size() : 0)
            << "\n"
               "  number of loaded data files: "
            << (m_kernel ? m_kernel->data_files().size() : 0)
            << "\n"
               "  number of loaded classes: "
            << (m_kernel ? m_kernel->classes().size() : 0)
            << "\n"
               "  number of loaded objects: "
            << (m_kernel ? m_kernel->objects().size() : 0) << std::endl;
}
 
// read OksKernel parameters from environment (silence mode)
 
void
OksConfiguration::init_env()
{
  m_oks_kernel_silence = (getenv("OKSCONFLIBS_NO_KERNEL_SILENCE") == nullptr);
}