dunedaq::snbmodules::GroupMetadata Class Reference

#include <group_metadata.hpp>

Inheritance diagram for dunedaq::snbmodules::GroupMetadata:

Collaboration diagram for dunedaq::snbmodules::GroupMetadata:

Public Member Functions
void	generate_metadata_file (std::filesystem::path dest) override
	Generaete metadata file to dest.
void	load_metadata_from_meta_file (std::filesystem::path src) override
	Load metadata file from src.
std::string	export_to_string () override
	Export metadata to string (json format)
void	from_string (const std::string &) override
	Import metadata from string (json format)
status_type::e_status	get_group_status () const
	Return a status of the group transfer based on the status of his transfers, The highest status in priority order is returned.
	GroupMetadata (std::string group_id, std::string source, const IPFormat &source_ip, protocol_type::e_protocol_type protocol, const nlohmann::json &protocol_options=nlohmann::json(), const std::vector< std::shared_ptr< TransferMetadata > > &transfers_meta=std::vector< std::shared_ptr< TransferMetadata > >())
	Constructor.
	GroupMetadata (const std::filesystem::path &src, bool is_path=true)
	Load from file constructor.
virtual	~GroupMetadata ()=default
bool	operator== (MetadataAbstract const &other) const override
	Operator ==.
bool	operator< (MetadataAbstract const &other) const override
	Operator <.
void	set_group_id (std::string transfer_id)
void	set_protocol (protocol_type::e_protocol_type protocol)
void	set_transfers_meta (std::vector< std::shared_ptr< TransferMetadata > > files_meta)
void	set_protocol_options (nlohmann::json protocol_options)
void	set_source_id (std::string source_id)
void	set_expected_files (std::set< std::string > expected_files)
TransferMetadata &	add_file (std::shared_ptr< TransferMetadata > meta)
void	add_expected_file (const std::filesystem::path &file)
std::string	get_group_id () const
protocol_type::e_protocol_type	get_protocol () const
std::vector< std::shared_ptr< TransferMetadata > > &	get_transfers_meta ()
const std::vector< std::shared_ptr< TransferMetadata > > &	get_transfers_meta () const
TransferMetadata &	get_transfer_meta_from_file_path (const std::string &file_path)
const std::set< std::string > &	get_expected_files () const
nlohmann::json	get_protocol_options () const
std::string	get_source_id () const
IPFormat	get_source_ip () const
std::string	to_string () const
Public Member Functions inherited from dunedaq::snbmodules::MetadataAbstract
virtual	~MetadataAbstract ()=default

Static Public Attributes
static const std::string	m_file_extension = ".gmetadata"

Private Attributes
std::string	m_group_id
	Unique identifier for the transfer.
protocol_type::e_protocol_type	m_protocol
	Protocol used for the transfer, every files in the transfer must use the same protocol.
nlohmann::json	m_protocol_options
	Specific options for the selected protocol.
std::vector< std::shared_ptr< TransferMetadata > >	m_transfers_meta
	Set of files to transfer.
std::set< std::string >	m_expected_files
	Set of expected files metadata to add later.
std::string	m_source_id
	Get source id.
IPFormat	m_source_ip
	Get source ip.

Detailed Description

Definition at line 25 of file group_metadata.hpp.

Constructor & Destructor Documentation

GroupMetadata() [1/2]

dunedaq::snbmodules::GroupMetadata::GroupMetadata ( std::string group_id, std::string source, const IPFormat & source_ip, protocol_type::e_protocol_type protocol, const nlohmann::json & protocol_options = nlohmann::json(), const std::vector< std::shared_ptr< TransferMetadata > > & transfers_meta = std::vector<std::shared_ptr<TransferMetadata>>() )

inline

Constructor.

Parameters

protocol
transfers_meta
transfer_id

Definition at line 54 of file group_metadata.hpp.

    : m_group_id(std::move(group_id))
    , m_protocol(protocol)
    , m_transfers_meta(transfers_meta)
    , m_expected_files(std::set<std::string>())
    , m_source_id(std::move(source))
    , m_source_ip(source_ip)
  {
    set_protocol_options(protocol_options);
  }

GroupMetadata() [2/2]

dunedaq::snbmodules::GroupMetadata::GroupMetadata ( const std::filesystem::path & src, bool is_path = true )

inlineexplicit

Load from file constructor.

Definition at line 72 of file group_metadata.hpp.

  {
    if (is_path) {
      load_metadata_from_meta_file(src);
    } else {
      from_string(src.string());
    }
  }

~GroupMetadata()

virtual dunedaq::snbmodules::GroupMetadata::~GroupMetadata ( )

virtualdefault

Member Function Documentation

add_expected_file()

void dunedaq::snbmodules::GroupMetadata::add_expected_file ( const std::filesystem::path & file )

inline

Definition at line 115 of file group_metadata.hpp.

  {
    // remove all occurences of ./ in the file path
    std::string file_path_str = file.string();
    std::string x = "./";
 
    size_t pos = 0;
    while (true) {
      pos = file_path_str.find(x, pos);
      if (pos == std::string::npos) {
        break;
      }
 
      file_path_str.replace(pos, x.length(), "");
    }
 
    m_expected_files.insert(std::filesystem::absolute(file_path_str));
  }

add_file()

TransferMetadata & dunedaq::snbmodules::GroupMetadata::add_file

(

std::shared_ptr< TransferMetadata >

meta

)

Definition at line 36 of file group_metadata.cpp.

{
  if (m_expected_files.find(meta->get_file_path()) != m_expected_files.end()) {
    m_expected_files.erase(meta->get_file_path());
    meta->set_group_id(m_group_id);
 
    return *m_transfers_meta.emplace_back(meta);
  } else if (meta->get_group_id() == m_group_id) {
    int pos = -1;
    int i = 0;
    for (const auto& m : m_transfers_meta) {
      if (m == meta) {
        // Already inserted, update the transfer
        pos = i;
        break;
      }
      i++;
    }
    if (pos != -1) {
      m_transfers_meta.erase(m_transfers_meta.begin() + pos);
    }
    return *m_transfers_meta.emplace_back(std::move(meta));
  } else {
    ers::fatal(MetadataNotExpectedInGroupError(ERS_HERE, m_group_id, meta->get_file_name()));
    return *m_transfers_meta[0]; // To avoid warning
  }
}

export_to_string()

std::string dunedaq::snbmodules::GroupMetadata::export_to_string ( )

overridevirtual

Export metadata to string (json format)

Returns

Metadata in string format

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 65 of file group_metadata.cpp.

{
  nlohmann::json j;
  j["transfer_id"] = get_group_id();
  j["source_id"] = get_source_id();
  j["source_ip"] = get_source_ip().get_ip_port();
  j["protocol"] = protocol_type::protocols_to_string(get_protocol());
  j["protocol_options"] = get_protocol_options().dump();
 
  std::vector<std::string> files;
  for (const auto& file : get_transfers_meta()) {
    files.push_back(file->get_file_path().string());
  }
  j["files"] = files;
 
  return j.dump();
}

from_string()

void dunedaq::snbmodules::GroupMetadata::from_string ( const std::string & s )

overridevirtual

Import metadata from string (json format)

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 84 of file group_metadata.cpp.

{
  nlohmann::json j = nlohmann::json::parse(str);
 
  TLOG() << "debug : Loading metadata from string " << str;
 
  if (j.contains("transfer_id")) {
    set_group_id(j["transfer_id"].get<std::string>());
  }
  if (j.contains("source_id")) {
    set_source_id(j["source_id"].get<std::string>());
  }
  if (j.contains("source_ip")) {
    m_source_ip = IPFormat(j["source_ip"].get<std::string>());
  }
  if (j.contains("protocol")) {
    set_protocol(protocol_type::string_to_protocols(j["protocol"].get<std::string>()).value());
  }
  if (j.contains("protocol_options")) {
    set_protocol_options(nlohmann::json::parse(j["protocol_options"].get<std::string>()));
  }
  if (j.contains("files")) {
    auto files = j["files"].get<std::vector<std::filesystem::path>>();
 
    for (const auto& file : files) {
      add_expected_file(file);
    }
  }
}

generate_metadata_file()

void dunedaq::snbmodules::GroupMetadata::generate_metadata_file ( std::filesystem::path dest )

overridevirtual

Generaete metadata file to dest.

Parameters

dest

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 115 of file group_metadata.cpp.

{
  std::ofstream metadata_file;
  metadata_file.open(dest.append(get_group_id()).string() + m_file_extension);
 
  metadata_file << export_to_string();
 
  metadata_file.close();
}

get_expected_files()

const std::set< std::string > & dunedaq::snbmodules::GroupMetadata::get_expected_files ( ) const

inline

Definition at line 140 of file group_metadata.hpp.

{ return m_expected_files; }

get_group_id()

std::string dunedaq::snbmodules::GroupMetadata::get_group_id ( ) const

inline

Definition at line 135 of file group_metadata.hpp.

{ return m_group_id; }

get_group_status()

status_type::e_status dunedaq::snbmodules::GroupMetadata::get_group_status ( ) const

inline

Return a status of the group transfer based on the status of his transfers, The highest status in priority order is returned.

Returns

Status of the group transfer

Definition at line 39 of file group_metadata.hpp.

  {
    status_type::e_status status = status_type::e_status::PREPARING;
    for (const auto& transfer : get_transfers_meta()) {
      if (transfer->get_status() > status) {
        status = transfer->get_status();
      }
    }
    return status;
  }

get_protocol()

protocol_type::e_protocol_type dunedaq::snbmodules::GroupMetadata::get_protocol ( ) const

inline

Definition at line 136 of file group_metadata.hpp.

{ return m_protocol; }

get_protocol_options()

nlohmann::json dunedaq::snbmodules::GroupMetadata::get_protocol_options ( ) const

inline

Definition at line 141 of file group_metadata.hpp.

{ return m_protocol_options; }

get_source_id()

std::string dunedaq::snbmodules::GroupMetadata::get_source_id ( ) const

inline

Definition at line 142 of file group_metadata.hpp.

{ return m_source_id; }

get_source_ip()

IPFormat dunedaq::snbmodules::GroupMetadata::get_source_ip ( ) const

inline

Definition at line 143 of file group_metadata.hpp.

{ return m_source_ip; }

get_transfer_meta_from_file_path()

TransferMetadata & dunedaq::snbmodules::GroupMetadata::get_transfer_meta_from_file_path

(

const std::string &

file_path

)

Definition at line 24 of file group_metadata.cpp.

{
  for (std::shared_ptr<TransferMetadata> meta : get_transfers_meta()) {
    if (meta->get_file_path() == file_path) {
      return *meta;
    }
  }
  ers::fatal(MetadataNotFoundInGroupError(ERS_HERE, m_group_id, file_path));
  return *m_transfers_meta[0]; // To avoid warning
}

get_transfers_meta() [1/2]

std::vector< std::shared_ptr< TransferMetadata > > & dunedaq::snbmodules::GroupMetadata::get_transfers_meta ( )

inline

Definition at line 137 of file group_metadata.hpp.

{ return m_transfers_meta; }

get_transfers_meta() [2/2]

const std::vector< std::shared_ptr< TransferMetadata > > & dunedaq::snbmodules::GroupMetadata::get_transfers_meta ( ) const

inline

Definition at line 138 of file group_metadata.hpp.

{ return m_transfers_meta; }

load_metadata_from_meta_file()

void dunedaq::snbmodules::GroupMetadata::load_metadata_from_meta_file ( std::filesystem::path src )

overridevirtual

Load metadata file from src.

Parameters

src

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 126 of file group_metadata.cpp.

{
  std::ifstream metadata_file;
  try {
    metadata_file.open(src.string());
  } catch (const std::exception& e) {
    ers::error(MetadataFileNotFoundError(ERS_HERE, e.what()));
    return;
  }
 
  std::stringstream buffer;
  buffer << metadata_file.rdbuf();
 
  from_string(buffer.str());
 
  metadata_file.close();
}

operator<()

bool dunedaq::snbmodules::GroupMetadata::operator< ( MetadataAbstract const & other ) const

inlineoverridevirtual

Operator <.

Parameters

other

Other GroupMetadata to compare

Returns

True if the transfer_id is less than the other transfer_id, false otherwise

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 95 of file group_metadata.hpp.

  {
    auto o = dynamic_cast<const GroupMetadata&>(other);
    return m_group_id.compare(o.m_group_id);
  }

operator==()

bool dunedaq::snbmodules::GroupMetadata::operator== ( MetadataAbstract const & other ) const

inlineoverridevirtual

Operator ==.

Parameters

other

Other GroupMetadata to compare

Returns

True if the transfer_id are equal, false otherwise

Implements dunedaq::snbmodules::MetadataAbstract.

Definition at line 86 of file group_metadata.hpp.

  {
    auto o = dynamic_cast<const GroupMetadata&>(other);
    return m_group_id == o.m_group_id;
  }

set_expected_files()

void dunedaq::snbmodules::GroupMetadata::set_expected_files ( std::set< std::string > expected_files )

inline

Definition at line 113 of file group_metadata.hpp.

{ m_expected_files = std::move(expected_files); }

set_group_id()

void dunedaq::snbmodules::GroupMetadata::set_group_id ( std::string transfer_id )

inline

Definition at line 102 of file group_metadata.hpp.

{ m_group_id = std::move(transfer_id); }

set_protocol()

void dunedaq::snbmodules::GroupMetadata::set_protocol ( protocol_type::e_protocol_type protocol )

inline

Definition at line 103 of file group_metadata.hpp.

{ m_protocol = protocol; }

set_protocol_options()

void dunedaq::snbmodules::GroupMetadata::set_protocol_options ( nlohmann::json protocol_options )

inline

Definition at line 108 of file group_metadata.hpp.

  {
    m_protocol_options = std::move(protocol_options);
  }

set_source_id()

void dunedaq::snbmodules::GroupMetadata::set_source_id ( std::string source_id )

inline

Definition at line 112 of file group_metadata.hpp.

{ m_source_id = std::move(source_id); }

set_transfers_meta()

void dunedaq::snbmodules::GroupMetadata::set_transfers_meta ( std::vector< std::shared_ptr< TransferMetadata > > files_meta )

inline

Definition at line 104 of file group_metadata.hpp.

  {
    m_transfers_meta = std::move(files_meta);
  }

to_string()

std::string dunedaq::snbmodules::GroupMetadata::to_string

(

)

const

Definition at line 145 of file group_metadata.cpp.

{
  std::string str;
  str += "transfer_id " + get_group_id() + " ";
  str += "protocol " + protocol_type::protocols_to_string(get_protocol()) + "\n";
 
  for (const auto& file : get_transfers_meta()) {
    str += "*file " + file->get_file_name() + "\n";
  }
  for (const auto& file : get_expected_files()) {
    str += "*expectedfile " + file + "\n";
  }
  return str;
}

Member Data Documentation

m_expected_files

std::set<std::string> dunedaq::snbmodules::GroupMetadata::m_expected_files

private

Set of expected files metadata to add later.

Definition at line 160 of file group_metadata.hpp.

m_file_extension

const std::string dunedaq::snbmodules::GroupMetadata::m_file_extension = ".gmetadata"

static

Definition at line 29 of file group_metadata.hpp.

m_group_id

std::string dunedaq::snbmodules::GroupMetadata::m_group_id

private

Unique identifier for the transfer.

Definition at line 148 of file group_metadata.hpp.

m_protocol

protocol_type::e_protocol_type dunedaq::snbmodules::GroupMetadata::m_protocol

private

Protocol used for the transfer, every files in the transfer must use the same protocol.

Definition at line 151 of file group_metadata.hpp.

m_protocol_options

nlohmann::json dunedaq::snbmodules::GroupMetadata::m_protocol_options

private

Specific options for the selected protocol.

Definition at line 154 of file group_metadata.hpp.

m_source_id

std::string dunedaq::snbmodules::GroupMetadata::m_source_id

private

Get source id.

Definition at line 163 of file group_metadata.hpp.

m_source_ip

IPFormat dunedaq::snbmodules::GroupMetadata::m_source_ip

private

Get source ip.

Definition at line 166 of file group_metadata.hpp.

m_transfers_meta

std::vector<std::shared_ptr<TransferMetadata> > dunedaq::snbmodules::GroupMetadata::m_transfers_meta

private

Set of files to transfer.

Definition at line 157 of file group_metadata.hpp.

---

The documentation for this class was generated from the following files:

• /github/workspace/dunedaq/sourcecode/snbmodules/include/snbmodules/group_metadata.hpp
• /github/workspace/dunedaq/sourcecode/snbmodules/src/common/group_metadata.cpp