dunedaq::datahandlinglibs::BufferedFileWriter< Alignment > Class Template Reference

#include <BufferedFileWriter.hpp>

Public Member Functions
	BufferedFileWriter (std::string filename, size_t buffer_size, std::string compression_algorithm="None", bool use_o_direct=true)
	BufferedFileWriter ()
	~BufferedFileWriter ()
	BufferedFileWriter (const BufferedFileWriter &)=delete
	BufferedFileWriter is not copy-constructible.
BufferedFileWriter &	operator= (const BufferedFileWriter &)=delete
	BufferedFileWriter is not copy-assginable.
	BufferedFileWriter (BufferedFileWriter &&)=delete
	BufferedFileWriter is not move-constructible.
BufferedFileWriter &	operator= (BufferedFileWriter &&)=delete
	BufferedFileWriter is not move-assignable.
void	open (std::string filename, size_t buffer_size, std::string compression_algorithm="None", bool use_o_direct=true)
bool	is_open () const
bool	write (const char *memory, const size_t size)
void	close ()
void	flush ()

Private Types
using	io_sink_t = boost::iostreams::file_descriptor_sink
using	aligned_allocator_t = boost::alignment::aligned_allocator<io_sink_t::char_type, Alignment>
using	filtering_ostream_t

Private Attributes
std::string	m_filename
size_t	m_buffer_size
std::string	m_compression_algorithm
int	m_fd
io_sink_t	m_sink
filtering_ostream_t	m_output_stream
bool	m_is_open = false
bool	m_use_o_direct = true

Detailed Description

template<size_t Alignment = 4096>
class dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >

Class to buffer and write data of a specified type to a file using O_DIRECT. In addition, data can be compressed. before being written.

Template Parameters

ReadoutType	Type of the data that will be written to file
Alignment	The alignment used for allocations of buffers. It has to fulfil certain system specific requirements such that file writing with O_DIRECT can work.

Definition at line 47 of file BufferedFileWriter.hpp.

Member Typedef Documentation

aligned_allocator_t

template<size_t Alignment = 4096>

using dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::aligned_allocator_t = boost::alignment::aligned_allocator<io_sink_t::char_type, Alignment>

private

Definition at line 50 of file BufferedFileWriter.hpp.

filtering_ostream_t

template<size_t Alignment = 4096>

using dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::filtering_ostream_t

private

Initial value:

 
    boost::iostreams::filtering_stream<boost::iostreams::output, char, std::char_traits<char>, aligned_allocator_t>

Definition at line 51 of file BufferedFileWriter.hpp.

io_sink_t

template<size_t Alignment = 4096>

using dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::io_sink_t = boost::iostreams::file_descriptor_sink

private

Definition at line 49 of file BufferedFileWriter.hpp.

Constructor & Destructor Documentation

BufferedFileWriter() [1/4]

template<size_t Alignment = 4096>

dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::BufferedFileWriter ( std::string filename, size_t buffer_size, std::string compression_algorithm = "None", bool use_o_direct = true )

inline

Constructor to construct and initialize an instance. The file will be open after initialization.

Parameters

filename	The file to be used. Existing data will be overwritten, if the file does not exist it will be created.
buffer_size	The size of the buffer that is used before data is written to the file. Make sure that this size fulfils size requirements of O_DIRECT, otherwise writes will fail.
compression_algorithm	The compression algorithm to use. Can be one of: None, zstd, lzma or zlib
use_o_direct	file descriptors : avoid excessive resource footprint. It also avoids the intermediate aligned buffer, requires the source latency buffer to be memory aligned.

Exceptions

CannotOpenFile	If the file can not be opened.
ConfigurationError	If the compression algorithm parameter is not recognized.

Definition at line 66 of file BufferedFileWriter.hpp.

  {
    open(filename, buffer_size, compression_algorithm, use_o_direct);
  }

BufferedFileWriter() [2/4]

template<size_t Alignment = 4096>

dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::BufferedFileWriter ( )

inline

Constructor to construct an instance without opening a file.

Definition at line 74 of file BufferedFileWriter.hpp.

{}

~BufferedFileWriter()

template<size_t Alignment = 4096>

dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::~BufferedFileWriter ( )

inline

Destructor that closes the file before destruction to make sure all data in any buffers is written to file.

Definition at line 79 of file BufferedFileWriter.hpp.

  {
    if (m_is_open)
      close();
  }

BufferedFileWriter() [3/4]

template<size_t Alignment = 4096>

dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::BufferedFileWriter ( const BufferedFileWriter< Alignment > & )

delete

BufferedFileWriter is not copy-constructible.

BufferedFileWriter() [4/4]

template<size_t Alignment = 4096>

dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::BufferedFileWriter ( BufferedFileWriter< Alignment > && )

delete

BufferedFileWriter is not move-constructible.

Member Function Documentation

close()

template<size_t Alignment = 4096>

void dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::close ( )

inline

Close the writer. All data from any buffers will be written to file.

Definition at line 167 of file BufferedFileWriter.hpp.

  {
    // Set the file descriptor to not use O_DIRECT. This is necessary because the write size has to be aligned for
    // O_DIRECT to succeed. This is not guaranteed for the data that remains in the buffer.
    fcntl(m_fd, F_SETFL, O_CREAT | O_WRONLY);
    m_output_stream.reset();
    m_is_open = false;
  }

flush()

template<size_t Alignment = 4096>

void dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::flush ( )

inline

If no compression is used, this writes all data from buffers to the file. In case that compression is used, this is not guaranteed.

Definition at line 180 of file BufferedFileWriter.hpp.

  {
    // Set the file descriptor to not use O_DIRECT. This is necessary because the write size has to be aligned for
    // O_DIRECT to succeed. This is not guaranteed for the data that remains in the buffer.
    fcntl(m_fd, F_SETFL, O_CREAT | O_WRONLY);
    // This does not flush the compressor as it is not flushable
    m_output_stream.flush();
    // Activate O_DIRECT again
    auto oflag = O_CREAT | O_WRONLY;
    if (m_use_o_direct) {
      oflag = oflag | O_DIRECT;
    }
    fcntl(m_fd, F_SETFL, oflag);
  }

is_open()

template<size_t Alignment = 4096>

bool dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::is_open ( ) const

inline

Check if the file is open.

Returns

true if the file is open, false otherwise.

Definition at line 149 of file BufferedFileWriter.hpp.

{ return m_is_open; }

open()

template<size_t Alignment = 4096>

void dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::open ( std::string filename, size_t buffer_size, std::string compression_algorithm = "None", bool use_o_direct = true )

inline

Open a file.

Parameters

filename	The file to be used. Existing data will be overwritten, if the file does not exist it will be created.
buffer_size	The size of the buffer that is used before data is written to the file. Make sure that this size fulfils size requirements of O_DIRECT, otherwise writes will fail.
compression_algorithm	The compression algorithm to use. Can be one of: None, zstd, lzma or zlib
use_o_direct	file descriptors : avoid excessive resource footprint. It also avoids the intermediate aligned buffer, requires the source latency buffer to be memory aligned.

Exceptions

CannotOpenFile	If the file can not be opened.
ConfigurationError	If the compression algorithm parameter is not recognized.

Definition at line 101 of file BufferedFileWriter.hpp.

  {
    m_use_o_direct = use_o_direct;
    if (m_is_open) {
      close();
    }
 
    m_filename = filename;
    m_buffer_size = buffer_size;
    m_compression_algorithm = compression_algorithm;
    auto oflag = O_CREAT | O_WRONLY;
    if (m_use_o_direct) {
      oflag = oflag | O_DIRECT;
    }
 
    m_fd = ::open(m_filename.c_str(), oflag, 0644);
    if (m_fd == -1) {
      throw BufferedReaderWriterCannotOpenFile(ERS_HERE, m_filename);
    }
 
    m_sink = io_sink_t(m_fd, boost::iostreams::file_descriptor_flags::close_handle);
    if (m_compression_algorithm == "zstd") {
      TLOG_DEBUG(TLVL_WORK_STEPS) << "Using zstd compression" << std::endl;
      m_output_stream.push(boost::iostreams::zstd_compressor(boost::iostreams::zstd::best_speed));
    } else if (m_compression_algorithm == "lzma") {
      TLOG_DEBUG(TLVL_WORK_STEPS) << "Using lzma compression" << std::endl;
      m_output_stream.push(boost::iostreams::lzma_compressor(boost::iostreams::lzma::best_speed));
    } else if (m_compression_algorithm == "zlib") {
      TLOG_DEBUG(TLVL_WORK_STEPS) << "Using zlib compression" << std::endl;
      m_output_stream.push(boost::iostreams::zlib_compressor(boost::iostreams::zlib::best_speed));
    } else if (m_compression_algorithm == "None") {
      TLOG_DEBUG(TLVL_WORK_STEPS) << "Running without compression" << std::endl;
    } else {
      throw BufferedReaderWriterConfigurationError(ERS_HERE,
                                                   "Non-recognized compression algorithm: " + m_compression_algorithm);
    }
 
    m_output_stream.push(m_sink, m_buffer_size);
    m_is_open = true;
  }

operator=() [1/2]

template<size_t Alignment = 4096>

BufferedFileWriter & dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::operator= ( BufferedFileWriter< Alignment > && )

delete

BufferedFileWriter is not move-assignable.

operator=() [2/2]

template<size_t Alignment = 4096>

BufferedFileWriter & dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::operator= ( const BufferedFileWriter< Alignment > & )

delete

BufferedFileWriter is not copy-assginable.

write()

template<size_t Alignment = 4096>

bool dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::write ( const char * memory, const size_t size )

inline

Write something to the buffer. If the buffer is full, all data from it will be written to file.

Parameters

element

The element to write.

Returns

true if the write was successful, false if the writer is not open or the write was not successful.

Definition at line 156 of file BufferedFileWriter.hpp.

  {
    if (!m_is_open)
      return false;
    m_output_stream.write(memory, size); // NOLINT
    return !m_output_stream.bad();
  }

Member Data Documentation

m_buffer_size

template<size_t Alignment = 4096>

size_t dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_buffer_size

private

Definition at line 198 of file BufferedFileWriter.hpp.

m_compression_algorithm

template<size_t Alignment = 4096>

std::string dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_compression_algorithm

private

Definition at line 199 of file BufferedFileWriter.hpp.

m_fd

template<size_t Alignment = 4096>

int dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_fd

private

Definition at line 202 of file BufferedFileWriter.hpp.

m_filename

template<size_t Alignment = 4096>

std::string dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_filename

private

Definition at line 197 of file BufferedFileWriter.hpp.

m_is_open

template<size_t Alignment = 4096>

bool dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_is_open = false

private

Definition at line 205 of file BufferedFileWriter.hpp.

m_output_stream

template<size_t Alignment = 4096>

filtering_ostream_t dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_output_stream

private

Definition at line 204 of file BufferedFileWriter.hpp.

m_sink

template<size_t Alignment = 4096>

io_sink_t dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_sink

private

Definition at line 203 of file BufferedFileWriter.hpp.

m_use_o_direct

template<size_t Alignment = 4096>

bool dunedaq::datahandlinglibs::BufferedFileWriter< Alignment >::m_use_o_direct = true

private

Definition at line 206 of file BufferedFileWriter.hpp.

---

The documentation for this class was generated from the following file:

• /github/workspace/dunedaq/sourcecode/datahandlinglibs/include/datahandlinglibs/utils/BufferedFileWriter.hpp