uhallibs::Axi4Lite Class Reference

Transport protocol to transfer an IPbus buffer over Axi4Lite mapped in a 32b address space. More...

#include <ProtocolAxi4Lite.hpp>

Inheritance diagram for uhallibs::Axi4Lite:

Collaboration diagram for uhallibs::Axi4Lite:

Classes
class	MappedFile

Public Member Functions
	Axi4Lite (const std::string &aId, const uhal::URI &aUri)
	Axi4Lite (const Axi4Lite &)=delete
Axi4Lite &	operator= (const Axi4Lite &)=delete
virtual	~Axi4Lite ()
	Destructor.

Private Types
typedef ipc::RobustMutex	IPCMutex_t
typedef std::unique_lock< IPCMutex_t >	IPCScopedLock_t
typedef IPbus< 2, 0 >	InnerProtocol
typedef std::chrono::steady_clock	SteadyClock_t

Private Member Functions
void	implementDispatch (std::shared_ptr< uhal::Buffers > aBuffers)
virtual void	Flush ()
	Concrete implementation of the synchronization function to block until all buffers have been sent, all replies received and all data validated.
virtual void	dispatchExceptionHandler ()
	Function which tidies up this protocol layer in the event of an exception.
uint32_t	getMaxSendSize ()
uint32_t	getMaxReplySize ()
void	connect ()
	Set up the connection to the device.
void	connect (IPCScopedLock_t &)
	Set up the connection to the device.
void	disconnect ()
	Close the connection to the device.
void	write (const std::shared_ptr< uhal::Buffers > &aBuffers)
	Write request packet to next page in host-to-FPGA device file.
void	read ()
	Read next pending reply packet from appropriate page of FPGA-to-host device file, and validate contents.

Static Private Member Functions
static std::string	getDevicePath (const uhal::URI &aUri)
static std::string	getSharedMemName (const std::string &)

Private Attributes
bool	mConnected
MappedFile	mMappedFile
ipc::SharedMemObject< IPCMutex_t >	mIPCMutex
bool	mIPCExternalSessionActive
uint64_t	mIPCSessionCount
std::chrono::microseconds	mSleepDuration
uint32_t	mNumberOfPages
uint32_t	mMaxInFlight
uint32_t	mPageSize
uint32_t	mMaxPacketSize
uint32_t	mIndexNextPage
uint32_t	mPublishedReplyPageCount
uint32_t	mReadReplyPageCount
std::deque< std::shared_ptr< uhal::Buffers > >	mReplyQueue
	The list of buffers still awaiting a reply.

Detailed Description

Transport protocol to transfer an IPbus buffer over Axi4Lite mapped in a 32b address space.

Definition at line 95 of file ProtocolAxi4Lite.hpp.

Member Typedef Documentation

InnerProtocol

IPbus< 2 , 0 > uhallibs::Axi4Lite::InnerProtocol

private

Definition at line 166 of file ProtocolAxi4Lite.hpp.

IPCMutex_t

ipc::RobustMutex uhallibs::Axi4Lite::IPCMutex_t

private

Definition at line 163 of file ProtocolAxi4Lite.hpp.

IPCScopedLock_t

std::unique_lock<IPCMutex_t> uhallibs::Axi4Lite::IPCScopedLock_t

private

Definition at line 164 of file ProtocolAxi4Lite.hpp.

SteadyClock_t

std::chrono::steady_clock uhallibs::Axi4Lite::SteadyClock_t

private

Definition at line 167 of file ProtocolAxi4Lite.hpp.

Constructor & Destructor Documentation

Axi4Lite() [1/2]

uhallibs::Axi4Lite::Axi4Lite	(	const std::string &	aId,
		const uhal::URI &	aUri )

Constructor

Parameters

aId	the uinique identifier that the client will be given.
aUri	a struct containing the full URI of the target.

Definition at line 191 of file ProtocolAxi4Lite.cpp.

    : IPbus<2, 0>(aId, aUri),
      mConnected(false),
      mMappedFile(getDevicePath(aUri), 64, PROT_WRITE),
      mIPCMutex(getSharedMemName(mMappedFile.getPath())),
      mNumberOfPages(0),
      mMaxInFlight(0),
      mPageSize(0),
      mMaxPacketSize(0),
      mIndexNextPage(0),
      mPublishedReplyPageCount(0),
      mReadReplyPageCount(0) {
 
        mSleepDuration = std::chrono::microseconds(50);
 
      }

Axi4Lite() [2/2]

uhallibs::Axi4Lite::Axi4Lite ( const Axi4Lite & )

delete

~Axi4Lite()

uhallibs::Axi4Lite::~Axi4Lite ( )

virtual

Destructor.

Definition at line 208 of file ProtocolAxi4Lite.cpp.

                    {
 
}

Member Function Documentation

connect() [1/2]

void uhallibs::Axi4Lite::connect ( )

private

Set up the connection to the device.

Definition at line 272 of file ProtocolAxi4Lite.cpp.

{
  IPCScopedLock_t lLockGuard(*mIPCMutex);
  connect(lLockGuard);
}

connect() [2/2]

void uhallibs::Axi4Lite::connect ( IPCScopedLock_t & aGuard )

private

Set up the connection to the device.

Definition at line 278 of file ProtocolAxi4Lite.cpp.

{
  // Read current value of session counter when reading status info from FPGA
  // (So that can check whether this info is up-to-date later on, when sending next request packet)
  mIPCExternalSessionActive = mIPCMutex->isActive() and (not mMappedFile.haveLock());
  mIPCSessionCount = mIPCMutex->getCounter();
 
  log ( uhal::Debug() , "Axi4Lite client is opening device file " , uhal::Quote ( mMappedFile.getPath() ) , " (device-to-client)" );
 
  // Minimal mapping to read the 
  mMappedFile.setLength(0x10);
  mMappedFile.open();
  std::vector<uint32_t> lStats;
  mMappedFile.read(0, 4, lStats);
  mMappedFile.close();
  aGuard.unlock();
 
  mNumberOfPages = lStats.at(0);
  if ( (mMaxInFlight == 0) or (mMaxInFlight > mNumberOfPages) )
    mMaxInFlight = mNumberOfPages;
  mPageSize = lStats.at(1);
  if ( (mMaxPacketSize == 0) or (mMaxPacketSize >= mPageSize) )
    mMaxPacketSize = mPageSize - 1;
  mIndexNextPage = lStats.at(2);
  mPublishedReplyPageCount = lStats.at(3);
  mReadReplyPageCount = mPublishedReplyPageCount;
 
  // 
  constexpr uint32_t lSafetyMargin(4096);
  // Set the memory mapping range to a value commensurate to the memory available in firmware
  mMappedFile.setLength(mNumberOfPages*mPageSize+4+lSafetyMargin);
  mMappedFile.open();
 
  mConnected=true;
 
  log ( uhal::Info() , "Axi4Lite client connected to device at ", uhal::Quote(mMappedFile.getPath()), ", FPGA has ", Integer(mNumberOfPages), " pages, each of size ", Integer(mPageSize), " words, index ", Integer(mIndexNextPage), " should be filled next" );
 
}

disconnect()

void uhallibs::Axi4Lite::disconnect ( )

private

Close the connection to the device.

Definition at line 317 of file ProtocolAxi4Lite.cpp.

{
  mMappedFile.close();
  mConnected = false;
}

dispatchExceptionHandler()

void uhallibs::Axi4Lite::dispatchExceptionHandler ( )

privatevirtual

Function which tidies up this protocol layer in the event of an exception.

Definition at line 240 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Notice(), "Axi4Lite client ", uhal::Quote(id()), " (URI: ", uhal::Quote(uri()), ") : closing device files since exception detected");
 
  ClientInterface::returnBufferToPool ( mReplyQueue );
 
  mMappedFile.unlock();
 
  disconnect();
 
  InnerProtocol::dispatchExceptionHandler();
}

Flush()

void uhallibs::Axi4Lite::Flush ( )

privatevirtual

Concrete implementation of the synchronization function to block until all buffers have been sent, all replies received and all data validated.

Definition at line 227 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : Flush method called");
  while ( !mReplyQueue.empty() )
    read();
 
  mMappedFile.unlock();
 
  IPCScopedLock_t lLockGuard(*mIPCMutex);
  mIPCMutex->endSession();
}

getDevicePath()

std::string uhallibs::Axi4Lite::getDevicePath ( const uhal::URI & aUri )

staticprivate

Definition at line 177 of file ProtocolAxi4Lite.cpp.

                                                     {
 
  std::string lPath = aUri.mHostname;
 
  auto it = std::find_if( aUri.mArguments.begin(), aUri.mArguments.end(),
    [](const std::pair<std::string, std::string>& element){ return element.first == "dev";} );
 
  if ( it != aUri.mArguments.end() ) {
    lPath += "/"+it->second;
  }
 
  return lPath;
}

getMaxReplySize()

uint32_t uhallibs::Axi4Lite::getMaxReplySize ( )

private

Return the maximum size of reply packet based on the buffer size in the target

Returns

the maximum size of reply packet

Definition at line 263 of file ProtocolAxi4Lite.cpp.

{
  if ( ! mConnected )
    connect();
 
  return mMaxPacketSize * 4;
}

getMaxSendSize()

uint32_t uhallibs::Axi4Lite::getMaxSendSize ( )

private

Return the maximum size to be sent based on the buffer size in the target

Returns

the maximum size to be sent

Definition at line 254 of file ProtocolAxi4Lite.cpp.

{
  if ( ! mConnected )
    connect();
 
  return mMaxPacketSize * 4;
}

getSharedMemName()

std::string uhallibs::Axi4Lite::getSharedMemName ( const std::string & aPath )

staticprivate

Definition at line 169 of file ProtocolAxi4Lite.cpp.

{
  std::string lSanitizedPath(aPath);
  std::replace(lSanitizedPath.begin(), lSanitizedPath.end(), '/', ':');
 
  return "/uhal::ipbusaxi4lite-2.0::" + lSanitizedPath;
}

implementDispatch()

void uhallibs::Axi4Lite::implementDispatch ( std::shared_ptr< uhal::Buffers > aBuffers )

private

Send the IPbus buffer to the target, read back the response and call the packing-protocol's validate function

Parameters

aBuffers

the buffer object wrapping the send and recieve buffers that are to be transported If multithreaded, adds buffer to the dispatch queue and returns. If single-threaded, calls the dispatch-worker dispatch function directly and blocks until the response is validated.

Definition at line 214 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : implementDispatch method called");
 
  if ( ! mConnected )
    connect();
 
  if ( mReplyQueue.size() == mMaxInFlight )
    read();
  write(aBuffers);
}

operator=()

Axi4Lite & uhallibs::Axi4Lite::operator= ( const Axi4Lite & )

delete

read()

void uhallibs::Axi4Lite::read ( )

private

Read next pending reply packet from appropriate page of FPGA-to-host device file, and validate contents.

Definition at line 355 of file ProtocolAxi4Lite.cpp.

{
  const size_t lPageIndexToRead = (mIndexNextPage - mReplyQueue.size() + mNumberOfPages) % mNumberOfPages;
  SteadyClock_t::time_point lStartTime = SteadyClock_t::now();
 
  if (mReadReplyPageCount == mPublishedReplyPageCount)
  {
    uint32_t lHwPublishedPageCount = 0x0;
 
    std::vector<uint32_t> lValues;
    while ( true ) {
      // FIXME : Improve by simply adding fileWrite method that takes uint32_t ref as argument (or returns uint32_t)
      IPCScopedLock_t lGuard(*mIPCMutex);
      mMappedFile.read(0, 4, lValues);
      lHwPublishedPageCount = lValues.at(3);
      log (uhal::Debug(), "Read status info from addr 0 (", Integer(lValues.at(0)), ", ", Integer(lValues.at(1)), ", ", Integer(lValues.at(2)), ", ", Integer(lValues.at(3)), "): ", PacketFmt((const uint8_t*)lValues.data(), 4 * lValues.size()));
 
      if (lHwPublishedPageCount != mPublishedReplyPageCount) {
        mPublishedReplyPageCount = lHwPublishedPageCount;
        break;
      }
      // FIXME: Throw if published page count is invalid number
 
      if (SteadyClock_t::now() - lStartTime > std::chrono::microseconds(getBoostTimeoutPeriod().total_microseconds())) {
        exception::Axi4LiteTimeout lExc;
        log(lExc, "Next page (index ", Integer(lPageIndexToRead), " count ", Integer(mPublishedReplyPageCount+1), ") of Axi4Lite device '" + mMappedFile.getPath() + "' is not ready after timeout period");
        throw lExc;
      }
 
      log(uhal::Debug(), "Axi4Lite client ", uhal::Quote(id()), " (URI: ", uhal::Quote(uri()), ") : Trying to read page index ", Integer(lPageIndexToRead), " = count ", Integer(mReadReplyPageCount+1), "; published page count is ", Integer(lHwPublishedPageCount), "; sleeping for ", mSleepDuration.count(), "us");
      if (mSleepDuration > std::chrono::microseconds(0))
        std::this_thread::sleep_for( mSleepDuration );
      lValues.clear();
    }
 
    log(uhal::Info(), "Axi4Lite client ", uhal::Quote(id()), " (URI: ", uhal::Quote(uri()), ") : Reading page ", Integer(lPageIndexToRead), " (published count ", Integer(lHwPublishedPageCount), ", surpasses required, ", Integer(mReadReplyPageCount + 1), ")");
 
  }
  mReadReplyPageCount++;
  
  // PART 1 : Read the page
  std::shared_ptr<uhal::Buffers> lBuffers = mReplyQueue.front();
  mReplyQueue.pop_front();
 
  uint32_t lNrWordsToRead(lBuffers->replyCounter() >> 2);
  lNrWordsToRead += 1;
 
  std::vector<uint32_t> lPageContents;
  IPCScopedLock_t lGuard(*mIPCMutex);
  mMappedFile.read(4 + lPageIndexToRead * mPageSize, lNrWordsToRead , lPageContents);
  lGuard.unlock();
  log (uhal::Debug(), "Read " , Integer(lNrWordsToRead), " 32-bit words from address " , Integer(4 + lPageIndexToRead * 4 * mPageSize), " ... ", PacketFmt((const uint8_t*)lPageContents.data(), 4 * lPageContents.size()));
 
  // PART 2 : Transfer to reply buffer
  const std::deque< std::pair< uint8_t* , uint32_t > >& lReplyBuffers ( lBuffers->getReplyBuffer() );
  size_t lNrWordsInPacket = (lPageContents.at(0) >> 16) + (lPageContents.at(0) & 0xFFFF);
  if (lNrWordsInPacket != (lBuffers->replyCounter() >> 2))
    log (uhal::Warning(), "Expected reply packet to contain ", Integer(lBuffers->replyCounter() >> 2), " words, but it actually contains ", Integer(lNrWordsInPacket), " words");
 
  size_t lNrBytesCopied = 0;
  for (const auto& lBuffer: lReplyBuffers)
  {
    // Don't copy more of page than was written to, for cases when less data received than expected
    if ( lNrBytesCopied >= 4*lNrWordsInPacket)
      break;
 
    size_t lNrBytesToCopy = std::min( lBuffer.second , uint32_t(4*lNrWordsInPacket - lNrBytesCopied) );
    memcpy ( lBuffer.first, &lPageContents.at(1 + (lNrBytesCopied / 4)), lNrBytesToCopy );
    lNrBytesCopied += lNrBytesToCopy;
  }
 
 
  // PART 3 : Validate the packet contents
  uhal::exception::exception* lExc = NULL;
  try
  {
    lExc = ClientInterface::validate ( lBuffers );
  }
  catch ( uhal::exception::exception& aExc )
  {
    uhal::exception::ValidationError lExc2;
    log ( lExc2 , "Exception caught during reply validation for Axi4Lite device with URI " , uhal::Quote ( this->uri() ) , "; what returned: " , uhal::Quote ( aExc.what() ) );
    throw lExc2;
  }
 
  if (lExc != NULL)
    lExc->throwAsDerivedType();
}

write()

void uhallibs::Axi4Lite::write ( const std::shared_ptr< uhal::Buffers > & aBuffers )

private

Write request packet to next page in host-to-FPGA device file.

Definition at line 323 of file ProtocolAxi4Lite.cpp.

{
  if (not mMappedFile.haveLock()) {
    mMappedFile.lock();
 
    IPCScopedLock_t lGuard(*mIPCMutex);
    mIPCMutex->startSession();
    mIPCSessionCount++;
 
    // If these two numbers don't match, another client/process has sent packets
    // more recently than this client has, so must re-read status info
    if (mIPCExternalSessionActive or (mIPCMutex->getCounter() != mIPCSessionCount)) {
      connect(lGuard);
    }
  }
 
  log (uhal::Info(), "Axi4Lite client ", uhal::Quote(id()), " (URI: ", uhal::Quote(uri()), ") : writing ", Integer(aBuffers->sendCounter() / 4), "-word packet to page ", Integer(mIndexNextPage), " in ", uhal::Quote(mMappedFile.getPath()));
 
  const uint32_t lHeaderWord = (0x10000 | (((aBuffers->sendCounter() / 4) - 1) & 0xFFFF));
  std::vector<std::pair<const uint8_t*, size_t> > lDataToWrite;
  lDataToWrite.push_back( std::make_pair(reinterpret_cast<const uint8_t*>(&lHeaderWord), sizeof lHeaderWord) );
  lDataToWrite.push_back( std::make_pair(aBuffers->getSendBuffer(), aBuffers->sendCounter()) );
 
  IPCScopedLock_t lGuard(*mIPCMutex);
  mMappedFile.write(mIndexNextPage * mPageSize, lDataToWrite);
  log (uhal::Debug(), "Wrote " , Integer((aBuffers->sendCounter() / 4) + 1), " 32-bit words at address " , Integer(mIndexNextPage * mPageSize), " ... ", PacketFmt(lDataToWrite));
 
  mIndexNextPage = (mIndexNextPage + 1) % mNumberOfPages;
  mReplyQueue.push_back(aBuffers);
}

Member Data Documentation

mConnected

bool uhallibs::Axi4Lite::mConnected

private

Definition at line 212 of file ProtocolAxi4Lite.hpp.

mIndexNextPage

uint32_t uhallibs::Axi4Lite::mIndexNextPage

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mIPCExternalSessionActive

bool uhallibs::Axi4Lite::mIPCExternalSessionActive

private

Definition at line 217 of file ProtocolAxi4Lite.hpp.

mIPCMutex

ipc::SharedMemObject<IPCMutex_t> uhallibs::Axi4Lite::mIPCMutex

private

Definition at line 216 of file ProtocolAxi4Lite.hpp.

mIPCSessionCount

uint64_t uhallibs::Axi4Lite::mIPCSessionCount

private

Definition at line 218 of file ProtocolAxi4Lite.hpp.

mMappedFile

MappedFile uhallibs::Axi4Lite::mMappedFile

private

Definition at line 214 of file ProtocolAxi4Lite.hpp.

mMaxInFlight

uint32_t uhallibs::Axi4Lite::mMaxInFlight

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mMaxPacketSize

uint32_t uhallibs::Axi4Lite::mMaxPacketSize

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mNumberOfPages

uint32_t uhallibs::Axi4Lite::mNumberOfPages

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mPageSize

uint32_t uhallibs::Axi4Lite::mPageSize

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mPublishedReplyPageCount

uint32_t uhallibs::Axi4Lite::mPublishedReplyPageCount

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mReadReplyPageCount

uint32_t uhallibs::Axi4Lite::mReadReplyPageCount

private

Definition at line 222 of file ProtocolAxi4Lite.hpp.

mReplyQueue

std::deque< std::shared_ptr< uhal::Buffers > > uhallibs::Axi4Lite::mReplyQueue

private

The list of buffers still awaiting a reply.

Definition at line 225 of file ProtocolAxi4Lite.hpp.

mSleepDuration

std::chrono::microseconds uhallibs::Axi4Lite::mSleepDuration

private

Definition at line 220 of file ProtocolAxi4Lite.hpp.

---

The documentation for this class was generated from the following files:

• /github/workspace/dunedaq/sourcecode/uhallibs/include/uhallibs/ProtocolAxi4Lite.hpp
• /github/workspace/dunedaq/sourcecode/uhallibs/src/ProtocolAxi4Lite.cpp