uhallibs::Axi4Lite Class Reference

Transport protocol to transfer an IPbus buffer over Axi4Lite mapped in a 32b address space. More...

#include <ProtocolAxi4Lite.hpp>

Inheritance diagram for uhallibs::Axi4Lite:

Collaboration diagram for uhallibs::Axi4Lite:

Classes
class	MappedFile

Public Member Functions
	Axi4Lite (const std::string &aId, const uhal::URI &aUri)
	Axi4Lite (const Axi4Lite &)=delete
Axi4Lite &	operator= (const Axi4Lite &)=delete
virtual	~Axi4Lite ()
	Destructor.

Private Types
typedef ipc::RobustMutex	IPCMutex_t
typedef std::unique_lock< IPCMutex_t >	IPCScopedLock_t
typedef IPbus< 2, 0 >	InnerProtocol
typedef std::chrono::steady_clock	SteadyClock_t

Private Member Functions
void	implementDispatch (std::shared_ptr< uhal::Buffers > aBuffers)
virtual void	Flush ()
virtual void	dispatchExceptionHandler ()
	Function which tidies up this protocol layer in the event of an exception.
uint32_t	getMaxSendSize ()
uint32_t	getMaxReplySize ()
void	connect ()
	Set up the connection to the device.
void	connect (IPCScopedLock_t &)
	Set up the connection to the device.
void	disconnect ()
	Close the connection to the device.
void	write (const std::shared_ptr< uhal::Buffers > &aBuffers)
	Write request packet to next page in host-to-FPGA device file.
void	read ()
	Read next pending reply packet from appropriate page of FPGA-to-host device file, and validate contents.

Static Private Member Functions
static std::string	getDevicePath (const uhal::URI &aUri)
static std::string	getSharedMemName (const std::string &)

Private Attributes
bool	mConnected
MappedFile	mMappedFile
ipc::SharedMemObject< IPCMutex_t >	mIPCMutex
bool	mIPCExternalSessionActive
uint64_t	mIPCSessionCount
std::chrono::microseconds	mSleepDuration
uint32_t	mNumberOfPages
uint32_t	mMaxInFlight
uint32_t	mPageSize
uint32_t	mMaxPacketSize
uint32_t	mIndexNextPage
uint32_t	mPublishedReplyPageCount
uint32_t	mReadReplyPageCount
std::deque< std::shared_ptr< uhal::Buffers > >	mReplyQueue
	The list of buffers still awaiting a reply.

Detailed Description

Transport protocol to transfer an IPbus buffer over Axi4Lite mapped in a 32b address space.

Definition at line 92 of file ProtocolAxi4Lite.hpp.

Member Typedef Documentation

InnerProtocol

typedef IPbus<2, 0> uhallibs::Axi4Lite::InnerProtocol

private

Definition at line 157 of file ProtocolAxi4Lite.hpp.

IPCMutex_t

typedef ipc::RobustMutex uhallibs::Axi4Lite::IPCMutex_t

private

Definition at line 154 of file ProtocolAxi4Lite.hpp.

IPCScopedLock_t

typedef std::unique_lock<IPCMutex_t> uhallibs::Axi4Lite::IPCScopedLock_t

private

Definition at line 155 of file ProtocolAxi4Lite.hpp.

SteadyClock_t

typedef std::chrono::steady_clock uhallibs::Axi4Lite::SteadyClock_t

private

Definition at line 158 of file ProtocolAxi4Lite.hpp.

Constructor & Destructor Documentation

Axi4Lite() [1/2]

uhallibs::Axi4Lite::Axi4Lite	(	const std::string &	aId,
		const uhal::URI &	aUri )

Constructor

Parameters

aId	the uinique identifier that the client will be given.
aUri	a struct containing the full URI of the target.

Definition at line 247 of file ProtocolAxi4Lite.cpp.

  : IPbus<2, 0>(aId, aUri)
  , mConnected(false)
  , mMappedFile(getDevicePath(aUri), 64, PROT_WRITE)
  , mIPCMutex(getSharedMemName(mMappedFile.getPath()))
  , mNumberOfPages(0)
  , mMaxInFlight(0)
  , mPageSize(0)
  , mMaxPacketSize(0)
  , mIndexNextPage(0)
  , mPublishedReplyPageCount(0)
  , mReadReplyPageCount(0)
{
 
  mSleepDuration = std::chrono::microseconds(50);
}

Axi4Lite() [2/2]

uhallibs::Axi4Lite::Axi4Lite ( const Axi4Lite & )

delete

~Axi4Lite()

uhallibs::Axi4Lite::~Axi4Lite ( )

virtual

Destructor.

Definition at line 264 of file ProtocolAxi4Lite.cpp.

{}

Member Function Documentation

connect() [1/2]

void uhallibs::Axi4Lite::connect ( )

private

Set up the connection to the device.

Definition at line 330 of file ProtocolAxi4Lite.cpp.

{
  IPCScopedLock_t lLockGuard(*mIPCMutex);
  connect(lLockGuard);
}

connect() [2/2]

void uhallibs::Axi4Lite::connect ( IPCScopedLock_t & aGuard )

private

Set up the connection to the device.

Definition at line 337 of file ProtocolAxi4Lite.cpp.

{
  // Read current value of session counter when reading status info from FPGA
  // (So that can check whether this info is up-to-date later on, when sending next request packet)
  mIPCExternalSessionActive = mIPCMutex->isActive() and (not mMappedFile.haveLock());
  mIPCSessionCount = mIPCMutex->getCounter();
 
  log(uhal::Debug(),
      "Axi4Lite client is opening device file ",
      uhal::Quote(mMappedFile.getPath()),
      " (device-to-client)");
 
  // Minimal mapping to read the
  mMappedFile.setLength(0x10);
  mMappedFile.open();
  std::vector<uint32_t> lStats;
  mMappedFile.read(0, 4, lStats);
  mMappedFile.close();
  aGuard.unlock();
 
  mNumberOfPages = lStats.at(0);
  if ((mMaxInFlight == 0) or (mMaxInFlight > mNumberOfPages))
    mMaxInFlight = mNumberOfPages;
  mPageSize = lStats.at(1);
  if ((mMaxPacketSize == 0) or (mMaxPacketSize >= mPageSize))
    mMaxPacketSize = mPageSize - 1;
  mIndexNextPage = lStats.at(2);
  mPublishedReplyPageCount = lStats.at(3);
  mReadReplyPageCount = mPublishedReplyPageCount;
 
  //
  constexpr uint32_t lSafetyMargin(4096);
  // Set the memory mapping range to a value commensurate to the memory available in firmware
  mMappedFile.setLength(mNumberOfPages * mPageSize + 4 + lSafetyMargin);
  mMappedFile.open();
 
  mConnected = true;
 
  log(uhal::Info(),
      "Axi4Lite client connected to device at ",
      uhal::Quote(mMappedFile.getPath()),
      ", FPGA has ",
      Integer(mNumberOfPages),
      " pages, each of size ",
      Integer(mPageSize),
      " words, index ",
      Integer(mIndexNextPage),
      " should be filled next");
}

disconnect()

void uhallibs::Axi4Lite::disconnect ( )

private

Close the connection to the device.

Definition at line 388 of file ProtocolAxi4Lite.cpp.

{
  mMappedFile.close();
  mConnected = false;
}

dispatchExceptionHandler()

void uhallibs::Axi4Lite::dispatchExceptionHandler ( )

privatevirtual

Function which tidies up this protocol layer in the event of an exception.

Definition at line 293 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Notice(),
      "Axi4Lite client ",
      uhal::Quote(id()),
      " (URI: ",
      uhal::Quote(uri()),
      ") : closing device files since exception detected");
 
  ClientInterface::returnBufferToPool(mReplyQueue);
 
  mMappedFile.unlock();
 
  disconnect();
 
  InnerProtocol::dispatchExceptionHandler();
}

Flush()

void uhallibs::Axi4Lite::Flush ( )

privatevirtual

Concrete implementation of the synchronization function to block until all buffers have been sent, all replies received and all data validated

Definition at line 280 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : Flush method called");
  while (!mReplyQueue.empty())
    read();
 
  mMappedFile.unlock();
 
  IPCScopedLock_t lLockGuard(*mIPCMutex);
  mIPCMutex->endSession();
}

getDevicePath()

std::string uhallibs::Axi4Lite::getDevicePath ( const uhal::URI & aUri )

staticprivate

Definition at line 231 of file ProtocolAxi4Lite.cpp.

{
 
  std::string lPath = aUri.mHostname;
 
  auto it = std::find_if(aUri.mArguments.begin(),
                         aUri.mArguments.end(),
                         [](const std::pair<std::string, std::string>& element) { return element.first == "dev"; });
 
  if (it != aUri.mArguments.end()) {
    lPath += "/" + it->second;
  }
 
  return lPath;
}

getMaxReplySize()

uint32_t uhallibs::Axi4Lite::getMaxReplySize ( )

private

Return the maximum size of reply packet based on the buffer size in the target

Returns

the maximum size of reply packet

Definition at line 321 of file ProtocolAxi4Lite.cpp.

{
  if (!mConnected)
    connect();
 
  return mMaxPacketSize * 4;
}

getMaxSendSize()

uint32_t uhallibs::Axi4Lite::getMaxSendSize ( )

private

Return the maximum size to be sent based on the buffer size in the target

Returns

the maximum size to be sent

Definition at line 312 of file ProtocolAxi4Lite.cpp.

{
  if (!mConnected)
    connect();
 
  return mMaxPacketSize * 4;
}

getSharedMemName()

std::string uhallibs::Axi4Lite::getSharedMemName ( const std::string & aPath )

staticprivate

Definition at line 222 of file ProtocolAxi4Lite.cpp.

{
  std::string lSanitizedPath(aPath);
  std::replace(lSanitizedPath.begin(), lSanitizedPath.end(), '/', ':');
 
  return "/uhal::ipbusaxi4lite-2.0::" + lSanitizedPath;
}

implementDispatch()

void uhallibs::Axi4Lite::implementDispatch ( std::shared_ptr< uhal::Buffers > aBuffers )

private

Send the IPbus buffer to the target, read back the response and call the packing-protocol's validate function

Parameters

aBuffers

the buffer object wrapping the send and recieve buffers that are to be transported If multithreaded, adds buffer to the dispatch queue and returns. If single-threaded, calls the dispatch-worker dispatch function directly and blocks until the response is validated.

Definition at line 267 of file ProtocolAxi4Lite.cpp.

{
  log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : implementDispatch method called");
 
  if (!mConnected)
    connect();
 
  if (mReplyQueue.size() == mMaxInFlight)
    read();
  write(aBuffers);
}

operator=()

Axi4Lite & uhallibs::Axi4Lite::operator= ( const Axi4Lite & )

delete

read()

void uhallibs::Axi4Lite::read ( )

private

Read next pending reply packet from appropriate page of FPGA-to-host device file, and validate contents.

Definition at line 443 of file ProtocolAxi4Lite.cpp.

{
  const size_t lPageIndexToRead = (mIndexNextPage - mReplyQueue.size() + mNumberOfPages) % mNumberOfPages;
  SteadyClock_t::time_point lStartTime = SteadyClock_t::now();
 
  if (mReadReplyPageCount == mPublishedReplyPageCount) {
    uint32_t lHwPublishedPageCount = 0x0;
 
    std::vector<uint32_t> lValues;
    while (true) {
      // FIXME : Improve by simply adding fileWrite method that takes uint32_t ref as argument (or returns uint32_t)
      IPCScopedLock_t lGuard(*mIPCMutex);
      mMappedFile.read(0, 4, lValues);
      lHwPublishedPageCount = lValues.at(3);
      log(uhal::Debug(),
          "Read status info from addr 0 (",
          Integer(lValues.at(0)),
          ", ",
          Integer(lValues.at(1)),
          ", ",
          Integer(lValues.at(2)),
          ", ",
          Integer(lValues.at(3)),
          "): ",
          PacketFmt((const uint8_t*)lValues.data(), 4 * lValues.size()));
 
      if (lHwPublishedPageCount != mPublishedReplyPageCount) {
        mPublishedReplyPageCount = lHwPublishedPageCount;
        break;
      }
      // FIXME: Throw if published page count is invalid number
 
      if (SteadyClock_t::now() - lStartTime > std::chrono::microseconds(getBoostTimeoutPeriod().total_microseconds())) {
        exception::Axi4LiteTimeout lExc;
        log(lExc,
            "Next page (index ",
            Integer(lPageIndexToRead),
            " count ",
            Integer(mPublishedReplyPageCount + 1),
            ") of Axi4Lite device '" + mMappedFile.getPath() + "' is not ready after timeout period");
        throw lExc;
      }
 
      log(uhal::Debug(),
          "Axi4Lite client ",
          uhal::Quote(id()),
          " (URI: ",
          uhal::Quote(uri()),
          ") : Trying to read page index ",
          Integer(lPageIndexToRead),
          " = count ",
          Integer(mReadReplyPageCount + 1),
          "; published page count is ",
          Integer(lHwPublishedPageCount),
          "; sleeping for ",
          mSleepDuration.count(),
          "us");
      if (mSleepDuration > std::chrono::microseconds(0))
        std::this_thread::sleep_for(mSleepDuration);
      lValues.clear();
    }
 
    log(uhal::Info(),
        "Axi4Lite client ",
        uhal::Quote(id()),
        " (URI: ",
        uhal::Quote(uri()),
        ") : Reading page ",
        Integer(lPageIndexToRead),
        " (published count ",
        Integer(lHwPublishedPageCount),
        ", surpasses required, ",
        Integer(mReadReplyPageCount + 1),
        ")");
  }
  mReadReplyPageCount++;
 
  // PART 1 : Read the page
  std::shared_ptr<uhal::Buffers> lBuffers = mReplyQueue.front();
  mReplyQueue.pop_front();
 
  uint32_t lNrWordsToRead(lBuffers->replyCounter() >> 2);
  lNrWordsToRead += 1;
 
  std::vector<uint32_t> lPageContents;
  IPCScopedLock_t lGuard(*mIPCMutex);
  mMappedFile.read(4 + lPageIndexToRead * mPageSize, lNrWordsToRead, lPageContents);
  lGuard.unlock();
  log(uhal::Debug(),
      "Read ",
      Integer(lNrWordsToRead),
      " 32-bit words from address ",
      Integer(4 + lPageIndexToRead * 4 * mPageSize),
      " ... ",
      PacketFmt((const uint8_t*)lPageContents.data(), 4 * lPageContents.size()));
 
  // PART 2 : Transfer to reply buffer
  const std::deque<std::pair<uint8_t*, uint32_t>>& lReplyBuffers(lBuffers->getReplyBuffer());
  size_t lNrWordsInPacket = (lPageContents.at(0) >> 16) + (lPageContents.at(0) & 0xFFFF);
  if (lNrWordsInPacket != (lBuffers->replyCounter() >> 2))
    log(uhal::Warning(),
        "Expected reply packet to contain ",
        Integer(lBuffers->replyCounter() >> 2),
        " words, but it actually contains ",
        Integer(lNrWordsInPacket),
        " words");
 
  size_t lNrBytesCopied = 0;
  for (const auto& lBuffer : lReplyBuffers) {
    // Don't copy more of page than was written to, for cases when less data received than expected
    if (lNrBytesCopied >= 4 * lNrWordsInPacket)
      break;
 
    size_t lNrBytesToCopy = std::min(lBuffer.second, uint32_t(4 * lNrWordsInPacket - lNrBytesCopied));
    memcpy(lBuffer.first, &lPageContents.at(1 + (lNrBytesCopied / 4)), lNrBytesToCopy);
    lNrBytesCopied += lNrBytesToCopy;
  }
 
  // PART 3 : Validate the packet contents
  uhal::exception::exception* lExc = NULL;
  try {
    lExc = ClientInterface::validate(lBuffers);
  } catch (uhal::exception::exception& aExc) {
    uhal::exception::ValidationError lExc2;
    log(lExc2,
        "Exception caught during reply validation for Axi4Lite device with URI ",
        uhal::Quote(this->uri()),
        "; what returned: ",
        uhal::Quote(aExc.what()));
    throw lExc2;
  }
 
  if (lExc != NULL)
    lExc->throwAsDerivedType();
}

write()

void uhallibs::Axi4Lite::write ( const std::shared_ptr< uhal::Buffers > & aBuffers )

private

Write request packet to next page in host-to-FPGA device file.

Definition at line 395 of file ProtocolAxi4Lite.cpp.

{
  if (not mMappedFile.haveLock()) {
    mMappedFile.lock();
 
    IPCScopedLock_t lGuard(*mIPCMutex);
    mIPCMutex->startSession();
    mIPCSessionCount++;
 
    // If these two numbers don't match, another client/process has sent packets
    // more recently than this client has, so must re-read status info
    if (mIPCExternalSessionActive or (mIPCMutex->getCounter() != mIPCSessionCount)) {
      connect(lGuard);
    }
  }
 
  log(uhal::Info(),
      "Axi4Lite client ",
      uhal::Quote(id()),
      " (URI: ",
      uhal::Quote(uri()),
      ") : writing ",
      Integer(aBuffers->sendCounter() / 4),
      "-word packet to page ",
      Integer(mIndexNextPage),
      " in ",
      uhal::Quote(mMappedFile.getPath()));
 
  const uint32_t lHeaderWord = (0x10000 | (((aBuffers->sendCounter() / 4) - 1) & 0xFFFF));
  std::vector<std::pair<const uint8_t*, size_t>> lDataToWrite;
  lDataToWrite.push_back(std::make_pair(reinterpret_cast<const uint8_t*>(&lHeaderWord), sizeof lHeaderWord));
  lDataToWrite.push_back(std::make_pair(aBuffers->getSendBuffer(), aBuffers->sendCounter()));
 
  IPCScopedLock_t lGuard(*mIPCMutex);
  mMappedFile.write(mIndexNextPage * mPageSize, lDataToWrite);
  log(uhal::Debug(),
      "Wrote ",
      Integer((aBuffers->sendCounter() / 4) + 1),
      " 32-bit words at address ",
      Integer(mIndexNextPage * mPageSize),
      " ... ",
      PacketFmt(lDataToWrite));
 
  mIndexNextPage = (mIndexNextPage + 1) % mNumberOfPages;
  mReplyQueue.push_back(aBuffers);
}

Member Data Documentation

mConnected

bool uhallibs::Axi4Lite::mConnected

private

Definition at line 205 of file ProtocolAxi4Lite.hpp.

mIndexNextPage

uint32_t uhallibs::Axi4Lite::mIndexNextPage

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mIPCExternalSessionActive

bool uhallibs::Axi4Lite::mIPCExternalSessionActive

private

Definition at line 210 of file ProtocolAxi4Lite.hpp.

mIPCMutex

ipc::SharedMemObject<IPCMutex_t> uhallibs::Axi4Lite::mIPCMutex

private

Definition at line 209 of file ProtocolAxi4Lite.hpp.

mIPCSessionCount

uint64_t uhallibs::Axi4Lite::mIPCSessionCount

private

Definition at line 211 of file ProtocolAxi4Lite.hpp.

mMappedFile

MappedFile uhallibs::Axi4Lite::mMappedFile

private

Definition at line 207 of file ProtocolAxi4Lite.hpp.

mMaxInFlight

uint32_t uhallibs::Axi4Lite::mMaxInFlight

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mMaxPacketSize

uint32_t uhallibs::Axi4Lite::mMaxPacketSize

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mNumberOfPages

uint32_t uhallibs::Axi4Lite::mNumberOfPages

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mPageSize

uint32_t uhallibs::Axi4Lite::mPageSize

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mPublishedReplyPageCount

uint32_t uhallibs::Axi4Lite::mPublishedReplyPageCount

private

Definition at line 215 of file ProtocolAxi4Lite.hpp.

mReadReplyPageCount

uint32_t uhallibs::Axi4Lite::mReadReplyPageCount

private

Definition at line 216 of file ProtocolAxi4Lite.hpp.

mReplyQueue

std::deque<std::shared_ptr<uhal::Buffers> > uhallibs::Axi4Lite::mReplyQueue

private

The list of buffers still awaiting a reply.

Definition at line 219 of file ProtocolAxi4Lite.hpp.

mSleepDuration

std::chrono::microseconds uhallibs::Axi4Lite::mSleepDuration

private

Definition at line 213 of file ProtocolAxi4Lite.hpp.

---

The documentation for this class was generated from the following files:

• /github/workspace/dunedaq/sourcecode/uhallibs/include/uhallibs/ProtocolAxi4Lite.hpp
• /github/workspace/dunedaq/sourcecode/uhallibs/src/ProtocolAxi4Lite.cpp