Line data Source code
1 : #include "uhallibs/ProtocolAxi4Lite.hpp"
2 :
3 : #include <fcntl.h>
4 : #include <sys/file.h>
5 : #include <sys/mman.h>
6 : #include <thread>
7 : #include <unistd.h>
8 :
9 : #include "uhallibs/formatters.hpp"
10 :
11 : #include "uhal/Buffers.hpp"
12 : #include "uhal/ClientFactory.hpp"
13 :
14 : using uhal::Integer;
15 :
16 : UHAL_REGISTER_EXTERNAL_CLIENT(uhallibs::Axi4Lite, "ipbusaxi4lite-2.0", "AXI4 Lite IPBus client")
17 : // Syntax: ipbus-qdma-axi4l-2.0:///sys/bus/pci/devices/0000:<bus>:<dev>.<func>/resource<bar#>
18 : // Syntax: ipbusaxi4lite-2.0://<bus>:<dev>.<func>/resource<bar#>
19 : //
20 : namespace uhallibs {
21 :
22 0 : Axi4Lite::MappedFile::MappedFile(const std::string& aPath, size_t aLength, int aProtFlags)
23 0 : : mPath(aPath)
24 0 : , mFd(-1)
25 0 : , mBar(nullptr)
26 0 : , mLength(aLength)
27 0 : , mProtFlags(aProtFlags)
28 0 : , mLocked(false)
29 0 : , mBufferSize(0)
30 0 : , mBuffer(nullptr)
31 : {
32 0 : }
33 :
34 0 : Axi4Lite::MappedFile::~MappedFile()
35 : {
36 0 : if (mBuffer != nullptr)
37 0 : free(mBuffer);
38 0 : close();
39 0 : }
40 :
41 : const std::string&
42 0 : Axi4Lite::MappedFile::getPath() const
43 : {
44 0 : return mPath;
45 : }
46 :
47 : void
48 0 : Axi4Lite::MappedFile::setPath(const std::string& aPath)
49 : {
50 0 : mPath = aPath;
51 0 : }
52 : void
53 0 : Axi4Lite::MappedFile::setLength(size_t aLength)
54 : {
55 0 : mLength = aLength;
56 0 : }
57 :
58 : void
59 0 : Axi4Lite::MappedFile::open()
60 : {
61 0 : if (mBar != nullptr)
62 : return;
63 :
64 0 : mFd = ::open(mPath.c_str(), (mProtFlags & PROT_WRITE) ? O_RDWR : O_RDONLY);
65 0 : if (mFd < 0) {
66 : return;
67 : }
68 :
69 0 : void* lBar = mmap(nullptr, 4 * mLength, mProtFlags, MAP_SHARED, mFd, 0);
70 0 : mBar = (lBar == MAP_FAILED ? nullptr : (uint32_t*)lBar);
71 : }
72 :
73 : void
74 0 : Axi4Lite::MappedFile::close()
75 : {
76 0 : if (mBar != nullptr)
77 0 : munmap(mBar, mLength);
78 0 : mBar = nullptr;
79 :
80 0 : if (mFd != -1) {
81 0 : if (haveLock())
82 0 : unlock();
83 0 : int rc = ::close(mFd);
84 0 : mFd = -1;
85 0 : if (rc == -1)
86 0 : log(uhal::Error(),
87 : "Failed to close file ",
88 0 : uhal::Quote(mPath),
89 : "; errno=",
90 0 : uhal::Integer(errno),
91 : ", meaning ",
92 0 : uhal::Quote(strerror(errno)));
93 : }
94 0 : }
95 :
96 : void
97 0 : Axi4Lite::MappedFile::createBuffer(const size_t aNrBytes)
98 : {
99 0 : if (mBuffer != NULL) {
100 0 : if (mBufferSize >= aNrBytes)
101 : return;
102 : else {
103 0 : free(mBuffer);
104 0 : mBuffer = NULL;
105 0 : mBufferSize = 0;
106 : }
107 : }
108 :
109 0 : posix_memalign((void**)&mBuffer, 4096 /*alignment*/, aNrBytes + 4096);
110 0 : if (mBuffer == NULL) {
111 0 : exception::Axi4LiteCommunicationError lExc;
112 0 : log(lExc, "Failed to allocate ", Integer(aNrBytes + 4096), " bytes in Axi4Lite::MappedFile::createBuffer");
113 0 : throw lExc;
114 0 : }
115 :
116 0 : mBufferSize = aNrBytes + 4096;
117 : }
118 :
119 : void
120 0 : Axi4Lite::MappedFile::read(const uint32_t aAddr, const uint32_t aNrWords, std::vector<uint32_t>& aValues)
121 : {
122 0 : if (mBar == nullptr)
123 0 : open();
124 :
125 0 : for (size_t i(0); i < aNrWords; ++i) {
126 0 : aValues.push_back(le32toh(mBar[aAddr + i]));
127 : }
128 0 : }
129 :
130 : void
131 0 : Axi4Lite::MappedFile::write(const uint32_t aAddr, const std::vector<uint32_t>& aValues)
132 : {
133 0 : write(aAddr, reinterpret_cast<const uint8_t*>(aValues.data()), 4 * aValues.size());
134 0 : }
135 :
136 : void
137 0 : Axi4Lite::MappedFile::write(const uint32_t aAddr, const uint8_t* const aPtr, const size_t aNrBytes)
138 : {
139 0 : if (mBar == nullptr)
140 0 : open();
141 :
142 0 : assert((aNrBytes % 4) == 0);
143 0 : uint32_t lNrWordsData = aNrBytes / 4;
144 :
145 0 : auto lPtr32 = reinterpret_cast<const uint32_t*>(aPtr);
146 0 : for (size_t i(0); i < lNrWordsData; ++i) {
147 0 : mBar[aAddr + i] = lPtr32[i];
148 : }
149 0 : }
150 :
151 : void
152 0 : Axi4Lite::MappedFile::write(const uint32_t aAddr, const std::vector<std::pair<const uint8_t*, size_t>>& aData)
153 : {
154 0 : if (mBar == nullptr)
155 0 : open();
156 :
157 : size_t lNrBytes = 0;
158 0 : for (size_t i = 0; i < aData.size(); i++)
159 0 : lNrBytes += aData.at(i).second;
160 :
161 0 : assert((lNrBytes % 4) == 0);
162 0 : size_t lNrWords = lNrBytes / 4;
163 :
164 0 : createBuffer(lNrBytes);
165 :
166 0 : size_t k(0);
167 0 : for (size_t i = 0; i < aData.size(); ++i) {
168 0 : for (size_t j = 0; j < aData.at(i).second; ++j) {
169 0 : mBuffer[k] = aData.at(i).first[j];
170 0 : ++k;
171 : }
172 : }
173 :
174 0 : auto mBuffer32b = reinterpret_cast<const uint32_t*>(mBuffer);
175 :
176 0 : for (size_t i(0); i < lNrWords; ++i) {
177 0 : mBar[aAddr + i] = htole32(mBuffer32b[i]);
178 : }
179 : // std::memcpy(mBar + aAddr, mBuffer, lNrBytes);
180 0 : }
181 :
182 : bool
183 0 : Axi4Lite::MappedFile::haveLock() const
184 : {
185 0 : return mLocked;
186 : }
187 :
188 : void
189 0 : Axi4Lite::MappedFile::lock()
190 : {
191 0 : if (flock(mFd, LOCK_EX) == -1) {
192 0 : ipc::exception::MutexError lExc;
193 0 : log(lExc,
194 : "Failed to lock device file ",
195 0 : uhal::Quote(mPath),
196 : "; errno=",
197 0 : Integer(errno),
198 : ", meaning ",
199 0 : uhal::Quote(strerror(errno)));
200 0 : throw lExc;
201 0 : }
202 0 : mLocked = true;
203 0 : }
204 :
205 : void
206 0 : Axi4Lite::MappedFile::unlock()
207 : {
208 0 : if (flock(mFd, LOCK_UN) == -1) {
209 0 : log(uhal::Warning(),
210 : "Failed to unlock device file ",
211 0 : uhal::Quote(mPath),
212 : "; errno=",
213 0 : Integer(errno),
214 : ", meaning ",
215 0 : uhal::Quote(strerror(errno)));
216 : } else
217 0 : mLocked = false;
218 0 : }
219 :
220 : // Axi4Lite Transport
221 : std::string
222 0 : Axi4Lite::getSharedMemName(const std::string& aPath)
223 : {
224 0 : std::string lSanitizedPath(aPath);
225 0 : std::replace(lSanitizedPath.begin(), lSanitizedPath.end(), '/', ':');
226 :
227 0 : return "/uhal::ipbusaxi4lite-2.0::" + lSanitizedPath;
228 0 : }
229 :
230 : std::string
231 0 : Axi4Lite::getDevicePath(const uhal::URI& aUri)
232 : {
233 :
234 0 : std::string lPath = aUri.mHostname;
235 :
236 0 : auto it = std::find_if(aUri.mArguments.begin(),
237 : aUri.mArguments.end(),
238 0 : [](const std::pair<std::string, std::string>& element) { return element.first == "dev"; });
239 :
240 0 : if (it != aUri.mArguments.end()) {
241 0 : lPath += "/" + it->second;
242 : }
243 :
244 0 : return lPath;
245 0 : }
246 :
247 0 : Axi4Lite::Axi4Lite(const std::string& aId, const uhal::URI& aUri)
248 : : IPbus<2, 0>(aId, aUri)
249 0 : , mConnected(false)
250 0 : , mMappedFile(getDevicePath(aUri), 64, PROT_WRITE)
251 0 : , mIPCMutex(getSharedMemName(mMappedFile.getPath()))
252 0 : , mNumberOfPages(0)
253 0 : , mMaxInFlight(0)
254 0 : , mPageSize(0)
255 0 : , mMaxPacketSize(0)
256 0 : , mIndexNextPage(0)
257 0 : , mPublishedReplyPageCount(0)
258 0 : , mReadReplyPageCount(0)
259 : {
260 :
261 0 : mSleepDuration = std::chrono::microseconds(50);
262 0 : }
263 :
264 0 : Axi4Lite::~Axi4Lite() {}
265 :
266 : void
267 0 : Axi4Lite::implementDispatch(std::shared_ptr<uhal::Buffers> aBuffers)
268 : {
269 0 : log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : implementDispatch method called");
270 :
271 0 : if (!mConnected)
272 0 : connect();
273 :
274 0 : if (mReplyQueue.size() == mMaxInFlight)
275 0 : read();
276 0 : write(aBuffers);
277 0 : }
278 :
279 : void
280 0 : Axi4Lite::Flush()
281 : {
282 0 : log(uhal::Debug(), "Axi4Lite client (URI: ", uhal::Quote(uri()), ") : Flush method called");
283 0 : while (!mReplyQueue.empty())
284 0 : read();
285 :
286 0 : mMappedFile.unlock();
287 :
288 0 : IPCScopedLock_t lLockGuard(*mIPCMutex);
289 0 : mIPCMutex->endSession();
290 0 : }
291 :
292 : void
293 0 : Axi4Lite::dispatchExceptionHandler()
294 : {
295 0 : log(uhal::Notice(),
296 : "Axi4Lite client ",
297 0 : uhal::Quote(id()),
298 : " (URI: ",
299 0 : uhal::Quote(uri()),
300 : ") : closing device files since exception detected");
301 :
302 0 : ClientInterface::returnBufferToPool(mReplyQueue);
303 :
304 0 : mMappedFile.unlock();
305 :
306 0 : disconnect();
307 :
308 0 : InnerProtocol::dispatchExceptionHandler();
309 0 : }
310 :
311 : uint32_t
312 0 : Axi4Lite::getMaxSendSize()
313 : {
314 0 : if (!mConnected)
315 0 : connect();
316 :
317 0 : return mMaxPacketSize * 4;
318 : }
319 :
320 : uint32_t
321 0 : Axi4Lite::getMaxReplySize()
322 : {
323 0 : if (!mConnected)
324 0 : connect();
325 :
326 0 : return mMaxPacketSize * 4;
327 : }
328 :
329 : void
330 0 : Axi4Lite::connect()
331 : {
332 0 : IPCScopedLock_t lLockGuard(*mIPCMutex);
333 0 : connect(lLockGuard);
334 0 : }
335 :
336 : void
337 0 : Axi4Lite::connect(IPCScopedLock_t& aGuard)
338 : {
339 : // Read current value of session counter when reading status info from FPGA
340 : // (So that can check whether this info is up-to-date later on, when sending next request packet)
341 0 : mIPCExternalSessionActive = mIPCMutex->isActive() and (not mMappedFile.haveLock());
342 0 : mIPCSessionCount = mIPCMutex->getCounter();
343 :
344 0 : log(uhal::Debug(),
345 : "Axi4Lite client is opening device file ",
346 0 : uhal::Quote(mMappedFile.getPath()),
347 : " (device-to-client)");
348 :
349 : // Minimal mapping to read the
350 0 : mMappedFile.setLength(0x10);
351 0 : mMappedFile.open();
352 0 : std::vector<uint32_t> lStats;
353 0 : mMappedFile.read(0, 4, lStats);
354 0 : mMappedFile.close();
355 0 : aGuard.unlock();
356 :
357 0 : mNumberOfPages = lStats.at(0);
358 0 : if ((mMaxInFlight == 0) or (mMaxInFlight > mNumberOfPages))
359 0 : mMaxInFlight = mNumberOfPages;
360 0 : mPageSize = lStats.at(1);
361 0 : if ((mMaxPacketSize == 0) or (mMaxPacketSize >= mPageSize))
362 0 : mMaxPacketSize = mPageSize - 1;
363 0 : mIndexNextPage = lStats.at(2);
364 0 : mPublishedReplyPageCount = lStats.at(3);
365 0 : mReadReplyPageCount = mPublishedReplyPageCount;
366 :
367 : //
368 0 : constexpr uint32_t lSafetyMargin(4096);
369 : // Set the memory mapping range to a value commensurate to the memory available in firmware
370 0 : mMappedFile.setLength(mNumberOfPages * mPageSize + 4 + lSafetyMargin);
371 0 : mMappedFile.open();
372 :
373 0 : mConnected = true;
374 :
375 0 : log(uhal::Info(),
376 : "Axi4Lite client connected to device at ",
377 0 : uhal::Quote(mMappedFile.getPath()),
378 : ", FPGA has ",
379 0 : Integer(mNumberOfPages),
380 : " pages, each of size ",
381 0 : Integer(mPageSize),
382 : " words, index ",
383 0 : Integer(mIndexNextPage),
384 : " should be filled next");
385 0 : }
386 :
387 : void
388 0 : Axi4Lite::disconnect()
389 : {
390 0 : mMappedFile.close();
391 0 : mConnected = false;
392 0 : }
393 :
394 : void
395 0 : Axi4Lite::write(const std::shared_ptr<uhal::Buffers>& aBuffers)
396 : {
397 0 : if (not mMappedFile.haveLock()) {
398 0 : mMappedFile.lock();
399 :
400 0 : IPCScopedLock_t lGuard(*mIPCMutex);
401 0 : mIPCMutex->startSession();
402 0 : mIPCSessionCount++;
403 :
404 : // If these two numbers don't match, another client/process has sent packets
405 : // more recently than this client has, so must re-read status info
406 0 : if (mIPCExternalSessionActive or (mIPCMutex->getCounter() != mIPCSessionCount)) {
407 0 : connect(lGuard);
408 : }
409 0 : }
410 :
411 0 : log(uhal::Info(),
412 : "Axi4Lite client ",
413 0 : uhal::Quote(id()),
414 : " (URI: ",
415 0 : uhal::Quote(uri()),
416 : ") : writing ",
417 0 : Integer(aBuffers->sendCounter() / 4),
418 : "-word packet to page ",
419 0 : Integer(mIndexNextPage),
420 : " in ",
421 0 : uhal::Quote(mMappedFile.getPath()));
422 :
423 0 : const uint32_t lHeaderWord = (0x10000 | (((aBuffers->sendCounter() / 4) - 1) & 0xFFFF));
424 0 : std::vector<std::pair<const uint8_t*, size_t>> lDataToWrite;
425 0 : lDataToWrite.push_back(std::make_pair(reinterpret_cast<const uint8_t*>(&lHeaderWord), sizeof lHeaderWord));
426 0 : lDataToWrite.push_back(std::make_pair(aBuffers->getSendBuffer(), aBuffers->sendCounter()));
427 :
428 0 : IPCScopedLock_t lGuard(*mIPCMutex);
429 0 : mMappedFile.write(mIndexNextPage * mPageSize, lDataToWrite);
430 0 : log(uhal::Debug(),
431 : "Wrote ",
432 0 : Integer((aBuffers->sendCounter() / 4) + 1),
433 : " 32-bit words at address ",
434 0 : Integer(mIndexNextPage * mPageSize),
435 : " ... ",
436 0 : PacketFmt(lDataToWrite));
437 :
438 0 : mIndexNextPage = (mIndexNextPage + 1) % mNumberOfPages;
439 0 : mReplyQueue.push_back(aBuffers);
440 0 : }
441 :
442 : void
443 0 : Axi4Lite::read()
444 : {
445 0 : const size_t lPageIndexToRead = (mIndexNextPage - mReplyQueue.size() + mNumberOfPages) % mNumberOfPages;
446 0 : SteadyClock_t::time_point lStartTime = SteadyClock_t::now();
447 :
448 0 : if (mReadReplyPageCount == mPublishedReplyPageCount) {
449 0 : uint32_t lHwPublishedPageCount = 0x0;
450 :
451 0 : std::vector<uint32_t> lValues;
452 0 : while (true) {
453 : // FIXME : Improve by simply adding fileWrite method that takes uint32_t ref as argument (or returns uint32_t)
454 0 : IPCScopedLock_t lGuard(*mIPCMutex);
455 0 : mMappedFile.read(0, 4, lValues);
456 0 : lHwPublishedPageCount = lValues.at(3);
457 0 : log(uhal::Debug(),
458 : "Read status info from addr 0 (",
459 0 : Integer(lValues.at(0)),
460 : ", ",
461 0 : Integer(lValues.at(1)),
462 : ", ",
463 0 : Integer(lValues.at(2)),
464 : ", ",
465 0 : Integer(lValues.at(3)),
466 : "): ",
467 0 : PacketFmt((const uint8_t*)lValues.data(), 4 * lValues.size()));
468 :
469 0 : if (lHwPublishedPageCount != mPublishedReplyPageCount) {
470 0 : mPublishedReplyPageCount = lHwPublishedPageCount;
471 0 : break;
472 : }
473 : // FIXME: Throw if published page count is invalid number
474 :
475 0 : if (SteadyClock_t::now() - lStartTime > std::chrono::microseconds(getBoostTimeoutPeriod().total_microseconds())) {
476 0 : exception::Axi4LiteTimeout lExc;
477 0 : log(lExc,
478 : "Next page (index ",
479 0 : Integer(lPageIndexToRead),
480 : " count ",
481 0 : Integer(mPublishedReplyPageCount + 1),
482 0 : ") of Axi4Lite device '" + mMappedFile.getPath() + "' is not ready after timeout period");
483 0 : throw lExc;
484 0 : }
485 :
486 0 : log(uhal::Debug(),
487 : "Axi4Lite client ",
488 0 : uhal::Quote(id()),
489 : " (URI: ",
490 0 : uhal::Quote(uri()),
491 : ") : Trying to read page index ",
492 0 : Integer(lPageIndexToRead),
493 : " = count ",
494 0 : Integer(mReadReplyPageCount + 1),
495 : "; published page count is ",
496 0 : Integer(lHwPublishedPageCount),
497 : "; sleeping for ",
498 0 : mSleepDuration.count(),
499 : "us");
500 0 : if (mSleepDuration > std::chrono::microseconds(0))
501 0 : std::this_thread::sleep_for(mSleepDuration);
502 0 : lValues.clear();
503 0 : }
504 :
505 0 : log(uhal::Info(),
506 : "Axi4Lite client ",
507 0 : uhal::Quote(id()),
508 : " (URI: ",
509 0 : uhal::Quote(uri()),
510 : ") : Reading page ",
511 0 : Integer(lPageIndexToRead),
512 : " (published count ",
513 0 : Integer(lHwPublishedPageCount),
514 : ", surpasses required, ",
515 0 : Integer(mReadReplyPageCount + 1),
516 : ")");
517 0 : }
518 0 : mReadReplyPageCount++;
519 :
520 : // PART 1 : Read the page
521 0 : std::shared_ptr<uhal::Buffers> lBuffers = mReplyQueue.front();
522 0 : mReplyQueue.pop_front();
523 :
524 0 : uint32_t lNrWordsToRead(lBuffers->replyCounter() >> 2);
525 0 : lNrWordsToRead += 1;
526 :
527 0 : std::vector<uint32_t> lPageContents;
528 0 : IPCScopedLock_t lGuard(*mIPCMutex);
529 0 : mMappedFile.read(4 + lPageIndexToRead * mPageSize, lNrWordsToRead, lPageContents);
530 0 : lGuard.unlock();
531 0 : log(uhal::Debug(),
532 : "Read ",
533 0 : Integer(lNrWordsToRead),
534 : " 32-bit words from address ",
535 0 : Integer(4 + lPageIndexToRead * 4 * mPageSize),
536 : " ... ",
537 0 : PacketFmt((const uint8_t*)lPageContents.data(), 4 * lPageContents.size()));
538 :
539 : // PART 2 : Transfer to reply buffer
540 0 : const std::deque<std::pair<uint8_t*, uint32_t>>& lReplyBuffers(lBuffers->getReplyBuffer());
541 0 : size_t lNrWordsInPacket = (lPageContents.at(0) >> 16) + (lPageContents.at(0) & 0xFFFF);
542 0 : if (lNrWordsInPacket != (lBuffers->replyCounter() >> 2))
543 0 : log(uhal::Warning(),
544 : "Expected reply packet to contain ",
545 0 : Integer(lBuffers->replyCounter() >> 2),
546 : " words, but it actually contains ",
547 0 : Integer(lNrWordsInPacket),
548 : " words");
549 :
550 0 : size_t lNrBytesCopied = 0;
551 0 : for (const auto& lBuffer : lReplyBuffers) {
552 : // Don't copy more of page than was written to, for cases when less data received than expected
553 0 : if (lNrBytesCopied >= 4 * lNrWordsInPacket)
554 : break;
555 :
556 0 : size_t lNrBytesToCopy = std::min(lBuffer.second, uint32_t(4 * lNrWordsInPacket - lNrBytesCopied));
557 0 : memcpy(lBuffer.first, &lPageContents.at(1 + (lNrBytesCopied / 4)), lNrBytesToCopy);
558 0 : lNrBytesCopied += lNrBytesToCopy;
559 : }
560 :
561 : // PART 3 : Validate the packet contents
562 0 : uhal::exception::exception* lExc = NULL;
563 0 : try {
564 0 : lExc = ClientInterface::validate(lBuffers);
565 0 : } catch (uhal::exception::exception& aExc) {
566 0 : uhal::exception::ValidationError lExc2;
567 0 : log(lExc2,
568 : "Exception caught during reply validation for Axi4Lite device with URI ",
569 0 : uhal::Quote(this->uri()),
570 : "; what returned: ",
571 0 : uhal::Quote(aExc.what()));
572 0 : throw lExc2;
573 0 : }
574 :
575 0 : if (lExc != NULL)
576 0 : lExc->throwAsDerivedType();
577 0 : }
578 :
579 : } // namespace uhal
|
