From ccc5ac433e8e8fdb70e3f6e6cb1b54d6e1ff5458 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 24 May 2024 16:43:19 +0000 Subject: [PATCH 01/10] fixing issue #34 adjusting the frequence of drops --- CMakeLists.txt | 7 +++++++ src/include/pmd.h | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ccd46f71..0fe5a21d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,4 +34,11 @@ include_directories(SYSTEM third_party/HdrHistogram_c/include/) include_directories(SYSTEM third_party/xxHash) set(BUILD_TESTING ON) +option(DEBUG "Enable debug mode" OFF) +if(DEBUG) + add_definitions(-DkTESTING=1) + message(STATUS "Debug mode is ON") +else() + message(STATUS "Debug mode is OFF") +endif() add_subdirectory(src) diff --git a/src/include/pmd.h b/src/include/pmd.h index 76f08906..2908c064 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -103,6 +104,27 @@ class TxRing : public PmdRing { void Init(); +#ifdef kTESTING + /// @brief Drops a random packet from the given array of packets. + /// @param pkts Array of packet pointers. + /// @param nb_pkts Number of packets in the array. + void DropRandomPacket(Packet **pkts, uint16_t nb_pkts) const { + unsigned int seed = 123; + static int counter = 0; + counter++; + if (counter == 10000000) { + int random_packet = rand_r(&seed) % nb_pkts; + LOG(INFO) << "Dropping random packet: " << random_packet << " from " + << nb_pkts << " packets."; + Packet::Free(pkts[random_packet]); + for (int i = random_packet; i < nb_pkts - 1; i++) { + pkts[i] = pkts[i + 1]; + } + counter = 0; + } + } +#endif + /** * @brief Tries to send a burst of packets through this TX ring. * @@ -139,6 +161,10 @@ class TxRing : public PmdRing { * @param nb_pkts Number of packets to send. */ void SendPackets(Packet **pkts, uint16_t nb_pkts) const { +#ifdef kTESTING + DropRandomPacket(pkts, nb_pkts); +#endif + uint16_t nb_remaining = nb_pkts; do { From 496496518f258a82550ed11252a939fd8fd5e91f Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 24 May 2024 16:45:07 +0000 Subject: [PATCH 02/10] fixing issue #34 increasing the frequency of drops --- src/include/pmd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index 2908c064..6e320d79 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -112,7 +112,7 @@ class TxRing : public PmdRing { unsigned int seed = 123; static int counter = 0; counter++; - if (counter == 10000000) { + if (counter == 1000000) { int random_packet = rand_r(&seed) % nb_pkts; LOG(INFO) << "Dropping random packet: " << random_packet << " from " << nb_pkts << " packets."; From 0e4d70c3880f434be072a5bd8287ce41aceb03d9 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 24 May 2024 17:52:26 +0000 Subject: [PATCH 03/10] fixing issue #34 resolving a bug in drop random function. --- src/include/pmd.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index 6e320d79..73acc571 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -108,19 +108,20 @@ class TxRing : public PmdRing { /// @brief Drops a random packet from the given array of packets. /// @param pkts Array of packet pointers. /// @param nb_pkts Number of packets in the array. - void DropRandomPacket(Packet **pkts, uint16_t nb_pkts) const { + void DropRandomPacket(Packet **pkts, uint16_t *nb_pkts) const { unsigned int seed = 123; static int counter = 0; counter++; if (counter == 1000000) { - int random_packet = rand_r(&seed) % nb_pkts; + int random_packet = rand_r(&seed) % *nb_pkts; LOG(INFO) << "Dropping random packet: " << random_packet << " from " - << nb_pkts << " packets."; + << *nb_pkts << " packets."; Packet::Free(pkts[random_packet]); - for (int i = random_packet; i < nb_pkts - 1; i++) { + for (int i = random_packet; i < *nb_pkts - 1; i++) { pkts[i] = pkts[i + 1]; } counter = 0; + *nb_pkts = *nb_pkts - 1; } } #endif @@ -162,7 +163,7 @@ class TxRing : public PmdRing { */ void SendPackets(Packet **pkts, uint16_t nb_pkts) const { #ifdef kTESTING - DropRandomPacket(pkts, nb_pkts); + DropRandomPacket(pkts, &nb_pkts); #endif uint16_t nb_remaining = nb_pkts; From f16ee143c05ec16eccb78a3c6672dc771c963666 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Sat, 25 May 2024 10:56:34 +0000 Subject: [PATCH 04/10] Fix issue 34: Add debug mode flag in CMakeLists.txt and also inline function. The code changes add a debug mode flag in the CMakeLists.txt file to control whether the code is built in debug mode or not. The flag is set to 0 when debug mode is off. This change improves the flexibility of the build process and allows for easier debugging when needed. --- CMakeLists.txt | 1 + src/include/pmd.h | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fe5a21d..1073b91b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ if(DEBUG) add_definitions(-DkTESTING=1) message(STATUS "Debug mode is ON") else() + add_definitions(-DkTESTING=0) message(STATUS "Debug mode is OFF") endif() add_subdirectory(src) diff --git a/src/include/pmd.h b/src/include/pmd.h index 73acc571..6af56c11 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -104,11 +104,11 @@ class TxRing : public PmdRing { void Init(); -#ifdef kTESTING +#if kTESTING /// @brief Drops a random packet from the given array of packets. /// @param pkts Array of packet pointers. /// @param nb_pkts Number of packets in the array. - void DropRandomPacket(Packet **pkts, uint16_t *nb_pkts) const { + inline void DropRandomPacket(Packet **pkts, uint16_t *nb_pkts) const { unsigned int seed = 123; static int counter = 0; counter++; @@ -162,7 +162,7 @@ class TxRing : public PmdRing { * @param nb_pkts Number of packets to send. */ void SendPackets(Packet **pkts, uint16_t nb_pkts) const { -#ifdef kTESTING +#if kTESTING DropRandomPacket(pkts, &nb_pkts); #endif From 9161daf6b5898ae225580372dc936a1c300727df Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Sat, 25 May 2024 12:09:52 +0000 Subject: [PATCH 05/10] fix issue #34: create an environ variable for testing machnet in pmd --- CMakeLists.txt | 8 -------- src/include/pmd.h | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1073b91b..ccd46f71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,12 +34,4 @@ include_directories(SYSTEM third_party/HdrHistogram_c/include/) include_directories(SYSTEM third_party/xxHash) set(BUILD_TESTING ON) -option(DEBUG "Enable debug mode" OFF) -if(DEBUG) - add_definitions(-DkTESTING=1) - message(STATUS "Debug mode is ON") -else() - add_definitions(-DkTESTING=0) - message(STATUS "Debug mode is OFF") -endif() add_subdirectory(src) diff --git a/src/include/pmd.h b/src/include/pmd.h index 6af56c11..ae1e70ed 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -59,23 +59,28 @@ class PmdRing { port_id_(port_id), ring_id_(ring_id), ndesc_(ndesc), - ppool_(nullptr) {} + ppool_(nullptr), + machnet_testing(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} PmdRing(const PmdPort *port, uint8_t port_id, uint16_t ring_id, uint16_t ndesc, uint32_t nmbufs, uint32_t mbuf_sz) : pmd_port_(port), port_id_(port_id), ring_id_(ring_id), ndesc_(ndesc), - ppool_(std::unique_ptr(new PacketPool(nmbufs, mbuf_sz))) {} + ppool_(std::unique_ptr(new PacketPool(nmbufs, mbuf_sz))), + machnet_testing(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} rte_mempool *GetPacketMemPool() const { return ppool_.get()->GetMemPool(); } + bool IsMachnetTestingEnabled() const { return machnet_testing; } + private: const PmdPort *pmd_port_; const uint8_t port_id_; const uint16_t ring_id_; const uint16_t ndesc_; const std::unique_ptr ppool_; + const bool machnet_testing; }; /* @@ -104,11 +109,10 @@ class TxRing : public PmdRing { void Init(); -#if kTESTING /// @brief Drops a random packet from the given array of packets. /// @param pkts Array of packet pointers. /// @param nb_pkts Number of packets in the array. - inline void DropRandomPacket(Packet **pkts, uint16_t *nb_pkts) const { + inline void pkt_idx_to_drop(Packet **pkts, uint16_t *nb_pkts) const { unsigned int seed = 123; static int counter = 0; counter++; @@ -124,7 +128,6 @@ class TxRing : public PmdRing { *nb_pkts = *nb_pkts - 1; } } -#endif /** * @brief Tries to send a burst of packets through this TX ring. @@ -162,9 +165,9 @@ class TxRing : public PmdRing { * @param nb_pkts Number of packets to send. */ void SendPackets(Packet **pkts, uint16_t nb_pkts) const { -#if kTESTING - DropRandomPacket(pkts, &nb_pkts); -#endif + if (IsMachnetTestingEnabled()) { + pkt_idx_to_drop(pkts, &nb_pkts); + } uint16_t nb_remaining = nb_pkts; From aeb48ca3d47aa4f8946210616d04d130b1494561 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 28 May 2024 09:25:07 +0000 Subject: [PATCH 06/10] Fix issue #34: Update PmdRing class to use machnet_testing_ member variable * The code changes update the PmdRing class to use the machnet_testing_ member variable instead of the machnet_testing variable. This change ensures consistency and improves code readability. * The extra function deleted and machnet_testing_ variable is used direction as public member. --- src/include/pmd.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index ae1e70ed..30754fac 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -60,7 +60,7 @@ class PmdRing { ring_id_(ring_id), ndesc_(ndesc), ppool_(nullptr), - machnet_testing(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} + machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} PmdRing(const PmdPort *port, uint8_t port_id, uint16_t ring_id, uint16_t ndesc, uint32_t nmbufs, uint32_t mbuf_sz) : pmd_port_(port), @@ -68,19 +68,19 @@ class PmdRing { ring_id_(ring_id), ndesc_(ndesc), ppool_(std::unique_ptr(new PacketPool(nmbufs, mbuf_sz))), - machnet_testing(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} + machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} rte_mempool *GetPacketMemPool() const { return ppool_.get()->GetMemPool(); } - bool IsMachnetTestingEnabled() const { return machnet_testing; } - private: const PmdPort *pmd_port_; const uint8_t port_id_; const uint16_t ring_id_; const uint16_t ndesc_; const std::unique_ptr ppool_; - const bool machnet_testing; + + public: + const bool machnet_testing_; }; /* @@ -165,7 +165,7 @@ class TxRing : public PmdRing { * @param nb_pkts Number of packets to send. */ void SendPackets(Packet **pkts, uint16_t nb_pkts) const { - if (IsMachnetTestingEnabled()) { + if (machnet_testing_) { pkt_idx_to_drop(pkts, &nb_pkts); } From 78acea93cb16ab543ad0f62e943624a8399bb672 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 28 May 2024 10:32:36 +0000 Subject: [PATCH 07/10] Fix issue #34: tried to use FastRand as eRPC --- src/include/pmd.h | 13 +++++++------ src/include/utils.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index 30754fac..3196bc20 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -16,6 +16,7 @@ #include "ether.h" #include "packet.h" #include "packet_pool.h" +#include "utils.h" namespace juggler { namespace dpdk { @@ -113,18 +114,18 @@ class TxRing : public PmdRing { /// @param pkts Array of packet pointers. /// @param nb_pkts Number of packets in the array. inline void pkt_idx_to_drop(Packet **pkts, uint16_t *nb_pkts) const { - unsigned int seed = 123; - static int counter = 0; - counter++; - if (counter == 1000000) { - int random_packet = rand_r(&seed) % *nb_pkts; + static juggler::utils::FastRand fast_rand; + static constexpr uint32_t kBillion = 1000000000; + // the value below is picked randomly as probability. + static constexpr float kDropProb = 0.000001; + if (fast_rand.next_u32() % kBillion < kDropProb * kBillion) { + int random_packet = fast_rand.next_u32() % *nb_pkts; LOG(INFO) << "Dropping random packet: " << random_packet << " from " << *nb_pkts << " packets."; Packet::Free(pkts[random_packet]); for (int i = random_packet; i < *nb_pkts - 1; i++) { pkts[i] = pkts[i + 1]; } - counter = 0; *nb_pkts = *nb_pkts - 1; } } diff --git a/src/include/utils.h b/src/include/utils.h index 40e66d74..eb8d684f 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -466,6 +467,34 @@ class ChronoTimer { std::chrono::time_point start_time_; }; +// Derived from eRPC +class SlowRand { + std::random_device rand_dev_; // Non-pseudorandom seed for twister + std::mt19937_64 mt_; + std::uniform_int_distribution dist_; + + public: + SlowRand() : mt_(rand_dev_()), dist_(0, UINT64_MAX) {} + + inline uint64_t next_u64() { return dist_(mt_); } +}; + +class FastRand { + public: + uint64_t seed_; + + /// Create a FastRand using a seed from SlowRand + FastRand() { + SlowRand slow_rand; + seed_ = slow_rand.next_u64(); + } + + inline uint32_t next_u32() { + seed_ = seed_ * 1103515245 + 12345; + return static_cast(seed_ >> 32); + } +}; + } // namespace utils } // namespace juggler From 4e034ccdc713ecc5b10438693f743592531dd24e Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 28 May 2024 10:42:35 +0000 Subject: [PATCH 08/10] Fix issue #34: Ensure that the env var is set and is 1 for dropping. --- src/include/pmd.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index 3196bc20..12f50078 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -61,7 +61,9 @@ class PmdRing { ring_id_(ring_id), ndesc_(ndesc), ppool_(nullptr), - machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} + machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr && + std::string(std::getenv("MACHNET_TESTING_ENABLED")) == + "1") {} PmdRing(const PmdPort *port, uint8_t port_id, uint16_t ring_id, uint16_t ndesc, uint32_t nmbufs, uint32_t mbuf_sz) : pmd_port_(port), @@ -69,7 +71,9 @@ class PmdRing { ring_id_(ring_id), ndesc_(ndesc), ppool_(std::unique_ptr(new PacketPool(nmbufs, mbuf_sz))), - machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr) {} + machnet_testing_(std::getenv("MACHNET_TESTING_ENABLED") != nullptr && + std::string(std::getenv("MACHNET_TESTING_ENABLED")) == + "1") {} rte_mempool *GetPacketMemPool() const { return ppool_.get()->GetMemPool(); } From 4877a1a8dedef1388044472a6ad54b5a33394470 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 28 May 2024 15:50:25 +0000 Subject: [PATCH 09/10] Fix issue #34: Resolving concurrency bug due to FastRand object which is not thread safe. * FastRand is now a variable private to each engine instance. --- src/include/pmd.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index 12f50078..ec62dbc4 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -86,6 +86,7 @@ class PmdRing { public: const bool machnet_testing_; + juggler::utils::FastRand fast_rand_; }; /* @@ -117,13 +118,12 @@ class TxRing : public PmdRing { /// @brief Drops a random packet from the given array of packets. /// @param pkts Array of packet pointers. /// @param nb_pkts Number of packets in the array. - inline void pkt_idx_to_drop(Packet **pkts, uint16_t *nb_pkts) const { - static juggler::utils::FastRand fast_rand; + inline void pkt_idx_to_drop(Packet **pkts, uint16_t *nb_pkts) { static constexpr uint32_t kBillion = 1000000000; // the value below is picked randomly as probability. static constexpr float kDropProb = 0.000001; - if (fast_rand.next_u32() % kBillion < kDropProb * kBillion) { - int random_packet = fast_rand.next_u32() % *nb_pkts; + if (fast_rand_.next_u32() % kBillion < kDropProb * kBillion) { + int random_packet = fast_rand_.next_u32() % *nb_pkts; LOG(INFO) << "Dropping random packet: " << random_packet << " from " << *nb_pkts << " packets."; Packet::Free(pkts[random_packet]); @@ -169,7 +169,7 @@ class TxRing : public PmdRing { * @param pkts Array of packet pointers to send. * @param nb_pkts Number of packets to send. */ - void SendPackets(Packet **pkts, uint16_t nb_pkts) const { + void SendPackets(Packet **pkts, uint16_t nb_pkts) { if (machnet_testing_) { pkt_idx_to_drop(pkts, &nb_pkts); } @@ -191,7 +191,7 @@ class TxRing : public PmdRing { * * @param batch Pointer to the PacketBatch. */ - void SendPackets(PacketBatch *batch) const { + void SendPackets(PacketBatch *batch) { SendPackets(batch->pkts(), batch->GetSize()); batch->Clear(); } From 57650f50713e33045d4f03ea4d08a2ab30fa40cf Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 28 May 2024 15:52:25 +0000 Subject: [PATCH 10/10] Fix issue #34: Remove extra include --- src/include/pmd.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/pmd.h b/src/include/pmd.h index ec62dbc4..0b8dc9b4 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -5,7 +5,6 @@ #include #include -#include #include #include #include