From 2c3000bba21c4c636860498530b57e842f56480f Mon Sep 17 00:00:00 2001 From: Lemmy Huang Date: Thu, 16 Mar 2023 11:40:29 +0800 Subject: [PATCH] refactor mbuf private data Signed-off-by: Lemmy Huang --- src/common/dpdk_common.h | 49 ++++++++++++++++++----------- src/lstack/core/lstack_dpdk.c | 2 +- src/lstack/core/lstack_stack_stat.c | 12 ++++--- src/ltran/CMakeLists.txt | 3 +- src/ltran/ltran_ethdev.c | 4 +-- src/ltran/ltran_forward.c | 14 ++++----- 6 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/common/dpdk_common.h b/src/common/dpdk_common.h index 1305819..6b107ae 100644 --- a/src/common/dpdk_common.h +++ b/src/common/dpdk_common.h @@ -16,32 +16,44 @@ #include #include #include +#include #include "gazelle_opt.h" #define GAZELLE_KNI_NAME "kni" // will be removed during dpdk update + /* Layout: - * | rte_mbuf | pbuf_custom| tcp_seg | gazelle_prive | payload | - * | 128 | 64 | 32 | 16 | - * rte_prefetch0 in lwip project,tcp_out.c,tcp_output_segment use constants - * cacheline is 64, make sure pbuf_custom in same cacheline + * | rte_mbuf | mbuf_private | payload | + * | 128 | | | **/ -struct pbuf; -#define LATENCY_TIMESTAMP_SIZE (sizeof(uint64_t) * 2) -#define MBUF_PRIVATE_SIZE 128 -#define LATENCY_OFFSET 96 -static __rte_always_inline uint64_t *mbuf_to_private(struct rte_mbuf *mbuf) +struct latency_timestamp { + uint64_t stamp; // time stamp + uint64_t check; // just for later vaild check +}; +struct mbuf_private { + /* struct pbuf_custom must at first */ + struct pbuf_custom pc; + /* don't use `struct tcp_seg` directly to avoid conflicts by include lwip tcp header */ + char ts[32]; // 32 > sizeof(struct tcp_seg) + struct latency_timestamp lt; +}; + +static __rte_always_inline struct mbuf_private *mbuf_to_private(const struct rte_mbuf *m) +{ + return (struct mbuf_private *)RTE_PTR_ADD(m, sizeof(struct rte_mbuf)); +} +static __rte_always_inline struct pbuf_custom *mbuf_to_pbuf(const struct rte_mbuf *m) { - return (uint64_t *)((uint8_t *)(mbuf) + sizeof(struct rte_mbuf) + LATENCY_OFFSET); + return &mbuf_to_private(m)->pc; } -static __rte_always_inline struct rte_mbuf *pbuf_to_mbuf(struct pbuf *p) +static __rte_always_inline struct rte_mbuf *pbuf_to_mbuf(const struct pbuf *p) { - return ((struct rte_mbuf *)(void *)((uint8_t *)(p) - sizeof(struct rte_mbuf))); + return (struct rte_mbuf *)RTE_PTR_SUB(p, sizeof(struct rte_mbuf)); } -static __rte_always_inline struct pbuf_custom *mbuf_to_pbuf(struct rte_mbuf *m) +static __rte_always_inline struct mbuf_private *pbuf_to_private(const struct pbuf *p) { - return ((struct pbuf_custom *)((uint8_t *)(m) + sizeof(struct rte_mbuf))); + return mbuf_to_private(pbuf_to_mbuf(p)); } /* NOTE!!! magic code, even the order. @@ -69,15 +81,16 @@ static __rte_always_inline void copy_mbuf(struct rte_mbuf *dst, struct rte_mbuf // copy private date. dst_data = (uint8_t *)mbuf_to_private(dst); src_data = (uint8_t *)mbuf_to_private(src); - rte_memcpy(dst_data, src_data, LATENCY_TIMESTAMP_SIZE); + rte_memcpy(dst_data, src_data, sizeof(struct mbuf_private)); } static __rte_always_inline void time_stamp_into_mbuf(uint32_t rx_count, struct rte_mbuf *buf[], uint64_t time_stamp) { + struct latency_timestamp *lt; for (uint32_t i = 0; i < rx_count; i++) { - uint64_t *priv = mbuf_to_private(buf[i]); - *priv = time_stamp; // time stamp - *(priv + 1) = ~(*priv); // just for later vaid check + lt = &mbuf_to_private(buf[i])->lt; + lt->stamp = time_stamp; + lt->check = ~(time_stamp); } } diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c index 2ecfd1d..7ded9e9 100644 --- a/src/lstack/core/lstack_dpdk.c +++ b/src/lstack/core/lstack_dpdk.c @@ -146,7 +146,7 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, } /* time stamp before pbuf_custom as priv_data */ - uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE); + uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE); pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, rte_socket_id()); if (pool == NULL) { LSTACK_LOG(ERR, LSTACK, "cannot create %s pool rte_err=%d\n", pool_name, rte_errno); diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c index 489b267..eed7fbf 100644 --- a/src/lstack/core/lstack_stack_stat.c +++ b/src/lstack/core/lstack_stack_stat.c @@ -51,16 +51,18 @@ uint64_t get_current_time(void) void calculate_lstack_latency(struct gazelle_stack_latency *stack_latency, const struct pbuf *pbuf, enum GAZELLE_LATENCY_TYPE type) { + uint64_t latency; + const struct latency_timestamp *lt; + if (pbuf == NULL) { return; } - const uint64_t *priv = (uint64_t *)((uint8_t *)(pbuf) + LATENCY_OFFSET); - if (*priv != ~(*(priv + 1)) || *priv < stack_latency->start_time) { + + lt = &pbuf_to_private(pbuf)->lt; + if (lt->stamp != ~(lt->check) || lt->stamp < stack_latency->start_time) { return; } - - uint64_t latency = get_current_time(); - latency = latency - *priv; + latency = get_current_time() - lt->stamp; struct stack_latency *latency_stat = (type == GAZELLE_LATENCY_LWIP) ? &stack_latency->lwip_latency : &stack_latency->read_latency; diff --git a/src/ltran/CMakeLists.txt b/src/ltran/CMakeLists.txt index 1c82dae..f37a232 100644 --- a/src/ltran/CMakeLists.txt +++ b/src/ltran/CMakeLists.txt @@ -12,6 +12,7 @@ cmake_minimum_required(VERSION 3.12.1) project(ltran) set(COMMON_DIR ${PROJECT_SOURCE_DIR}/../common) +set(LWIP_DIR /usr/include/lwip) set(CMAKE_VERBOSE_MAKEFILE ON) if (CMAKE_C_COMPILER_ID STREQUAL "GNU") @@ -31,7 +32,7 @@ add_executable(ltran main.c ltran_param.c ltran_config.c ltran_ethdev.c ltran_st ltran_forward.c ltran_timer.c ${COMMON_DIR}/gazelle_dfx_msg.c ${COMMON_DIR}/dpdk_common.c ${COMMON_DIR}/gazelle_parse_config.c) -target_include_directories(ltran PRIVATE ${COMMON_DIR} ${PROJECT_SOURCE_DIR}) +target_include_directories(ltran PRIVATE ${COMMON_DIR} ${PROJECT_SOURCE_DIR} ${LWIP_DIR}) target_compile_options(ltran PRIVATE -march=native -fno-strict-aliasing -D__ARM_FEATURE_CRC32=1 -DRTE_MACHINE_CPUFLAG_NEON -DRTE_MACHINE_CPUFLAG_CRC32 -DRTE_MACHINE_CPUFLAG_PMULL -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_SHA1 -DRTE_MACHINE_CPUFLAG_SHA2 -include rte_config.h diff --git a/src/ltran/ltran_ethdev.c b/src/ltran/ltran_ethdev.c index e0c824a..e2eb4a8 100644 --- a/src/ltran/ltran_ethdev.c +++ b/src/ltran/ltran_ethdev.c @@ -147,7 +147,7 @@ static struct rte_mempool *ltran_create_rx_mbuf_pool(uint32_t bond_port_index) return NULL; } - uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE); + uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE); return rte_pktmbuf_pool_create(mbuf_pool_name, num_mbufs, GAZELLE_MBUFS_CACHE_SIZE, private_size, RTE_MBUF_DEFAULT_BUF_SIZE, (int32_t)rte_socket_id()); } @@ -166,7 +166,7 @@ static struct rte_mempool *ltran_create_tx_mbuf_pool(uint32_t bond_port_index) return NULL; } - uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE); + uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE); return rte_pktmbuf_pool_create(mbuf_pool_name, num_mbufs, GAZELLE_MBUFS_CACHE_SIZE, private_size, RTE_MBUF_DEFAULT_BUF_SIZE, (int32_t)rte_socket_id()); } diff --git a/src/ltran/ltran_forward.c b/src/ltran/ltran_forward.c index 4d9c1bb..8629acb 100644 --- a/src/ltran/ltran_forward.c +++ b/src/ltran/ltran_forward.c @@ -50,22 +50,22 @@ static __rte_always_inline struct gazelle_stack *get_kni_stack(void) static void calculate_ltran_latency(struct gazelle_stack *stack, const struct rte_mbuf *mbuf) { + struct latency_timestamp *lt; uint64_t latency; - uint64_t *priv = NULL; - priv = (uint64_t *)RTE_PTR_ADD(mbuf, sizeof(struct rte_mbuf) + LATENCY_OFFSET); - // priv--time stamp priv+1 --- vaild check - if (*priv != ~(*(priv + 1))) { + lt = &mbuf_to_private(mbuf)->lt; + // vaild check + if (lt->stamp != ~(lt->check)) { return; } // time stamp must > start time - if (*priv < get_start_time_stamp()) { - *priv = 0; + if (lt->stamp < get_start_time_stamp()) { + lt->stamp = 0; return; } - latency = get_current_time() - *priv; + latency = get_current_time() - lt->stamp; stack->stack_stats.latency_total += latency; stack->stack_stats.latency_pkts++; -- 2.23.0