packages/gazelle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
gazelle/0038-solve-problem-that-rte_pktmbuf_poll_creat-in-same-nu.patch
hantwofish af32524b45 sync solve problem that rte_pktmbuf_poll_creat in same numa . 
(cherry picked from commit f9784aa3960e2a2fbb560a9d8e9d896b2848f8c4)
2023-11-18 15:23:40 +08:00

160 lines
6.5 KiB
Diff
Raw Blame History

From 2a5891557162fb5743f13cb550dc7ff109476176 Mon Sep 17 00:00:00 2001
From: hantwofish <hankangkang5@huawei.com>
Date: Wed, 1 Nov 2023 18:34:07 +0800
Subject: [PATCH] solve problem that rte_pktmbuf_poll_creat in same numa .
---
src/lstack/core/lstack_dpdk.c | 11 +++---
src/lstack/core/lstack_protocol_stack.c | 52 ++++++++++++++++++++-----
src/lstack/include/lstack_dpdk.h | 2 +-
3 files changed, 50 insertions(+), 15 deletions(-)
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 48fa67d..88a6c45 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -110,7 +110,7 @@ int32_t dpdk_eal_init(void)
{
int32_t ret;
struct cfg_params *global_params = get_global_cfg_params();
-
+
ret = rte_eal_init(global_params->dpdk_argc, global_params->dpdk_argv);
if (ret < 0) {
if (rte_errno == EALREADY) {
@@ -137,7 +137,7 @@ int32_t dpdk_eal_init(void)
}
struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
- uint32_t mbuf_cache_size, uint16_t queue_id)
+ uint32_t mbuf_cache_size, uint16_t queue_id, unsigned numa_id)
{
int32_t ret;
char pool_name[PATH_MAX];
@@ -145,12 +145,13 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
ret = snprintf_s(pool_name, sizeof(pool_name), PATH_MAX - 1, "%s_%hu", name, queue_id);
if (ret < 0) {
+ LSTACK_LOG(ERR, LSTACK, "snprintf_s fail ret=%d \n", ret);
return NULL;
}
/* time stamp before pbuf_custom as priv_data */
uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE);
- pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, rte_socket_id());
+ pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, numa_id);
if (pool == NULL) {
LSTACK_LOG(ERR, LSTACK, "cannot create %s pool rte_err=%d\n", pool_name, rte_errno);
}
@@ -611,7 +612,7 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i
int32_t ret;
struct rte_mempool *rxtx_pktmbuf_pool = get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx];
-
+
uint16_t socket_id = 0;
struct cfg_params *cfg = get_global_cfg_params();
if (!cfg->use_ltran && cfg->num_process == 1) {
@@ -664,7 +665,7 @@ int32_t dpdk_ethdev_start(void)
int32_t dpdk_init_lstack_kni(void)
{
struct protocol_stack_group *stack_group = get_protocol_stack_group();
- stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0);
+ stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0, rte_socket_id());
if (stack_group->kni_pktmbuf_pool == NULL) {
return -1;
}
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 3662948..2fd8d0a 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -523,6 +523,46 @@ static void gazelle_listen_thread(void *arg)
recv_pkts_from_other_process(cfg_param->process_idx, arg);
}
+int32_t stack_group_init_mempool(void)
+{
+ struct cfg_params *global_cfg_parmas = get_global_cfg_params();
+ struct protocol_stack_group *stack_group = get_protocol_stack_group();
+ struct rte_mempool *rxtx_mbuf = NULL;
+ uint32_t cpu_id = 0;
+ unsigned numa_id = 0;
+ int queue_id = 0;
+
+ LSTACK_LOG(INFO, LSTACK,
+ "config::num_cpu=%d num_process=%d \n", global_cfg_parmas->num_cpu, global_cfg_parmas->num_process);
+
+ uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count;
+
+ for (int cpu_idx = 0; cpu_idx < global_cfg_parmas->num_cpu; cpu_idx++) {
+ cpu_id = global_cfg_parmas->cpus[cpu_idx];
+ numa_id = numa_node_of_cpu(cpu_id);
+
+ for (int process_idx = 0; process_idx < global_cfg_parmas->num_process; process_idx++) {
+ queue_id = cpu_idx * global_cfg_parmas->num_process + process_idx;
+ if (queue_id >= PROTOCOL_STACK_MAX) {
+ LSTACK_LOG(ERR, LSTACK, "index is over\n");
+ return -1;
+ }
+
+ rxtx_mbuf = create_pktmbuf_mempool(
+ "rxtx_mbuf", total_mbufs / stack_group->stack_num, RXTX_CACHE_SZ, queue_id, numa_id);
+ if (rxtx_mbuf == NULL) {
+ LSTACK_LOG(ERR, LSTACK, "cpuid=%u, numid=%d , rxtx_mbuf idx= %d create_pktmbuf_mempool fail\n",
+ cpu_id, numa_id, queue_id);
+ return -1;
+ }
+
+ get_protocol_stack_group()->total_rxtx_pktmbuf_pool[queue_id] = rxtx_mbuf;
+ }
+ }
+
+ return 0;
+}
+
int32_t stack_group_init(void)
{
struct protocol_stack_group *stack_group = get_protocol_stack_group();
@@ -542,14 +582,8 @@ int32_t stack_group_init(void)
stack_group->stack_setup_fail = 0;
if (get_global_cfg_params()->is_primary) {
- uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count;
- for (uint16_t idx = 0; idx < get_global_cfg_params()->tot_queue_num; idx++) {
- struct rte_mempool* rxtx_mbuf = create_pktmbuf_mempool("rxtx_mbuf",
- total_mbufs / stack_group->stack_num, RXTX_CACHE_SZ, idx);
- if (rxtx_mbuf == NULL) {
- return -1;
- }
- get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx] = rxtx_mbuf;
+ if (stack_group_init_mempool() != 0) {
+ return -1;
}
}
@@ -970,7 +1004,7 @@ void stack_recvlist_count(struct rpc_msg *msg)
list_for_each_safe(node, temp, list) {
count++;
}
-
+
msg->result = count;
}
diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h
index 6ca4f3b..1a054d6 100644
--- a/src/lstack/include/lstack_dpdk.h
+++ b/src/lstack/include/lstack_dpdk.h
@@ -56,7 +56,7 @@ void dpdk_restore_pci(void);
bool port_in_stack_queue(uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port);
uint16_t get_port_id(void);
struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
- uint32_t mbuf_cache_size, uint16_t queue_id);
+ uint32_t mbuf_cache_size, uint16_t queue_id, unsigned numa_id);
void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id);
int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num);
--
2.23.0
Reference in New Issue View Git Blame Copy Permalink