From 55ed3c6aaccf320a7d3240753a5aabe400ac4bd3 Mon Sep 17 00:00:00 2001
From: yinbin6 <yinbin8@huawei.com>
Date: Fri, 7 Jun 2024 17:06:50 +0800
Subject: [PATCH] make rpc_msg_max recv_ring_size-configurable


diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c
index 9efdbaa..0e5fbf3 100644
--- a/src/lstack/core/lstack_cfg.c
+++ b/src/lstack/core/lstack_cfg.c
@@ -68,6 +68,7 @@ static int32_t parse_nic_read_number(void);
 static int32_t parse_tcp_conn_count(void);
 static int32_t parse_mbuf_count_per_conn(void);
 static int32_t parse_send_ring_size(void);
+static int32_t parse_recv_ring_size(void);
 static int32_t parse_num_process(void);
 static int32_t parse_process_numa(void);
 static int32_t parse_process_index(void);
@@ -83,6 +84,7 @@ static int32_t parse_nic_txqueue_size(void);
 static int32_t parse_stack_thread_mode(void);
 static int32_t parse_nic_vlan_mode(void);
 static int32_t parse_defaule_nonblock_mode(void);
+static int32_t parse_rpc_msg_max(void);
 
 #define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \
     do { \
@@ -132,6 +134,7 @@ static struct config_vector_t g_config_tbl[] = {
     { "rpc_number", parse_rpc_number },
     { "nic_read_number", parse_nic_read_number },
     { "send_ring_size", parse_send_ring_size },
+    { "recv_ring_size", parse_recv_ring_size },
     { "num_process",  parse_num_process },
     { "process_numa", parse_process_numa },
     { "process_idx", parse_process_index },
@@ -146,6 +149,7 @@ static struct config_vector_t g_config_tbl[] = {
     { "stack_thread_mode", parse_stack_thread_mode },
     { "nic_vlan_mode", parse_nic_vlan_mode },
     { "nonblock_mode", parse_defaule_nonblock_mode },
+    { "rpc_msg_max", parse_rpc_msg_max },
     { NULL,           NULL }
 };
 
@@ -908,6 +912,14 @@ static int32_t parse_send_ring_size(void)
     return ret;
 }
 
+static int32_t parse_recv_ring_size(void)
+{
+    int32_t ret;
+    /* recv ring size default value is 128 */
+    PARSE_ARG(g_config_params.recv_ring_size, "recv_ring_size", 128, 1, SOCK_RECV_RING_SIZE_MAX, ret);
+    return ret;
+}
+
 static int32_t parse_mbuf_count_per_conn(void)
 {
     int32_t ret;
@@ -1356,3 +1368,15 @@ static int32_t parse_defaule_nonblock_mode(void)
     }
     return ret;
 }
+
+static int32_t parse_rpc_msg_max(void)
+{
+    int32_t ret;
+    PARSE_ARG(g_config_params.rpc_msg_max, "rpc_msg_max", 4096, 1, 8192, ret);
+    if (ret != 0) {
+        LSTACK_PRE_LOG(LSTACK_ERR, "cfg: invalid rpc msg max value %d ret=%d. only support 1~8192\n",
+            g_config_params.rpc_msg_max, ret);
+    }
+    return ret;
+}
+
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 30bd827..b98ba84 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -13,6 +13,7 @@
 #include <rte_mempool.h>
 
 #include "lstack_log.h"
+#include "lstack_cfg.h"
 #include "lstack_dpdk.h"
 #include "lstack_rpc_proc.h"
 #include "lstack_stack_stat.h"
@@ -71,7 +72,7 @@ static struct rpc_msg *rpc_msg_alloc(rpc_msg_func func)
             exit(-1);
         }
 
-        g_rpc_pool->mempool = create_mempool("rpc_pool", RPC_MSG_MAX, sizeof(struct rpc_msg),
+        g_rpc_pool->mempool = create_mempool("rpc_pool", get_global_cfg_params()->rpc_msg_max, sizeof(struct rpc_msg),
             0, rte_gettid());
         if (g_rpc_pool->mempool == NULL) {
             LSTACK_LOG(INFO, LSTACK, "rpc_pool create failed, errno is %d\n", errno);
diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h
index a00e47a..94878de 100644
--- a/src/lstack/include/lstack_cfg.h
+++ b/src/lstack/include/lstack_cfg.h
@@ -110,6 +110,7 @@ struct cfg_params {
     struct secondary_attach_arg sec_attach_arg;
     char unix_socket_filename[NAME_MAX];
     uint16_t send_ring_size;
+    uint16_t recv_ring_size;
     bool tuple_filter;
     int8_t bond_mode;
     int32_t bond_miimon;
@@ -119,6 +120,7 @@ struct cfg_params {
     struct cfg_nic_params nic;
     bool stack_mode_rtc;
     bool nonblock_mode;
+    uint32_t rpc_msg_max;
 };
 
 struct cfg_params *get_global_cfg_params(void);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 93fa40c..c210ab9 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -25,8 +25,9 @@
 #include "lstack_ethdev.h"
 #include "gazelle_opt.h"
 
-#define SOCK_RECV_RING_SIZE         (128)
+#define SOCK_RECV_RING_SIZE         (get_global_cfg_params()->recv_ring_size)
 #define SOCK_RECV_FREE_THRES        (32)
+#define SOCK_RECV_RING_SIZE_MAX     (2048)
 #define SOCK_SEND_RING_SIZE_MAX     (2048)
 #define SOCK_SEND_REPLENISH_THRES   (16)
 #define WAKEUP_MAX_NUM              (32)
diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h
index 8e97c11..0c51848 100644
--- a/src/lstack/include/lstack_thread_rpc.h
+++ b/src/lstack/include/lstack_thread_rpc.h
@@ -26,8 +26,6 @@
 #define MSG_ARG_4                      (4)
 #define RPM_MSG_ARG_SIZE               (5)
 
-#define RPC_MSG_MAX            4096
-#define RPC_MSG_MASK           (RPC_MSG_MAX - 1)
 typedef struct lockless_queue rpc_queue;
 
 struct rpc_stats {
diff --git a/src/lstack/lstack.conf b/src/lstack/lstack.conf
index a7f4e75..4c3784f 100644
--- a/src/lstack/lstack.conf
+++ b/src/lstack/lstack.conf
@@ -22,8 +22,12 @@ tcp_conn_count = 1500
 mbuf_count_per_conn = 170
 
 # send ring size, default is 32, max is 2048
+# if udp pktlen exceeds 45952(32 * 1436)B, send_ring_size must be at least 64.
 send_ring_size = 32
 
+#recv ring size, default is 128, max is 2048
+recv_ring_size = 128
+
 #protocol stack thread per loop params
 #read data form protocol stack into recv_ring
 read_connect_number = 4
@@ -67,3 +71,6 @@ nic_vlan_mode=-1
 bond_mode=-1
 #bond slave mac, separated by ; , only support 2 slave mac
 #bond_slave_mac="aa:bb:cc:dd:ee:ff;gg:hh:ii:jj:kk:ll"
+
+#maximum number of rpc memory pools
+rpc_msg_max=4096
-- 
2.33.0