packages/gazelle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!167 [sync] PR-164: rxtx mbuf pool size config by conf

Browse Source 
From: @openeuler-sync-bot 
Reviewed-by: @wu-changsheng 
Signed-off-by: @wu-changsheng
This commit is contained in:
openeuler-ci-bot 2022-12-13 15:44:46 +00:00 committed by Gitee
commit 9c0c601473
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 429 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
0151-dfx-gazellectl-add-pcb-wins-info.patch Normal file
View File

@ -0,0 +1,222 @@
From 0a034f7bf644660c09a12af6c5e5d4cc3e1521d2 Mon Sep 17 00:00:00 2001
From: wu-changsheng <wuchangsheng2@huawei.com>
Date: Mon, 12 Dec 2022 11:37:45 +0800
Subject: [PATCH 1/4] dfx:gazellectl add pcb wins info
---
src/common/gazelle_dfx_msg.h | 12 ++++++---
src/lstack/core/lstack_lwip.c | 49 ++++++++++++++---------------------
src/ltran/ltran_dfx.c | 21 +++++++++------
3 files changed, 42 insertions(+), 40 deletions(-)
diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 0bdd238..d14f1b9 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -157,10 +157,16 @@ struct gazelle_stat_lstack_conn_info {
uint32_t send_ring_cnt;
uint32_t recv_ring_cnt;
uint32_t tcp_sub_state;
- uint32_t send_back_cnt;
+ uint32_t cwn;
+ uint32_t rcv_wnd;
+ uint32_t snd_wnd;
+ uint32_t snd_buf;
+ uint32_t lastack;
+ uint32_t snd_nxt;
int32_t fd;
- uint64_t recv_all;
- uint64_t send_all;
+ uint32_t events;
+ uint32_t epoll_events;
+ uint32_t eventlist;
};
struct gazelle_stat_lstack_conn {
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 2cda2d9..d4a2861 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -125,21 +125,22 @@ static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, u
}
/* true: need replenish again */
-static bool replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring)
+static bool replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring, uint32_t replenish_cnt)
{
void *pbuf[SOCK_SEND_RING_SIZE];
- uint32_t replenish_cnt = gazelle_ring_free_count(ring);
-
uint32_t alloc_num = LWIP_MIN(replenish_cnt, RING_SIZE(SOCK_SEND_RING_SIZE));
if (gazelle_alloc_mbuf_with_reserve(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)pbuf, alloc_num) != 0) {
stack->stats.tx_allocmbuf_fail++;
return true;
}
- for (uint32_t i = 0; i < alloc_num; i++) {
+ uint32_t i = 0;
+ for (; i < alloc_num - 1; i++) {
+ rte_prefetch0(mbuf_to_pbuf((void *)pbuf[i + 1]));
pbuf[i] = init_mbuf_to_pbuf(pbuf[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
}
+ pbuf[i] = init_mbuf_to_pbuf((struct rte_mbuf *)pbuf[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
uint32_t num = gazelle_ring_sp_enqueue(ring, pbuf, alloc_num);
for (uint32_t i = num; i < alloc_num; i++) {
@@ -173,7 +174,7 @@ void gazelle_init_sock(int32_t fd)
LSTACK_LOG(ERR, LSTACK, "sock_send create failed. errno: %d.\n", rte_errno);
return;
}
- (void)replenish_send_idlembuf(stack, sock->send_ring);
+ (void)replenish_send_idlembuf(stack, sock->send_ring, RING_SIZE(SOCK_SEND_RING_SIZE));
sock->stack = stack;
sock->stack->conn_num++;
@@ -297,7 +298,6 @@ struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8
void write_lwip_over(struct lwip_sock *sock)
{
sock->send_pre_del = NULL;
- sock->send_all++;
sock->stack->stats.write_lwip_cnt++;
}
@@ -363,10 +363,8 @@ static inline ssize_t app_direct_write(struct protocol_stack *stack, struct lwip
pbufs[i] = init_mbuf_to_pbuf((struct rte_mbuf *)pbufs[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
pbufs[i - 1]->next = pbufs[i];
}
- if (write_num > 1) {
- pbufs[i] = init_mbuf_to_pbuf((struct rte_mbuf *)pbufs[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
- pbufs[i - 1]->next = pbufs[i];
- }
+ pbufs[i] = init_mbuf_to_pbuf((struct rte_mbuf *)pbufs[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
+ pbufs[i - 1]->next = pbufs[i];
ssize_t send_len = do_app_write(pbufs, buf, len, write_num);
@@ -543,8 +541,9 @@ static inline bool replenish_send_ring(struct protocol_stack *stack, struct lwip
{
bool replenish_again = false;
- if (gazelle_ring_readable_count(sock->send_ring) < SOCK_SEND_REPLENISH_THRES) {
- replenish_again = replenish_send_idlembuf(stack, sock->send_ring);
+ uint32_t replenish_cnt = gazelle_ring_free_count(sock->send_ring);
+ if (replenish_cnt >= SOCK_SEND_REPLENISH_THRES) {
+ replenish_again = replenish_send_idlembuf(stack, sock->send_ring, replenish_cnt);
}
if ((sock->epoll_events & EPOLLOUT) && NETCONN_IS_OUTIDLE(sock)) {
@@ -701,7 +700,6 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
calculate_lstack_latency(&sock->stack->latency, pbufs[i], GAZELLE_LATENCY_LWIP);
}
- sock->recv_all += read_count;
sock->stack->stats.read_lwip_cnt += read_count;
if (recv_len == 0) {
GAZELLE_RETURN(EAGAIN);
@@ -982,19 +980,6 @@ void gazelle_connected_callback(struct netconn *conn)
add_sock_event(sock, EPOLLOUT);
}
-static uint32_t send_back_count(struct lwip_sock *sock)
-{
- uint32_t count = 0;
- struct pbuf *pbuf = sock->send_lastdata;
-
- while (pbuf) {
- count++;
- pbuf = pbuf->next;
- }
-
- return count;
-}
-
static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const struct tcp_pcb *pcb)
{
struct netconn *netconn = (struct netconn *)pcb->callback_arg;
@@ -1005,6 +990,12 @@ static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const s
conn->r_port = pcb->remote_port;
conn->in_send = pcb->snd_queuelen;
conn->tcp_sub_state = pcb->state;
+ conn->cwn = pcb->cwnd;
+ conn->rcv_wnd = pcb->rcv_wnd;
+ conn->snd_wnd = pcb->snd_wnd;
+ conn->snd_buf = pcb->snd_buf;
+ conn->lastack = pcb->lastack;
+ conn->snd_nxt = pcb->snd_nxt;
if (netconn != NULL && netconn->recvmbox != NULL) {
conn->recv_cnt = rte_ring_count(netconn->recvmbox->ring);
@@ -1014,10 +1005,10 @@ static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const s
if (netconn->socket > 0 && sock != NULL && sock->recv_ring != NULL && sock->send_ring != NULL) {
conn->recv_ring_cnt = gazelle_ring_readable_count(sock->recv_ring);
conn->recv_ring_cnt += (sock->recv_lastdata) ? 1 : 0;
- conn->send_back_cnt = send_back_count(sock);
conn->send_ring_cnt = gazelle_ring_readover_count(sock->send_ring);
- conn->recv_all = sock->recv_all;
- conn->send_all = sock->send_all;
+ conn->events = sock->events;
+ conn->epoll_events = sock->epoll_events;
+ conn->eventlist = !list_is_null(&sock->event_list);
}
}
}
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 54839af..3801f19 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -12,6 +12,7 @@
#include <stdio.h>
#include <stdint.h>
+#include <sys/time.h>
#include <arpa/inet.h>
#include <getopt.h>
@@ -871,12 +872,14 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
char str_rip[GAZELLE_SUBNET_LENGTH_MAX] = {0};
struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf;
struct gazelle_stat_lstack_conn *conn = &stat->data.conn;
+ struct timeval time = {0};
+ gettimeofday(&time, NULL);
printf("Active Internet connections (servers and established)\n");
do {
- printf("\n------ stack tid: %6u ------\n", stat->tid);
- printf("No. Proto lwip_recv recv_ring recv_all in_send send_ring send_back_cnt send_all "
- "fd Local Address Foreign Address State\n");
+ printf("\n------ stack tid: %6u ------time=%lu\n", stat->tid, time.tv_sec * 1000000 + time.tv_usec);
+ printf("No. Proto lwip_recv recv_ring in_send send_ring cwn rcv_wnd snd_wnd snd_buf snd_nxt"
+ " lastack events epoll_ev evlist fd Local Address Foreign Address State\n");
uint32_t unread_pkts = 0;
uint32_t unsend_pkts = 0;
for (i = 0; i < conn->conn_num && i < GAZELLE_LSTACK_MAX_CONN; i++) {
@@ -885,10 +888,12 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
rip.s_addr = conn_info->rip;
lip.s_addr = conn_info->lip;
if ((conn_info->state == GAZELLE_ACTIVE_LIST) || (conn_info->state == GAZELLE_TIME_WAIT_LIST)) {
- printf("%-6utcp %-11u%-11u%-10lu%-9u%-11u%-15d%-10lu%-7d%s:%hu %s:%hu %s\n", i,
- conn_info->recv_cnt, conn_info->recv_ring_cnt, conn_info->recv_all, conn_info->in_send,
- conn_info->send_ring_cnt, conn_info->send_back_cnt, conn_info->send_all,
- conn_info->fd, inet_ntop(AF_INET, &lip, str_ip, sizeof(str_ip)), conn_info->l_port,
+ printf("%-6utcp %-10u%-10u%-8u%-10u%-9d%-9d%-10d%-10d%-15u%-15u%-10x%-10x%-7d%-7d"
+ "%s:%hu %s:%hu %s\n", i, conn_info->recv_cnt, conn_info->recv_ring_cnt, conn_info->in_send,
+ conn_info->send_ring_cnt, conn_info->cwn, conn_info->rcv_wnd, conn_info->snd_wnd,
+ conn_info->snd_buf, conn_info->snd_nxt, conn_info->lastack, conn_info->events,
+ conn_info->epoll_events, conn_info->eventlist, conn_info->fd,
+ inet_ntop(AF_INET, &lip, str_ip, sizeof(str_ip)), conn_info->l_port,
inet_ntop(AF_INET, &rip, str_rip, sizeof(str_rip)), conn_info->r_port,
tcp_state_to_str(conn_info->tcp_sub_state));
} else if (conn_info->state == GAZELLE_LISTEN_LIST) {
@@ -899,7 +904,7 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
inet_ntop(AF_INET, &lip, str_ip, sizeof(str_ip)), conn_info->l_port, conn_info->state);
}
unread_pkts += conn_info->recv_ring_cnt + conn_info->recv_cnt;
- unsend_pkts += conn_info->send_ring_cnt + conn_info->in_send + conn_info->send_back_cnt;
+ unsend_pkts += conn_info->send_ring_cnt + conn_info->in_send;
}
if (conn->conn_num > 0) {
printf("Total unread pkts:%u unsend pkts:%u\n", unread_pkts, unsend_pkts);
--
2.23.0

73
0152-fix-genarate-out-event-untimely.patch Normal file
View File

@ -0,0 +1,73 @@
From 5b3acd311fe20e9055b5faeced028e3099773e5b Mon Sep 17 00:00:00 2001
From: wu-changsheng <wuchangsheng2@huawei.com>
Date: Mon, 12 Dec 2022 11:48:31 +0800
Subject: [PATCH 2/4] fix genarate out event untimely
---
src/lstack/core/lstack_lwip.c | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index d4a2861..0aaae30 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -125,25 +125,29 @@ static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, u
}
/* true: need replenish again */
-static bool replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring, uint32_t replenish_cnt)
+static bool replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring)
{
void *pbuf[SOCK_SEND_RING_SIZE];
- uint32_t alloc_num = LWIP_MIN(replenish_cnt, RING_SIZE(SOCK_SEND_RING_SIZE));
- if (gazelle_alloc_mbuf_with_reserve(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)pbuf, alloc_num) != 0) {
+ uint32_t replenish_cnt = gazelle_ring_free_count(ring);
+ if (replenish_cnt == 0) {
+ return false;
+ }
+
+ if (gazelle_alloc_mbuf_with_reserve(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)pbuf, replenish_cnt) != 0) {
stack->stats.tx_allocmbuf_fail++;
return true;
}
uint32_t i = 0;
- for (; i < alloc_num - 1; i++) {
+ for (; i < replenish_cnt - 1; i++) {
rte_prefetch0(mbuf_to_pbuf((void *)pbuf[i + 1]));
pbuf[i] = init_mbuf_to_pbuf(pbuf[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
}
pbuf[i] = init_mbuf_to_pbuf((struct rte_mbuf *)pbuf[i], PBUF_TRANSPORT, MBUF_MAX_DATA_LEN, PBUF_RAM);
- uint32_t num = gazelle_ring_sp_enqueue(ring, pbuf, alloc_num);
- for (uint32_t i = num; i < alloc_num; i++) {
+ uint32_t num = gazelle_ring_sp_enqueue(ring, pbuf, replenish_cnt);
+ for (uint32_t i = num; i < replenish_cnt; i++) {
pbuf_free(pbuf[i]);
}
@@ -174,7 +178,7 @@ void gazelle_init_sock(int32_t fd)
LSTACK_LOG(ERR, LSTACK, "sock_send create failed. errno: %d.\n", rte_errno);
return;
}
- (void)replenish_send_idlembuf(stack, sock->send_ring, RING_SIZE(SOCK_SEND_RING_SIZE));
+ (void)replenish_send_idlembuf(stack, sock->send_ring);
sock->stack = stack;
sock->stack->conn_num++;
@@ -541,10 +545,7 @@ static inline bool replenish_send_ring(struct protocol_stack *stack, struct lwip
{
bool replenish_again = false;
- uint32_t replenish_cnt = gazelle_ring_free_count(sock->send_ring);
- if (replenish_cnt >= SOCK_SEND_REPLENISH_THRES) {
- replenish_again = replenish_send_idlembuf(stack, sock->send_ring, replenish_cnt);
- }
+ replenish_again = replenish_send_idlembuf(stack, sock->send_ring);
if ((sock->epoll_events & EPOLLOUT) && NETCONN_IS_OUTIDLE(sock)) {
add_sock_event(sock, EPOLLOUT);
--
2.23.0

126
0153-rxtx-mbuf-pool-size-config-by-conf.patch Normal file
View File

@ -0,0 +1,126 @@
From 00252d8017d1f58787052b74ec8cd434ea3cacc1 Mon Sep 17 00:00:00 2001
From: wu-changsheng <wuchangsheng2@huawei.com>
Date: Tue, 13 Dec 2022 22:37:12 +0800
Subject: [PATCH 4/4] rxtx mbuf pool size config by conf
---
src/common/gazelle_opt.h | 2 ++
src/lstack/core/lstack_cfg.c | 18 ++++++++++++++++++
src/lstack/core/lstack_dpdk.c | 4 ++--
src/lstack/include/lstack_cfg.h | 1 +
src/lstack/include/lstack_dpdk.h | 1 -
src/lstack/lstack.conf | 3 +++
6 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h
index aac1ec9..a23b329 100644
--- a/src/common/gazelle_opt.h
+++ b/src/common/gazelle_opt.h
@@ -47,6 +47,8 @@
#define RTE_TEST_TX_DESC_DEFAULT 2048
#define RTE_TEST_RX_DESC_DEFAULT 4096
+#define RXTX_NB_MBUF_DEFAULT (128 * 2000) /* mbuf per connect * connect num. size of mbuf is 2536 Byte */
+
#define MBUF_MAX_DATA_LEN 1460
#define DPDK_PKT_BURST_SIZE 512
diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c
index 52ff3f5..5a6eb2a 100644
--- a/src/lstack/core/lstack_cfg.c
+++ b/src/lstack/core/lstack_cfg.c
@@ -57,6 +57,7 @@ static int32_t parse_kni_switch(void);
static int32_t parse_listen_shadow(void);
static int32_t parse_app_bind_numa(void);
static int32_t parse_unix_prefix(void);
+static int32_t parse_rxtx_pool_size(void);
struct config_vector_t {
const char *name;
@@ -77,6 +78,7 @@ static struct config_vector_t g_config_tbl[] = {
{ "listen_shadow", parse_listen_shadow },
{ "app_bind_numa", parse_app_bind_numa },
{ "unix_prefix", parse_unix_prefix },
+ { "mbuf_pool_size", parse_rxtx_pool_size },
{ NULL, NULL }
};
@@ -698,6 +700,22 @@ static int32_t parse_use_ltran(void)
return 0;
}
+static int32_t parse_rxtx_pool_size(void)
+{
+ const config_setting_t *arg = NULL;
+
+ arg = config_lookup(&g_config, "mbuf_pool_size");
+ if (arg == NULL) {
+ g_config_params.mbuf_pool_size = RXTX_NB_MBUF_DEFAULT;
+ return 0;
+ }
+
+ int32_t val = config_setting_get_int(arg);
+ g_config_params.mbuf_pool_size = val;
+
+ return 0;
+}
+
static int32_t parse_listen_shadow(void)
{
const config_setting_t *arg = NULL;
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index b9f2793..4be31c3 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -178,8 +178,8 @@ int32_t pktmbuf_pool_init(struct protocol_stack *stack, uint16_t stack_num)
return -1;
}
- stack->rxtx_pktmbuf_pool = create_pktmbuf_mempool("rxtx_mbuf", RXTX_NB_MBUF / stack_num, RXTX_CACHE_SZ,
- stack->queue_id);
+ stack->rxtx_pktmbuf_pool = create_pktmbuf_mempool("rxtx_mbuf",
+ get_global_cfg_params()->mbuf_pool_size / stack_num, RXTX_CACHE_SZ, stack->queue_id);
if (stack->rxtx_pktmbuf_pool == NULL) {
return -1;
}
diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h
index 3864250..053208b 100644
--- a/src/lstack/include/lstack_cfg.h
+++ b/src/lstack/include/lstack_cfg.h
@@ -73,6 +73,7 @@ struct cfg_params {
uint16_t lpm_rx_pkts;
uint32_t lpm_detect_ms;
uint32_t lpm_pkts_in_detect;
+ uint32_t mbuf_pool_size;
bool use_ltran; // ture:lstack read from nic false:read form ltran
bool kni_switch;
bool listen_shadow; // true:listen in all stack thread. false:listen in one stack thread.
diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h
index 54b8fc2..a0f09af 100644
--- a/src/lstack/include/lstack_dpdk.h
+++ b/src/lstack/include/lstack_dpdk.h
@@ -15,7 +15,6 @@
#include "gazelle_opt.h"
-#define RXTX_NB_MBUF (512 * 2000) /* mbuf per connect * connect num. size of mbuf is 2536 Byte */
#define RXTX_CACHE_SZ (VDEV_RX_QUEUE_SZ)
#define KNI_NB_MBUF (DEFAULT_RING_SIZE << 4)
diff --git a/src/lstack/lstack.conf b/src/lstack/lstack.conf
index 1a91aee..22d519e 100644
--- a/src/lstack/lstack.conf
+++ b/src/lstack/lstack.conf
@@ -15,6 +15,9 @@ kni_switch=0
low_power_mode=0
listen_shadow=0
+
+#number of mbuf for tx and rx. per mbuf is 2560 Byte.
+#mbuf_pool_size = 256000
num_cpus="2"
num_wakeup="3"
--
2.23.0

9
gazelle.spec
View File

@ -2,7 +2,7 @@
Name: gazelle
Version: 1.0.1
Release: 28
Release: 29
Summary: gazelle is a high performance user-mode stack
License: MulanPSL-2.0
URL: https://gitee.com/openeuler/gazelle
@ -165,6 +165,9 @@ Patch9147: 0147-add-TSO.patch
Patch9148: 0148-optimize-app-thread-write-buff-block.patch
Patch9149: 0149-expand-rxtx-mbuf-pool.patch
Patch9150: 0150-add-pdump-support-in-ltran.patch
Patch9151: 0151-dfx-gazellectl-add-pcb-wins-info.patch
Patch9152: 0152-fix-genarate-out-event-untimely.patch
Patch9153: 0153-rxtx-mbuf-pool-size-config-by-conf.patch
%description
%{name} is a high performance user-mode stack.
@ -205,6 +208,10 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b
%config(noreplace) %{conf_path}/ltran.conf
%changelog
* Tue Dec 13 2022 wuchangsheng <wuchangsheng2@huawei.com> - 1.0.1-29
- dfx add pcb windows info
rxtx mbuf pool size config by conf
* Mon Dec 12 2022 kircher <majun65@huawei.com> - 1.0.1-28
- add pdump support in ltran