From 9e6197b3d39a244ed7900bfc2df9d0f2290fcfb2 Mon Sep 17 00:00:00 2001 From: yangchen Date: Mon, 17 Jun 2024 10:41:54 +0800 Subject: [PATCH] redis perf: add tx driver cache --- src/lstack/api/dir.mk | 2 +- src/lstack/api/lstack_tx_cache.c | 47 ++++++++++++++++++++++ src/lstack/core/lstack_cfg.c | 13 ++++++ src/lstack/core/lstack_protocol_stack.c | 1 + src/lstack/include/lstack_cfg.h | 1 + src/lstack/include/lstack_protocol_stack.h | 2 + src/lstack/include/lstack_tx_cache.h | 28 +++++++++++++ src/lstack/netif/lstack_ethdev.c | 30 +++++++++++--- 8 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 src/lstack/api/lstack_tx_cache.c create mode 100644 src/lstack/include/lstack_tx_cache.h diff --git a/src/lstack/api/dir.mk b/src/lstack/api/dir.mk index 729690d..70bc59d 100644 --- a/src/lstack/api/dir.mk +++ b/src/lstack/api/dir.mk @@ -8,7 +8,7 @@ # PURPOSE. # See the Mulan PSL v2 for more details. -SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c lstack_dummy_api.c +SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c lstack_dummy_api.c lstack_tx_cache.c $(eval $(call register_dir, api, $(SRC))) diff --git a/src/lstack/api/lstack_tx_cache.c b/src/lstack/api/lstack_tx_cache.c new file mode 100644 index 0000000..42aef57 --- /dev/null +++ b/src/lstack/api/lstack_tx_cache.c @@ -0,0 +1,47 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +* gazelle is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +#include "lwip/sockets.h" +#include "lstack_cfg.h" +#include "lstack_protocol_stack.h" +#include "lstack_tx_cache.h" + +void stack_send_pkts(struct protocol_stack *stack) +{ + if (!get_global_cfg_params()->send_cache_mode) { + return; + } + + uint32_t send_num = stack->tx_cache.send_end - stack->tx_cache.send_start; + + if (send_num == 0) { + return; + } + + uint32_t start = stack->tx_cache.send_start & STACK_SEND_MASK; + uint32_t end = stack->tx_cache.send_end & STACK_SEND_MASK; + uint32_t sent_pkts = 0; + + if (start < end) { + sent_pkts = stack->dev_ops.tx_xmit(stack, &stack->tx_cache.send_pkts[start], send_num); + } else { + send_num = STACK_SEND_MAX - start; + sent_pkts = stack->dev_ops.tx_xmit(stack, &stack->tx_cache.send_pkts[start], send_num); + if (sent_pkts == send_num) { + sent_pkts += stack->dev_ops.tx_xmit(stack, stack->tx_cache.send_pkts, end); + } + } + + stack->tx_cache.send_start += sent_pkts; + stack->stats.tx += sent_pkts; +} + diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c index 0e5fbf3..56f290e 100644 --- a/src/lstack/core/lstack_cfg.c +++ b/src/lstack/core/lstack_cfg.c @@ -85,6 +85,7 @@ static int32_t parse_stack_thread_mode(void); static int32_t parse_nic_vlan_mode(void); static int32_t parse_defaule_nonblock_mode(void); static int32_t parse_rpc_msg_max(void); +static int32_t parse_send_cache_mode(void); #define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \ do { \ @@ -150,6 +151,7 @@ static struct config_vector_t g_config_tbl[] = { { "nic_vlan_mode", parse_nic_vlan_mode }, { "nonblock_mode", parse_defaule_nonblock_mode }, { "rpc_msg_max", parse_rpc_msg_max }, + { "send_cache_mode", parse_send_cache_mode }, { NULL, NULL } }; @@ -1380,3 +1382,14 @@ static int32_t parse_rpc_msg_max(void) return ret; } +static int32_t parse_send_cache_mode(void) +{ + int32_t ret; + PARSE_ARG(g_config_params.send_cache_mode, "send_cache_mode", 0, 0, 1, ret); + if (ret != 0) { + LSTACK_PRE_LOG(LSTACK_ERR, "cfg: invalid send cache mode value %d. only support 0 or 1\n", + g_config_params.send_cache_mode); + } + return ret; +} + diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index c5a265e..98d6549 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -480,6 +480,7 @@ int stack_polling(uint32_t wakeup_tick) do_lwip_read_recvlist(stack, read_connect_number); if ((wakeup_tick & 0xf) == 0) { wakeup_stack_epoll(stack); + stack_send_pkts(stack); } /* run to completion mode currently does not support sockmap */ diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h index 94878de..b9721ff 100644 --- a/src/lstack/include/lstack_cfg.h +++ b/src/lstack/include/lstack_cfg.h @@ -121,6 +121,7 @@ struct cfg_params { bool stack_mode_rtc; bool nonblock_mode; uint32_t rpc_msg_max; + bool send_cache_mode; }; struct cfg_params *get_global_cfg_params(void); diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h index c210ab9..ab27dfa 100644 --- a/src/lstack/include/lstack_protocol_stack.h +++ b/src/lstack/include/lstack_protocol_stack.h @@ -23,6 +23,7 @@ #include "gazelle_dfx_msg.h" #include "lstack_thread_rpc.h" #include "lstack_ethdev.h" +#include "lstack_tx_cache.h" #include "gazelle_opt.h" #define SOCK_RECV_RING_SIZE (get_global_cfg_params()->recv_ring_size) @@ -76,6 +77,7 @@ struct protocol_stack { uint32_t tx_ring_used; struct rte_mbuf *pkts[NIC_QUEUE_SIZE_MAX]; + struct lstack_tx_cache tx_cache; struct list_node recv_list; struct list_node same_node_recv_list; /* used for same node processes communication */ struct list_node wakeup_list; diff --git a/src/lstack/include/lstack_tx_cache.h b/src/lstack/include/lstack_tx_cache.h new file mode 100644 index 0000000..3991b16 --- /dev/null +++ b/src/lstack/include/lstack_tx_cache.h @@ -0,0 +1,28 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +* gazelle is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +#ifndef _LSTACK_TX_CACHE_H_ +#define _LSTACK_TX_CACHE_H_ + +#define STACK_SEND_MAX (2048) +#define STACK_SEND_MASK (STACK_SEND_MAX - 1) +#define STACK_SEND_INDEX(index) ((index) & STACK_SEND_MASK) + +struct lstack_tx_cache { + uint32_t send_start; + uint32_t send_end; + struct rte_mbuf *send_pkts[STACK_SEND_MAX]; +}; + +void stack_send_pkts(struct protocol_stack *stack); + +#endif /* _LSTACK_TX_CACHE_H_ */ diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index 048ea92..77172f8 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -34,6 +34,7 @@ #include "lstack_protocol_stack.h" #include "lstack_thread_rpc.h" #include "lstack_flow.h" +#include "lstack_tx_cache.h" #include "lstack_ethdev.h" /* FRAME_MTU + 14byte header */ @@ -186,6 +187,19 @@ int32_t eth_dev_poll(void) return nr_pkts; } +static void eth_dev_send_pkt(struct protocol_stack *stack, struct rte_mbuf *mbuf) +{ + do { + if (STACK_SEND_INDEX(stack->tx_cache.send_end + 1) != STACK_SEND_INDEX(stack->tx_cache.send_start)) { + stack->tx_cache.send_pkts[STACK_SEND_INDEX(stack->tx_cache.send_end)] = mbuf; + stack->tx_cache.send_end++; + return; + } + stack_send_pkts(stack); + stack->stats.send_pkts_fail++; + } while (1); +} + static err_t eth_dev_output(struct netif *netif, struct pbuf *pbuf) { struct protocol_stack *stack = get_protocol_stack(); @@ -231,12 +245,16 @@ static err_t eth_dev_output(struct netif *netif, struct pbuf *pbuf) pbuf = pbuf->next; } - uint32_t sent_pkts = stack->dev_ops.tx_xmit(stack, &first_mbuf, 1); - stack->stats.tx += sent_pkts; - if (sent_pkts < 1) { - stack->stats.tx_drop++; - rte_pktmbuf_free(first_mbuf); - return ERR_MEM; + if (!get_global_cfg_params()->send_cache_mode) { + uint32_t sent_pkts = stack->dev_ops.tx_xmit(stack, &first_mbuf, 1); + stack->stats.tx += sent_pkts; + if (sent_pkts < 1) { + stack->stats.tx_drop++; + rte_pktmbuf_free(first_mbuf); + return ERR_MEM; + } + } else { + eth_dev_send_pkt(stack, first_mbuf); } return ERR_OK; -- 2.33.0