From 1ce8e3b67d32803bd912e747571e4d4618e9c3c2 Mon Sep 17 00:00:00 2001 From: wu-changsheng Date: Wed, 21 Dec 2022 23:18:00 +0800 Subject: [PATCH] add mempool dfx info and write without epoll/poll (cherry picked from commit cce7ebb5649ba84f4195b92d7dbbbe8833d88677) --- 0168-dfx-add-mempool-count-info.patch | 151 +++++++++++++++++++ 0169--write-support-without-epoll-poll.patch | 82 ++++++++++ gazelle.spec | 8 +- 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 0168-dfx-add-mempool-count-info.patch create mode 100644 0169--write-support-without-epoll-poll.patch diff --git a/0168-dfx-add-mempool-count-info.patch b/0168-dfx-add-mempool-count-info.patch new file mode 100644 index 0000000..f3a7ee9 --- /dev/null +++ b/0168-dfx-add-mempool-count-info.patch @@ -0,0 +1,151 @@ +From 7472711c423224694f438db1c49f0c4196b0728d Mon Sep 17 00:00:00 2001 +From: wu-changsheng +Date: Tue, 20 Dec 2022 22:34:09 +0800 +Subject: [PATCH 1/2] dfx add-mempool-count-info + +--- + src/common/gazelle_dfx_msg.h | 1 + + src/lstack/core/lstack_lwip.c | 15 +++++++++++++-- + src/lstack/core/lstack_stack_stat.c | 3 +++ + src/lstack/core/lstack_thread_rpc.c | 16 ++++++++++++++++ + src/lstack/include/lstack_lwip.h | 1 + + src/lstack/include/lstack_thread_rpc.h | 1 + + src/ltran/ltran_dfx.c | 3 ++- + 7 files changed, 37 insertions(+), 3 deletions(-) + +diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h +index 83b6fe9..9225b53 100644 +--- a/src/common/gazelle_dfx_msg.h ++++ b/src/common/gazelle_dfx_msg.h +@@ -82,6 +82,7 @@ struct gazelle_stat_pkts { + uint64_t recv_list_cnt; + uint64_t call_alloc_fail; + uint64_t send_list_cnt; ++ uint32_t mempool_freecnt; + struct gazelle_stack_stat stack_stat; + struct gazelle_wakeup_stat wakeup_stat; + }; +diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c +index 32d21b6..9442d3f 100644 +--- a/src/lstack/core/lstack_lwip.c ++++ b/src/lstack/core/lstack_lwip.c +@@ -1273,12 +1273,23 @@ static uint32_t get_list_count(struct list_node *list) + return count; + } + ++void stack_mempool_size(struct rpc_msg *msg) ++{ ++ struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p; ++ ++ msg->result = rte_mempool_avail_count(stack->rxtx_pktmbuf_pool); ++} ++ + void stack_sendlist_count(struct rpc_msg *msg) + { +- msg->result = get_list_count(&get_protocol_stack()->send_list); ++ struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p; ++ ++ msg->result = get_list_count(&stack->send_list); + } + + void stack_recvlist_count(struct rpc_msg *msg) + { +- msg->result = get_list_count(&get_protocol_stack()->recv_list); ++ struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p; ++ ++ msg->result = get_list_count(&stack->recv_list); + } +diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c +index 45f84a7..59c8e66 100644 +--- a/src/lstack/core/lstack_stack_stat.c ++++ b/src/lstack/core/lstack_stack_stat.c +@@ -146,6 +146,9 @@ static void get_stack_stats(struct gazelle_stack_dfx_data *dfx, struct protocol_ + int32_t rpc_call_result = rpc_call_msgcnt(stack); + dfx->data.pkts.call_msg_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; + ++ rpc_call_result = rpc_call_mempoolsize(stack); ++ dfx->data.pkts.mempool_freecnt = (rpc_call_result < 0) ? 0 : rpc_call_result; ++ + rpc_call_result = rpc_call_recvlistcnt(stack); + dfx->data.pkts.recv_list_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; + +diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c +index 29ca4e4..08fe20d 100644 +--- a/src/lstack/core/lstack_thread_rpc.c ++++ b/src/lstack/core/lstack_thread_rpc.c +@@ -205,6 +205,18 @@ int32_t rpc_call_thread_regphase2(struct protocol_stack *stack, void *conn) + return rpc_sync_call(&stack->rpc_queue, msg); + } + ++int32_t rpc_call_mempoolsize(struct protocol_stack *stack) ++{ ++ struct rpc_msg *msg = rpc_msg_alloc(stack, stack_mempool_size); ++ if (msg == NULL) { ++ return -1; ++ } ++ ++ msg->args[MSG_ARG_0].p = stack; ++ ++ return rpc_sync_call(&stack->rpc_queue, msg); ++} ++ + int32_t rpc_call_sendlistcnt(struct protocol_stack *stack) + { + struct rpc_msg *msg = rpc_msg_alloc(stack, stack_sendlist_count); +@@ -212,6 +224,8 @@ int32_t rpc_call_sendlistcnt(struct protocol_stack *stack) + return -1; + } + ++ msg->args[MSG_ARG_0].p = stack; ++ + return rpc_sync_call(&stack->rpc_queue, msg); + } + +@@ -222,6 +236,8 @@ int32_t rpc_call_recvlistcnt(struct protocol_stack *stack) + return -1; + } + ++ msg->args[MSG_ARG_0].p = stack; ++ + return rpc_sync_call(&stack->rpc_queue, msg); + } + +diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h +index b24006a..6f5b4f4 100644 +--- a/src/lstack/include/lstack_lwip.h ++++ b/src/lstack/include/lstack_lwip.h +@@ -47,5 +47,6 @@ ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags) + ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags); + ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags); + void rpc_replenish(struct rpc_msg *msg); ++void stack_mempool_size(struct rpc_msg *msg); + + #endif +diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h +index 2c1202e..aff30dc 100644 +--- a/src/lstack/include/lstack_thread_rpc.h ++++ b/src/lstack/include/lstack_thread_rpc.h +@@ -77,5 +77,6 @@ int32_t rpc_call_setsockopt(int fd, int level, int optname, const void *optval, + int32_t rpc_call_fcntl(int fd, int cmd, long val); + int32_t rpc_call_ioctl(int fd, long cmd, void *argp); + int32_t rpc_call_replenish(struct protocol_stack *stack, struct lwip_sock *sock); ++int32_t rpc_call_mempoolsize(struct protocol_stack *stack); + + #endif +diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c +index 651f279..7dda51c 100644 +--- a/src/ltran/ltran_dfx.c ++++ b/src/ltran/ltran_dfx.c +@@ -579,7 +579,8 @@ static void show_lstack_stats(struct gazelle_stack_dfx_data *lstack_stat) + printf("call_msg: %-19"PRIu64" ", lstack_stat->data.pkts.call_msg_cnt); + printf("call_alloc_fail: %-12"PRIu64" ", lstack_stat->data.pkts.call_alloc_fail); + printf("call_null: %-18"PRIu64" \n", lstack_stat->data.pkts.stack_stat.call_null); +- printf("send_pkts_fail: %-13"PRIu64" \n", lstack_stat->data.pkts.stack_stat.send_pkts_fail); ++ printf("send_pkts_fail: %-13"PRIu64" ", lstack_stat->data.pkts.stack_stat.send_pkts_fail); ++ printf("mempool_freecnt: %-12"PRIu32" \n", lstack_stat->data.pkts.mempool_freecnt); + } + + static void gazelle_print_lstack_stat_detail(struct gazelle_stack_dfx_data *lstack_stat, +-- +2.23.0 + diff --git a/0169--write-support-without-epoll-poll.patch b/0169--write-support-without-epoll-poll.patch new file mode 100644 index 0000000..ffee8b7 --- /dev/null +++ b/0169--write-support-without-epoll-poll.patch @@ -0,0 +1,82 @@ +From d4d8516449ed6c8d7685cc1aaec403763a5d5afa Mon Sep 17 00:00:00 2001 +From: wu-changsheng +Date: Wed, 21 Dec 2022 23:04:57 +0800 +Subject: [PATCH 2/2] write support without epoll/poll + +--- + src/common/dpdk_common.h | 4 ++-- + src/lstack/core/lstack_lwip.c | 18 ++++++++++++------ + 2 files changed, 14 insertions(+), 8 deletions(-) + +diff --git a/src/common/dpdk_common.h b/src/common/dpdk_common.h +index c93f506..e4a447a 100644 +--- a/src/common/dpdk_common.h ++++ b/src/common/dpdk_common.h +@@ -25,9 +25,9 @@ + #define PTR_TO_PRIVATE(mbuf) RTE_PTR_ADD(mbuf, sizeof(struct rte_mbuf)) + + /* Layout: +- * | rte_mbuf | gazelle_prive | pbuf | custom_free_function | tcp_seg | payload | ++ * | rte_mbuf | gazelle_prive | custom_free_function | tcp_seg | payload | ++ * | 128 | 16 | 56 | 32 | + * rte_prefetch0 in lwip project,tcp_out.c,tcp_output_segment use constants +- sizeof(struct rte_mbuf) + GAZELLE_MBUFF_PRIV_SIZE = 128 + 16 + **/ + struct pbuf; + static inline struct rte_mbuf *pbuf_to_mbuf(struct pbuf *p) +diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c +index 9442d3f..018b6c6 100644 +--- a/src/lstack/core/lstack_lwip.c ++++ b/src/lstack/core/lstack_lwip.c +@@ -500,8 +500,7 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) + } + + struct protocol_stack *stack = sock->stack; +- struct wakeup_poll *wakeup = sock->wakeup; +- if (!stack|| len == 0 || !wakeup) { ++ if (!stack|| len == 0) { + return 0; + } + +@@ -517,6 +516,7 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) + + uint32_t write_num = (len - send_len + MBUF_MAX_DATA_LEN - 1) / MBUF_MAX_DATA_LEN; + uint32_t write_avail = gazelle_ring_readable_count(sock->send_ring); ++ struct wakeup_poll *wakeup = sock->wakeup; + + /* send_ring is full, data attach last pbuf */ + if (write_avail == 0) { +@@ -524,10 +524,14 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) + if (last_pbuf) { + send_len += app_direct_attach(stack, last_pbuf, (char *)buf + send_len, len - send_len, write_num); + gazelle_ring_lastover(last_pbuf); +- wakeup->stat.app_write_cnt += write_num; ++ if (wakeup) { ++ wakeup->stat.app_write_cnt += write_num; ++ } + } else { + (void)rpc_call_replenish(stack, sock); +- wakeup->stat.app_write_rpc++; ++ if (wakeup) { ++ wakeup->stat.app_write_rpc++; ++ } + } + sock->remain_len = 0; + return send_len; +@@ -536,9 +540,11 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) + /* send_ring have idle */ + send_len += (write_num <= write_avail) ? app_buff_write(sock, (char *)buf + send_len, len - send_len, write_num) : + app_direct_write(stack, sock, (char *)buf + send_len, len - send_len, write_num); +- wakeup->stat.app_write_cnt += write_num; ++ if (wakeup) { ++ wakeup->stat.app_write_cnt += write_num; ++ } + +- if (wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)) { ++ if (wakeup && wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)) { + del_data_out_event(sock); + } + +-- +2.23.0 + diff --git a/gazelle.spec b/gazelle.spec index b75dddf..c8f070c 100644 --- a/gazelle.spec +++ b/gazelle.spec @@ -2,7 +2,7 @@ Name: gazelle Version: 1.0.1 -Release: 38 +Release: 39 Summary: gazelle is a high performance user-mode stack License: MulanPSL-2.0 URL: https://gitee.com/openeuler/gazelle @@ -182,6 +182,8 @@ Patch9164: 0164-rpc-dont-send.patch Patch9165: 0165-recv-pbuf-free-timely.patch Patch9166: 0166-optimite-send-pkts-dul-index.patch Patch9167: 0167-expand-data-recv-buff.patch +Patch9168: 0168-dfx-add-mempool-count-info.patch +Patch9169: 0169--write-support-without-epoll-poll.patch %description %{name} is a high performance user-mode stack. @@ -222,6 +224,10 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b %config(noreplace) %{conf_path}/ltran.conf %changelog +* Wed Dec 21 2022 wuchangsheng - 1.0.1-39 +- add mempool dfx info + write support without epoll/poll + * Tue Dec 20 2022 wuchangsheng - 1.0.1-38 - optimite recv data buff and send pkts index