diff --git a/0215-gazelle-send-recv-thread-bind-numa.patch b/0215-gazelle-send-recv-thread-bind-numa.patch new file mode 100644 index 0000000..b91b197 --- /dev/null +++ b/0215-gazelle-send-recv-thread-bind-numa.patch @@ -0,0 +1,65 @@ +From b658a975bb76d3b39edb7fd73ab5815c302bce02 Mon Sep 17 00:00:00 2001 +From: kircher +Date: Tue, 21 Mar 2023 11:08:46 +0800 +Subject: [PATCH] gazelle send/recv thread bind numa + +--- + src/lstack/core/lstack_lwip.c | 21 +++++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c +index 60abfe8..7355d7b 100644 +--- a/src/lstack/core/lstack_lwip.c ++++ b/src/lstack/core/lstack_lwip.c +@@ -97,6 +97,7 @@ static void reset_sock_data(struct lwip_sock *sock) + sock->events = 0; + sock->call_num = 0; + sock->remain_len = 0; ++ sock->already_bind_numa = 0; + + if (sock->recv_lastdata) { + pbuf_free(sock->recv_lastdata); +@@ -882,6 +883,21 @@ ssize_t gazelle_same_node_ring_send(struct lwip_sock *sock, const void *buf, siz + return act_len; + } + ++PER_THREAD uint16_t stack_sock_num[GAZELLE_MAX_STACK_NUM] = {0}; ++PER_THREAD uint16_t max_sock_stack = 0; ++ ++static void thread_bind_stack(struct lwip_sock *sock) { ++ if (likely(!sock->stack || sock->already_bind_numa)) { ++ return; ++ } ++ sock->already_bind_numa = 1; ++ stack_sock_num[sock->stack->stack_idx]++; ++ if (stack_sock_num[sock->stack->stack_idx] > max_sock_stack) { ++ max_sock_stack = stack_sock_num[sock->stack->stack_idx]; ++ bind_to_stack_numa(sock->stack); ++ } ++} ++ + ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags) + { + if (buf == NULL) { +@@ -893,6 +909,9 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags) + } + + struct lwip_sock *sock = get_socket_by_fd(fd); ++ ++ thread_bind_stack(sock); ++ + if (sock->same_node_tx_ring != NULL) { + return gazelle_same_node_ring_send(sock, buf, len, flags); + } +@@ -973,6 +992,8 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags) + return 0; + } + ++ thread_bind_stack(sock); ++ + if (sock->same_node_rx_ring != NULL) { + return gazelle_same_node_ring_recv(sock, buf, len, flags); + } +-- +2.33.0 + diff --git a/0216-waiting-when-primary-process-not-start-already.patch b/0216-waiting-when-primary-process-not-start-already.patch new file mode 100644 index 0000000..43c1571 --- /dev/null +++ b/0216-waiting-when-primary-process-not-start-already.patch @@ -0,0 +1,137 @@ +From f17b74371b14b97521462724069812fc9728b9ed Mon Sep 17 00:00:00 2001 +From: kircher +Date: Tue, 21 Mar 2023 15:36:52 +0800 +Subject: [PATCH] waiting when primary process not start already + +--- + src/common/gazelle_opt.h | 1 + + src/lstack/api/lstack_signal.c | 7 +++-- + src/lstack/core/lstack_control_plane.c | 6 ++++ + src/lstack/core/lstack_init.c | 34 +++++++++++++++++++++++ + src/lstack/include/lstack_control_plane.h | 1 + + 5 files changed, 47 insertions(+), 2 deletions(-) + +diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h +index 4c0eef3..fe0483b 100644 +--- a/src/common/gazelle_opt.h ++++ b/src/common/gazelle_opt.h +@@ -85,6 +85,7 @@ + #define GAZELLE_SOCK_FILENAME_MAXLEN 128 + + #define GAZELLE_RUN_DIR "/var/run/gazelle/" ++#define GAZELLE_PRIMARY_START_PATH "/var/run/gazelle/gazelle_primary" + #define GAZELLE_FILE_PERMISSION 0700 + + #define SEND_TIME_WAIT_NS 20000 +diff --git a/src/lstack/api/lstack_signal.c b/src/lstack/api/lstack_signal.c +index e73bc61..03118f5 100644 +--- a/src/lstack/api/lstack_signal.c ++++ b/src/lstack/api/lstack_signal.c +@@ -57,12 +57,15 @@ static inline bool match_hijack_signal(int sig) + static void lstack_sig_default_handler(int sig) + { + LSTACK_LOG(ERR, LSTACK, "lstack dumped，caught signal：%d\n", sig); +- dump_stack(); +- lwip_exit(); ++ if (get_global_cfg_params() && get_global_cfg_params()->is_primary) { ++ delete_primary_path(); ++ } + if (!use_ltran()) { + dpdk_kni_release(); + } + control_fd_close(); ++ dump_stack(); ++ lwip_exit(); + (void)kill(getpid(), sig); + } + +diff --git a/src/lstack/core/lstack_control_plane.c b/src/lstack/core/lstack_control_plane.c +index bc2c55d..e705cae 100644 +--- a/src/lstack/core/lstack_control_plane.c ++++ b/src/lstack/core/lstack_control_plane.c +@@ -473,6 +473,12 @@ void control_fd_close(void) + } + } + ++void delete_primary_path(void) { ++ if (!remove(GAZELLE_PRIMARY_START_PATH)) { ++ LSTACK_LOG(ERR, LSTACK, "delete %s failed\n", GAZELLE_PRIMARY_START_PATH); ++ } ++} ++ + int32_t control_init_client(bool is_reconnect) + { + int32_t ret; +diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c +index e8fa0dc..3537002 100644 +--- a/src/lstack/core/lstack_init.c ++++ b/src/lstack/core/lstack_init.c +@@ -151,6 +151,32 @@ bool select_thread_path(void) + return true; + } + ++static void check_process_start(void) { ++ if (get_global_cfg_params()->is_primary) { ++ return; ++ } ++ ++ while (!fopen(GAZELLE_PRIMARY_START_PATH, "r")) { ++ printf("please make sure the primary process start already!\n"); ++ sleep(1); ++ } ++} ++ ++static int32_t set_process_start_flag(void) { ++ if (!get_global_cfg_params()->is_primary) { ++ return 0; ++ } ++ ++ FILE *fp = NULL; ++ fp = fopen(GAZELLE_PRIMARY_START_PATH, "w"); ++ if (fp == NULL) { ++ LSTACK_PRE_LOG(LSTACK_ERR, "set primary proceaa start flag failed!\n"); ++ return -1; ++ } ++ (void)fclose(fp); ++ return 0; ++} ++ + static int32_t check_process_conflict(void) + { + int32_t ret; +@@ -340,6 +366,10 @@ __attribute__((constructor)) void gazelle_network_init(void) + } + LSTACK_PRE_LOG(LSTACK_INFO, "cfg_init success\n"); + ++ /* ++ * check primary process start */ ++ check_process_start(); ++ + /* + * check conflict */ + if (check_process_conflict() < 0) { +@@ -402,6 +432,10 @@ __attribute__((constructor)) void gazelle_network_init(void) + set_kni_ip_mac(); + } + ++ if (set_process_start_flag() != 0) { ++ LSTACK_EXIT(1, "set_process_start_flag failed\n"); ++ } ++ + posix_api->ues_posix = 0; + LSTACK_LOG(INFO, LSTACK, "gazelle_network_init success\n"); + rte_smp_mb(); +diff --git a/src/lstack/include/lstack_control_plane.h b/src/lstack/include/lstack_control_plane.h +index d631ea6..aed5443 100644 +--- a/src/lstack/include/lstack_control_plane.h ++++ b/src/lstack/include/lstack_control_plane.h +@@ -32,5 +32,6 @@ bool get_register_state(void); + void thread_register_phase1(struct rpc_msg *msg); + void thread_register_phase2(struct rpc_msg *msg); + void control_fd_close(void); ++void delete_primary_path(void); + + #endif /* GAZELLE_CONTROL_PLANE_H */ +-- +2.33.0 + diff --git a/gazelle.spec b/gazelle.spec index a6bfa11..7783522 100644 --- a/gazelle.spec +++ b/gazelle.spec @@ -2,7 +2,7 @@ Name: gazelle Version: 1.0.1 -Release: 53 +Release: 54 Summary: gazelle is a high performance user-mode stack License: MulanPSL-2.0 URL: https://gitee.com/openeuler/gazelle @@ -229,6 +229,8 @@ Patch9211: 0211-refactor-mbuf-private-data.patch Patch9212: 0212-fix-kernel-scoket-select-path-error.patch Patch9213: 0213-discard-wakeup_num-parameter.patch Patch9214: 0214-fix-parse-args-error.patch +Patch9215: 0215-gazelle-send-recv-thread-bind-numa.patch +Patch9216: 0216-waiting-when-primary-process-not-start-already.patch %description %{name} is a high performance user-mode stack. @@ -269,6 +271,10 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b %config(noreplace) %{conf_path}/ltran.conf %changelog +* Thu Apr 20 UTC wu-changsheng - 1.0.1-54 +- waiting when primary process not start already +- gazelle send/recv thread bind numa + * Tue Mar 21 2023 jiangheng12 - 1.0.1-53 - fix parse args error - discard wakeup_num parameter