sync fix EPOLLIN event error

(cherry picked from commit 3b9b083b6a7c88ad7888483c6d027acc0abad1f3)
This commit is contained in:
yinbin 2023-12-16 15:16:21 +08:00 committed by openeuler-sync-bot
parent 0d8bbab6cb
commit b17dbaebfc
7 changed files with 1082 additions and 1 deletions

View File

@ -0,0 +1,24 @@
From 3f93ac98b09a3bdb0dc557f9b93a008d809cd5c3 Mon Sep 17 00:00:00 2001
From: yinbin <yinbin8@huawei.com>
Date: Sun, 10 Dec 2023 14:55:16 +0800
Subject: [PATCH] dfx: fix kernel_events stat
---
src/lstack/core/lstack_stack_stat.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c
index 2d85efa..e4ccc41 100644
--- a/src/lstack/core/lstack_stack_stat.c
+++ b/src/lstack/core/lstack_stack_stat.c
@@ -136,6 +136,7 @@ static void get_wakeup_stat(struct protocol_stack_group *stack_group, struct pro
struct wakeup_poll *wakeup = container_of(node, struct wakeup_poll, poll_list);
if (wakeup->bind_stack == stack) {
+ stat->kernel_events += wakeup->stat.kernel_events;
stat->app_events += wakeup->stat.app_events;
stat->read_null += wakeup->stat.read_null;
stat->app_write_cnt += wakeup->stat.app_write_cnt;
--
2.27.0

View File

@ -0,0 +1,111 @@
From 5e6a73b66e0b231237ea3a3bbedfe4dd4d974f45 Mon Sep 17 00:00:00 2001
From: hantwofish <hankangkang5@huawei.com>
Date: Mon, 11 Dec 2023 05:03:04 +0800
Subject: [PATCH] add keep-alive info
---
src/common/gazelle_dfx_msg.h | 2 ++
src/lstack/core/lstack_lwip.c | 2 ++
src/ltran/ltran_dfx.c | 25 ++++++++++++++++++++++---
3 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 3d559c3..478c440 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -197,6 +197,8 @@ struct gazelle_stat_lstack_conn_info {
uint32_t eventlist;
uint32_t keepalive;
uint32_t keep_idle;
+ uint32_t keep_intvl;
+ uint32_t keep_cnt;
};
struct gazelle_stat_lstack_conn {
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index c4b1ebc..cbb771f 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -1151,6 +1151,8 @@ static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const s
conn->rcv_nxt = pcb->rcv_nxt;
conn->keepalive = (ip_get_option(pcb, SOF_KEEPALIVE) != 0);
conn->keep_idle = pcb->keep_idle;
+ conn->keep_intvl = pcb->keep_intvl;
+ conn->keep_cnt = pcb->keep_cnt;
if (netconn != NULL && netconn->recvmbox != NULL) {
conn->recv_cnt = rte_ring_count(netconn->recvmbox->ring);
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index ebe1323..2a84cb8 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -52,6 +52,7 @@
#define GAZELLE_MAX_LATENCY_TIME 1800 // max latency time 30mins
#define GAZELLE_DECIMAL 10
+#define GAZELLE_KEEPALIVE_STR_LEN 35
static int32_t g_unix_fd = -1;
static int32_t g_ltran_rate_show_flag = GAZELLE_OFF; // not show when first get total statistics
@@ -995,6 +996,20 @@ static void gazelle_print_lstack_stat_snmp(void *buf, const struct gazelle_stat_
} while (true);
}
+static void gazelle_keepalive_string(char* str, int buff_len, struct gazelle_stat_lstack_conn_info *conn_info)
+{
+ if (conn_info->keepalive == 0) {
+ return;
+ }
+ int ret = sprintf_s(str, buff_len - 1, "(%u,%u,%u)", conn_info->keep_idle, conn_info->keep_intvl,
+ conn_info->keep_cnt);
+ if (ret < 0) {
+ printf("gazelle_keepalive_string sprintf_s fail ret=%d\n", ret);
+ return;
+ }
+ str[strlen(str)] = '\0';
+}
+
static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_msg_request *req_msg)
{
uint32_t i;
@@ -1007,6 +1022,7 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
struct gazelle_stat_lstack_conn *conn = &stat->data.conn;
struct timeval time = {0};
gettimeofday(&time, NULL);
+ char keepalive_info_str[GAZELLE_KEEPALIVE_STR_LEN] = {0};
printf("Active Internet connections (servers and established)\n");
do {
@@ -1014,7 +1030,7 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
printf("No. Proto lwip_recv recv_ring in_send send_ring cwn rcv_wnd snd_wnd snd_buf snd_nxt"
" lastack rcv_nxt events epoll_ev evlist fd Local Address"
" Foreign Address State"
- " keep-alive keep-idle\n");
+ " keep-alive keep-alive(idle,intvl,cnt)\n");
uint32_t unread_pkts = 0;
uint32_t unsend_pkts = 0;
for (i = 0; i < conn->conn_num && i < GAZELLE_LSTACK_MAX_CONN; i++) {
@@ -1027,15 +1043,18 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
if ((conn_info->state == GAZELLE_ACTIVE_LIST) || (conn_info->state == GAZELLE_TIME_WAIT_LIST)) {
inet_ntop(domain, lip, str_ip, sizeof(str_ip));
inet_ntop(domain, rip, str_rip, sizeof(str_rip));
+
+ gazelle_keepalive_string(keepalive_info_str, sizeof(keepalive_info_str)/sizeof(char), conn_info);
+
sprintf_s(str_laddr, sizeof(str_laddr), "%s:%hu", str_ip, conn_info->l_port);
sprintf_s(str_raddr, sizeof(str_raddr), "%s:%hu", str_rip, conn_info->r_port);
printf("%-6utcp %-10u%-10u%-8u%-10u%-9d%-9d%-10d%-10d%-15u%-15u%-15u%-10x%-10x%-7d%-7d"
- "%-52s %-52s %s %-5d %-9u\n", i, conn_info->recv_cnt, conn_info->recv_ring_cnt, conn_info->in_send,
+ "%-52s %-52s %s %-5d %s\n", i, conn_info->recv_cnt, conn_info->recv_ring_cnt, conn_info->in_send,
conn_info->send_ring_cnt, conn_info->cwn, conn_info->rcv_wnd, conn_info->snd_wnd,
conn_info->snd_buf, conn_info->snd_nxt, conn_info->lastack, conn_info->rcv_nxt, conn_info->events,
conn_info->epoll_events, conn_info->eventlist, conn_info->fd,
str_laddr, str_raddr, tcp_state_to_str(conn_info->tcp_sub_state),
- conn_info->keepalive, conn_info->keep_idle);
+ conn_info->keepalive, keepalive_info_str);
} else if (conn_info->state == GAZELLE_LISTEN_LIST) {
inet_ntop(domain, lip, str_ip, sizeof(str_ip));
sprintf_s(str_laddr, sizeof(str_laddr), "%s:%hu", str_ip, conn_info->l_port);
--
2.27.0

View File

@ -0,0 +1,137 @@
From f4abd3b3fd5004405cb186981b93f5d40e4648db Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Sun, 10 Dec 2023 17:47:56 +0800
Subject: [PATCH] fix close can't exit
---
src/lstack/core/lstack_lwip.c | 12 ++++++++----
src/lstack/core/lstack_protocol_stack.c | 25 +++++++++++++++----------
src/lstack/core/lstack_thread_rpc.c | 6 +++---
src/lstack/include/lstack_lwip.h | 4 ++--
4 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index c4b1ebc..73a6f12 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -646,14 +646,18 @@ bool do_lwip_replenish_sendring(struct protocol_stack *stack, struct lwip_sock *
return replenish_again;
}
-bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
- size_t len, int32_t flags)
+int do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
+ size_t len, int32_t flags)
{
+ ssize_t ret;
/* send all send_ring, so len set lwip send max. */
if (NETCONN_IS_UDP(sock)) {
- (void)lwip_send(fd, sock, len, flags);
+ ret = lwip_send(fd, sock, len, flags);
} else {
- (void)lwip_send(fd, sock, UINT16_MAX, flags);
+ ret = lwip_send(fd, sock, UINT16_MAX, flags);
+ }
+ if (ret < 0 && (errno == ENOTCONN || errno == ECONNRESET || errno == ECONNABORTED)) {
+ return -1;
}
return do_lwip_replenish_sendring(stack, sock);
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index f61e7a8..8dbd9ad 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -699,10 +699,10 @@ void stack_close(struct rpc_msg *msg)
struct protocol_stack *stack = get_protocol_stack_by_fd(fd);
struct lwip_sock *sock = get_socket(fd);
- if (sock && NETCONN_IS_DATAOUT(sock)) {
+ if (sock && __atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) > 0) {
msg->recall_flag = 1;
rpc_call(&stack->rpc_queue, msg); /* until stack_send recall finish */
- return;
+ return;
}
msg->result = lwip_close(fd);
@@ -860,27 +860,32 @@ void stack_send(struct rpc_msg *msg)
int32_t fd = msg->args[MSG_ARG_0].i;
size_t len = msg->args[MSG_ARG_1].size;
struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p;
- bool replenish_again;
+ int replenish_again;
struct lwip_sock *sock = get_socket(fd);
if (sock == NULL) {
msg->result = -1;
LSTACK_LOG(ERR, LSTACK, "get sock error! fd=%d, len=%ld\n", fd, len);
- rpc_msg_free(msg);
+ __sync_fetch_and_sub(&sock->call_num, 1);
return;
}
replenish_again = do_lwip_send(stack, sock->conn->socket, sock, len, 0);
- __sync_fetch_and_sub(&sock->call_num, 1);
- if (!NETCONN_IS_DATAOUT(sock) && !replenish_again) {
+ if (replenish_again < 0) {
+ __sync_fetch_and_sub(&sock->call_num, 1);
return;
- } else {
- if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0) {
- msg->recall_flag = 1;
+ }
+
+ if (NETCONN_IS_DATAOUT(sock) || replenish_again > 0) {
+ if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 1) {
+ msg->recall_flag = 1;
rpc_call(&stack->rpc_queue, msg);
- __sync_fetch_and_add(&sock->call_num, 1);
+ return;
}
}
+
+ __sync_fetch_and_sub(&sock->call_num, 1);
+ return;
}
/* any protocol stack thread receives arp packet and sync it to other threads so that it can have the arp table */
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 4aceee6..0b2a62a 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -109,14 +109,14 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
}
if (!msg->recall_flag) {
- if (msg->sync_flag) {
+ if (msg->sync_flag) {
pthread_spin_unlock(&msg->lock);
} else {
rpc_msg_free(msg);
}
} else {
- msg->recall_flag = 0;
- }
+ msg->recall_flag = 0;
+ }
}
}
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index 4a13204..a11489c 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -50,8 +50,8 @@ ssize_t do_lwip_read_from_stack(int32_t fd, void *buf, size_t len, int32_t flags
void do_lwip_read_recvlist(struct protocol_stack *stack, uint32_t max_num);
void do_lwip_add_recvlist(int32_t fd);
-bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
- size_t len, int32_t flags);
+int do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
+ size_t len, int32_t flags);
uint32_t do_lwip_get_conntable(struct gazelle_stat_lstack_conn_info *conn, uint32_t max_num);
uint32_t do_lwip_get_connnum(void);
--
2.27.0

View File

@ -0,0 +1,72 @@
From 95784bb047ae30ebff9d224ef178430fda91027e Mon Sep 17 00:00:00 2001
From: hantwofish <hankangkang5@huawei.com>
Date: Fri, 15 Dec 2023 03:34:46 +0800
Subject: [PATCH] mod unix time stamp to local time
---
src/ltran/ltran_dfx.c | 26 +++++++++++++++++++++-----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 2a84cb8..25d4f35 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -53,6 +53,7 @@
#define GAZELLE_DECIMAL 10
#define GAZELLE_KEEPALIVE_STR_LEN 35
+#define GAZELLE_TIME_STR_LEN 25
static int32_t g_unix_fd = -1;
static int32_t g_ltran_rate_show_flag = GAZELLE_OFF; // not show when first get total statistics
@@ -1001,13 +1002,27 @@ static void gazelle_keepalive_string(char* str, int buff_len, struct gazelle_sta
if (conn_info->keepalive == 0) {
return;
}
- int ret = sprintf_s(str, buff_len - 1, "(%u,%u,%u)", conn_info->keep_idle, conn_info->keep_intvl,
+ int ret = sprintf_s(str, buff_len, "(%u,%u,%u)", conn_info->keep_idle, conn_info->keep_intvl,
conn_info->keep_cnt);
if (ret < 0) {
printf("gazelle_keepalive_string sprintf_s fail ret=%d\n", ret);
return;
}
- str[strlen(str)] = '\0';
+}
+
+static void gazelle_localtime_string(char* str, int buff_len)
+{
+ struct timeval time = {0};
+ gettimeofday(&time, NULL);
+ struct tm* tm;
+ time_t t = time.tv_sec;
+ tm = localtime(&t);
+ int ret = sprintf_s(str, buff_len, "%d-%d-%d %d:%d:%d",
+ tm->tm_yday + 1900, tm->tm_mon + 1, tm->tm_yday, tm->tm_hour, tm->tm_min, tm->tm_sec);
+ if (ret < 0) {
+ printf("gazelle_localtime_string sprintf_s fail ret=%d\n", ret);
+ return;
+ }
}
static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_msg_request *req_msg)
@@ -1020,13 +1035,14 @@ static void gazelle_print_lstack_stat_conn(void *buf, const struct gazelle_stat_
char str_raddr[INET6_ADDRSTRLEN + 6] = {0};
struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf;
struct gazelle_stat_lstack_conn *conn = &stat->data.conn;
- struct timeval time = {0};
- gettimeofday(&time, NULL);
+
char keepalive_info_str[GAZELLE_KEEPALIVE_STR_LEN] = {0};
+ char sys_local_time_str[GAZELLE_TIME_STR_LEN] = {0};
+ gazelle_localtime_string(sys_local_time_str, GAZELLE_TIME_STR_LEN);
printf("Active Internet connections (servers and established)\n");
do {
- printf("\n------ stack tid: %6u ------time=%lu\n", stat->tid, time.tv_sec * 1000000 + time.tv_usec);
+ printf("\n------ stack tid: %6u ------time=%s\n", stat->tid, sys_local_time_str);
printf("No. Proto lwip_recv recv_ring in_send send_ring cwn rcv_wnd snd_wnd snd_buf snd_nxt"
" lastack rcv_nxt events epoll_ev evlist fd Local Address"
" Foreign Address State"
--
2.27.0

View File

@ -0,0 +1,671 @@
From 95c0a884ff26b42a75ee35639d789b40af131fd3 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Tue, 12 Dec 2023 19:24:14 +0800
Subject: [PATCH] optimize gazelle exit process 1. close all fds 2. lstack
thread exits, then gazelle process exits
---
src/common/gazelle_base_func.h | 2 +
src/common/gazelle_dfx_msg.c | 8 +-
src/lstack/api/dir.mk | 2 +-
src/lstack/api/lstack_dummy_api.c | 55 +++++++++++++
src/lstack/api/lstack_rtc_api.c | 1 +
src/lstack/api/lstack_signal.c | 5 +-
src/lstack/api/lstack_wrap.c | 12 +++
src/lstack/core/lstack_dpdk.c | 2 +-
src/lstack/core/lstack_init.c | 18 ++++-
src/lstack/core/lstack_lwip.c | 16 ++--
src/lstack/core/lstack_protocol_stack.c | 93 ++++++++++++++++++----
src/lstack/core/lstack_thread_rpc.c | 22 ++++-
src/lstack/include/lstack_dummy_api.h | 23 ++++++
src/lstack/include/lstack_lwip.h | 4 +-
src/lstack/include/lstack_protocol_stack.h | 6 +-
src/lstack/include/lstack_thread_rpc.h | 3 +-
src/lstack/include/lstack_wrap.h | 1 +
17 files changed, 237 insertions(+), 36 deletions(-)
create mode 100644 src/lstack/api/lstack_dummy_api.c
create mode 100644 src/lstack/include/lstack_dummy_api.h
diff --git a/src/common/gazelle_base_func.h b/src/common/gazelle_base_func.h
index d21ef5f..2d629c1 100644
--- a/src/common/gazelle_base_func.h
+++ b/src/common/gazelle_base_func.h
@@ -34,6 +34,8 @@ int32_t check_and_set_run_dir(void);
int32_t filename_check(const char* args);
+void gazelle_exit(void);
+
#undef container_of
#define container_of(ptr, type, field) ((type *)(void*)(((char *)(ptr)) - offsetof(type, field)))
diff --git a/src/common/gazelle_dfx_msg.c b/src/common/gazelle_dfx_msg.c
index 5fe4e06..ec23401 100644
--- a/src/common/gazelle_dfx_msg.c
+++ b/src/common/gazelle_dfx_msg.c
@@ -29,19 +29,19 @@ int read_specied_len(int fd, char *buf, size_t target_size)
while (total_read < target_size) {
int ret = poll(fds, 1, GAZELLECTL_TIMEOUT);
if (ret < 0) {
- printf("read_specied_len:: poll ret=%d \n", ret);
+ printf("read_specied_len: poll ret=%d \n", ret);
return -1;
} else if (ret == 0) {
- printf("read_specied_len:: time out \n");
+ printf("read_specied_len: time out \n");
return -1;
}
if (fds[0].revents & POLLIN) {
int n = read(fd, buf + total_read, target_size - total_read);
if (n < 0) {
- printf("read_specied_len:: read ret=%d \n", ret);
+ printf("read_specied_len: read ret=%d \n", ret);
return -1;
} else if (n == 0) {
- printf("read_specied_len:: Connection closed \n");
+ printf("read_specied_len: Connection closed \n");
return -1;
}
total_read += n;
diff --git a/src/lstack/api/dir.mk b/src/lstack/api/dir.mk
index ffbb137..729690d 100644
--- a/src/lstack/api/dir.mk
+++ b/src/lstack/api/dir.mk
@@ -8,7 +8,7 @@
# PURPOSE.
# See the Mulan PSL v2 for more details.
-SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c
+SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c lstack_dummy_api.c
$(eval $(call register_dir, api, $(SRC)))
diff --git a/src/lstack/api/lstack_dummy_api.c b/src/lstack/api/lstack_dummy_api.c
new file mode 100644
index 0000000..f327916
--- /dev/null
+++ b/src/lstack/api/lstack_dummy_api.c
@@ -0,0 +1,55 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+#include <sys/socket.h>
+#include <unistd.h>
+#include <errno.h>
+
+#define DUMMY_SLEEP_S 5
+
+static inline ssize_t dummy_exit(void)
+{
+ sleep(DUMMY_SLEEP_S);
+ errno = ENOTCONN;
+ return -1;
+}
+
+int dummy_socket(int domain, int type, int protocol)
+{
+ sleep(DUMMY_SLEEP_S);
+ return -1;
+}
+
+ssize_t dummy_write(int s, const void *mem, size_t size)
+{
+ return dummy_exit();
+}
+
+ssize_t dummy_writev(int s, const struct iovec *iov, int iovcnt)
+{
+ return dummy_exit();
+}
+
+ssize_t dummy_send(int sockfd, const void *buf, size_t len, int flags)
+{
+ return dummy_exit();
+}
+
+ssize_t dummy_sendmsg(int s, const struct msghdr *message, int flags)
+{
+ return dummy_exit();
+}
+
+ssize_t dummy_sendto(int sockfd, const void *buf, size_t len, int flags,
+ const struct sockaddr *addr, socklen_t addrlen)
+{
+ return dummy_exit();
+}
diff --git a/src/lstack/api/lstack_rtc_api.c b/src/lstack/api/lstack_rtc_api.c
index 50d72bc..d29e51e 100644
--- a/src/lstack/api/lstack_rtc_api.c
+++ b/src/lstack/api/lstack_rtc_api.c
@@ -20,6 +20,7 @@
#include "lstack_log.h"
#include "lstack_cfg.h"
#include "lstack_protocol_stack.h"
+#include "lstack_thread_rpc.h"
#include "lstack_rtc_api.h"
int rtc_poll(struct pollfd *fds, nfds_t nfds, int timeout)
diff --git a/src/lstack/api/lstack_signal.c b/src/lstack/api/lstack_signal.c
index 285aaf3..6da6eb4 100644
--- a/src/lstack/api/lstack_signal.c
+++ b/src/lstack/api/lstack_signal.c
@@ -18,6 +18,7 @@
#include <lwip/lwipsock.h>
#include <lwip/posix_api.h>
+#include "gazelle_base_func.h"
#include "lstack_cfg.h"
#include "dpdk_common.h"
#include "lstack_log.h"
@@ -60,15 +61,13 @@ static void lstack_sig_default_handler(int sig)
if (get_global_cfg_params() && get_global_cfg_params()->is_primary) {
delete_primary_path();
}
- if (!use_ltran()) {
- dpdk_kni_release();
- }
control_fd_close();
/* When operations such as pressing Ctrl+C or Kill, the call stack exit is not displayed. */
if (sig != SIGINT && sig != SIGTERM && sig != SIGKILL) {
dump_stack();
}
lwip_exit();
+ gazelle_exit();
(void)kill(getpid(), sig);
}
diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index 65a0a5a..89f54f8 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -36,6 +36,7 @@
#include "lstack_rtc_api.h"
#include "lstack_rtw_api.h"
+#include "lstack_dummy_api.h"
#ifndef SOCK_TYPE_MASK
#define SOCK_TYPE_MASK 0xf
@@ -112,6 +113,17 @@ void wrap_api_init(void)
}
}
+void wrap_api_set_dummy(void)
+{
+ g_wrap_api->socket_fn = dummy_socket;
+ g_wrap_api->send_fn = dummy_send;
+ g_wrap_api->write_fn = dummy_write;
+ g_wrap_api->writev_fn = dummy_writev;
+ g_wrap_api->send_msg = dummy_sendmsg;
+ g_wrap_api->send_to = dummy_sendto;
+ rte_wmb();
+}
+
static inline int32_t do_epoll_create1(int32_t flags)
{
if (select_posix_path() == PATH_KERNEL) {
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 8950591..e20dea8 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -897,4 +897,4 @@ void dpdk_nic_features_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id)
dfx->data.nic_features.tx_offload = dev_conf.txmode.offloads;
dfx->data.nic_features.rx_offload = dev_conf.rxmode.offloads;
return;
-}
\ No newline at end of file
+}
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index a3ca4ff..fef2942 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -107,10 +107,24 @@ static int32_t check_process_conflict(void)
return 0;
}
+void gazelle_exit(void)
+{
+ if (!get_global_cfg_params()->stack_mode_rtc) {
+ wrap_api_set_dummy();
+ /* 1: wait until app thread call send functio complete */
+ sleep(1);
+ stack_group_exit();
+ }
+ if (!use_ltran()) {
+ dpdk_kni_release();
+ }
+}
+
__attribute__((destructor)) void gazelle_network_exit(void)
{
if (posix_api != NULL && !posix_api->ues_posix) {
lwip_exit();
+ gazelle_exit();
}
if (!use_ltran()) {
@@ -118,8 +132,6 @@ __attribute__((destructor)) void gazelle_network_exit(void)
if (ret < 0) {
LSTACK_LOG(ERR, LSTACK, "rte_pdump_uninit failed\n");
}
-
- dpdk_kni_release();
}
}
@@ -289,6 +301,7 @@ __attribute__((constructor)) void gazelle_network_init(void)
if (!get_global_cfg_params()->stack_mode_rtc) {
if (stack_setup_thread() != 0) {
+ gazelle_exit();
LSTACK_EXIT(1, "stack_setup_thread failed\n");
}
}
@@ -301,6 +314,7 @@ __attribute__((constructor)) void gazelle_network_init(void)
}
if (set_process_start_flag() != 0) {
+ gazelle_exit();
LSTACK_EXIT(1, "set_process_start_flag failed\n");
}
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index fb286d6..15f99f9 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -201,7 +201,7 @@ void do_lwip_init_sock(int32_t fd)
init_list_node_null(&sock->event_list);
}
-void do_lwip_clean_sock(int32_t fd)
+void do_lwip_clean_sock(int fd)
{
struct lwip_sock *sock = get_socket_by_fd(fd);
if (sock == NULL || sock->stack == NULL) {
@@ -1193,7 +1193,15 @@ void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_soc
}
}
-int32_t do_lwip_socket(int domain, int type, int protocol)
+int do_lwip_close(int fd)
+{
+ int ret = lwip_close(fd);
+ do_lwip_clean_sock(fd);
+ posix_api->close_fn(fd);
+ return ret;
+}
+
+int do_lwip_socket(int domain, int type, int protocol)
{
int32_t fd = lwip_socket(domain, type, 0);
if (fd < 0) {
@@ -1204,9 +1212,7 @@ int32_t do_lwip_socket(int domain, int type, int protocol)
struct lwip_sock *sock = get_socket(fd);
if (sock == NULL || sock->stack == NULL) {
- lwip_close(fd);
- do_lwip_clean_sock(fd);
- posix_api->close_fn(fd);
+ do_lwip_close(fd);
return -1;
}
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 8dbd9ad..3123ca3 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -45,6 +45,22 @@ static struct protocol_stack_group g_stack_group = {0};
typedef void *(*stack_thread_func)(void *arg);
+static void stack_set_state(struct protocol_stack *stack, enum rte_lcore_state_t state)
+{
+ __atomic_store_n(&stack->state, state, __ATOMIC_RELEASE);
+}
+
+static enum rte_lcore_state_t stack_get_state(struct protocol_stack *stack)
+{
+ return __atomic_load_n(&stack->state, __ATOMIC_ACQUIRE);
+}
+
+static void stack_wait_quit(struct protocol_stack *stack)
+{
+ while (__atomic_load_n(&stack->state, __ATOMIC_ACQUIRE) != WAIT) {
+ rte_pause();
+ }
+}
void bind_to_stack_numa(struct protocol_stack *stack)
{
@@ -436,8 +452,9 @@ END:
return NULL;
}
-void stack_polling(uint32_t wakeup_tick)
+int stack_polling(uint32_t wakeup_tick)
{
+ int force_quit;
struct cfg_params *cfg = get_global_cfg_params();
uint8_t use_ltran_flag = cfg->use_ltran;
bool kni_switch = cfg->kni_switch;
@@ -448,7 +465,7 @@ void stack_polling(uint32_t wakeup_tick)
uint32_t read_connect_number = cfg->read_connect_number;
struct protocol_stack *stack = get_protocol_stack();
- poll_rpc_msg(stack, rpc_number);
+ force_quit = poll_rpc_msg(stack, rpc_number);
gazelle_eth_dev_poll(stack, use_ltran_flag, nic_read_number);
sys_timer_run();
if (cfg->low_power_mod != 0) {
@@ -456,7 +473,7 @@ void stack_polling(uint32_t wakeup_tick)
}
if (stack_mode_rtc) {
- return;
+ return force_quit;
}
do_lwip_read_recvlist(stack, read_connect_number);
@@ -482,7 +499,7 @@ void stack_polling(uint32_t wakeup_tick)
kni_handle_rx(stack->port_id);
}
}
- return;
+ return force_quit;
}
static void* gazelle_stack_thread(void *arg)
@@ -512,11 +529,14 @@ static void* gazelle_stack_thread(void *arg)
return NULL;
}
- for (;;) {
- stack_polling(wakeup_tick);
+ stack_set_state(stack, RUNNING);
+
+ while (stack_polling(wakeup_tick) == 0) {
wakeup_tick++;
}
+ stack_set_state(stack, WAIT);
+
return NULL;
}
@@ -576,6 +596,7 @@ int32_t stack_group_init(void)
LSTACK_LOG(ERR, LSTACK, "sem_init failed errno=%d\n", errno);
return -1;
}
+
stack_group->stack_setup_fail = 0;
if (get_global_cfg_params()->is_primary) {
@@ -705,14 +726,10 @@ void stack_close(struct rpc_msg *msg)
return;
}
- msg->result = lwip_close(fd);
+ msg->result = do_lwip_close(fd);
if (msg->result != 0) {
LSTACK_LOG(ERR, LSTACK, "tid %ld, fd %d failed %ld\n", get_stack_tid(), msg->args[MSG_ARG_0].i, msg->result);
}
-
- do_lwip_clean_sock(fd);
-
- posix_api->close_fn(fd);
}
void stack_shutdown(struct rpc_msg *msg)
@@ -775,9 +792,7 @@ void stack_accept(struct rpc_msg *msg)
struct lwip_sock *sock = get_socket(accept_fd);
if (sock == NULL || sock->stack == NULL) {
- lwip_close(accept_fd);
- do_lwip_clean_sock(accept_fd);
- posix_api->close_fn(accept_fd);
+ do_lwip_close(accept_fd);
LSTACK_LOG(ERR, LSTACK, "fd %d ret %d\n", fd, accept_fd);
return;
}
@@ -866,7 +881,6 @@ void stack_send(struct rpc_msg *msg)
if (sock == NULL) {
msg->result = -1;
LSTACK_LOG(ERR, LSTACK, "get sock error! fd=%d, len=%ld\n", fd, len);
- __sync_fetch_and_sub(&sock->call_num, 1);
return;
}
@@ -1260,3 +1274,52 @@ int32_t stack_broadcast_accept(int32_t fd, struct sockaddr *addr, socklen_t *add
return stack_broadcast_accept4(fd, addr, addrlen, 0);
}
+static void stack_all_fds_close(void)
+{
+ for (int i = 3; i < GAZELLE_MAX_CLIENTS + GAZELLE_RESERVED_CLIENTS; i++) {
+ struct lwip_sock *sock = get_socket(i);
+ if (sock && sock->stack == get_protocol_stack()) {
+ do_lwip_close(i);
+ }
+ }
+}
+
+static void stack_exit(void)
+{
+ stack_all_fds_close();
+}
+
+void stack_exit_by_rpc(struct rpc_msg *msg)
+{
+ stack_exit();
+}
+
+void stack_group_exit(void)
+{
+ int i;
+ struct protocol_stack_group *stack_group = get_protocol_stack_group();
+ struct protocol_stack *stack = get_protocol_stack();
+
+ for (i = 0; i < stack_group->stack_num; i++) {
+ if ((stack_group->stacks[i] == NULL) ||
+ stack_get_state(stack_group->stacks[i]) != RUNNING) {
+ continue;
+ }
+
+ if (stack != stack_group->stacks[i]) {
+ rpc_call_stack_exit(stack_group->stacks[i]);
+ }
+ }
+
+ if (stack != NULL) {
+ stack_exit();
+ }
+
+ for (i = 0; i < stack_group->stack_num; i++) {
+ if (stack_group->stacks[i] == NULL || stack == stack_group->stacks[i]) {
+ continue;
+ }
+ /* wait stack thread quit */
+ stack_wait_quit(stack_group->stacks[i]);
+ }
+}
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 0b2a62a..2af30d7 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -90,8 +90,9 @@ static inline __attribute__((always_inline)) int32_t rpc_sync_call(lockless_queu
return ret;
}
-void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
+int poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
{
+ int force_quit = 0;
struct rpc_msg *msg = NULL;
while (max_num--) {
@@ -108,6 +109,10 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
stack->stats.call_null++;
}
+ if (msg->func == stack_exit_by_rpc) {
+ force_quit = 1;
+ }
+
if (!msg->recall_flag) {
if (msg->sync_flag) {
pthread_spin_unlock(&msg->lock);
@@ -118,6 +123,8 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
msg->recall_flag = 0;
}
}
+
+ return force_quit;
}
int32_t rpc_call_conntable(struct protocol_stack *stack, void *conn_table, uint32_t max_conn)
@@ -246,6 +253,7 @@ int32_t rpc_call_arp(struct protocol_stack *stack, struct rte_mbuf *mbuf)
int32_t rpc_call_socket(int32_t domain, int32_t type, int32_t protocol)
{
struct protocol_stack *stack = get_bind_protocol_stack();
+
struct rpc_msg *msg = rpc_msg_alloc(stack, stack_socket);
if (msg == NULL) {
return -1;
@@ -271,6 +279,18 @@ int32_t rpc_call_close(int fd)
return rpc_sync_call(&stack->rpc_queue, msg);
}
+int32_t rpc_call_stack_exit(struct protocol_stack *stack)
+{
+ struct rpc_msg *msg = rpc_msg_alloc(stack, stack_exit_by_rpc);
+ if (msg == NULL) {
+ LSTACK_LOG(INFO, LSTACK, "rpc msg alloc failed\n");
+ return -1;
+ }
+
+ rpc_call(&stack->rpc_queue, msg);
+ return 0;
+}
+
int32_t rpc_call_shutdown(int fd, int how)
{
struct protocol_stack *stack = get_protocol_stack_by_fd(fd);
diff --git a/src/lstack/include/lstack_dummy_api.h b/src/lstack/include/lstack_dummy_api.h
new file mode 100644
index 0000000..48bce31
--- /dev/null
+++ b/src/lstack/include/lstack_dummy_api.h
@@ -0,0 +1,23 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef _LSTACK_DUMMY_API_H_
+#define _LSTACK_DUMMY_API_H_
+
+int dummy_socket(int domain, int type, int protocol);
+ssize_t dummy_write(int s, const void *mem, size_t size);
+ssize_t dummy_writev(int s, const struct iovec *iov, int iovcnt);
+ssize_t dummy_sendmsg(int s, const struct msghdr *message, int flags);
+ssize_t dummy_send(int sockfd, const void *buf, size_t len, int flags);
+ssize_t dummy_sendto(int sockfd, const void *buf, size_t len, int flags,
+ const struct sockaddr *addr, socklen_t addrlen);
+#endif /* __LSTACK_DUMMY_API_H_ */
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index a11489c..0a82781 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -28,9 +28,9 @@ struct rpc_msg;
struct rte_mbuf;
struct protocol_stack;
-int32_t do_lwip_socket(int domain, int type, int protocol);
+int do_lwip_socket(int domain, int type, int protocol);
+int do_lwip_close(int32_t fd);
void do_lwip_init_sock(int32_t fd);
-void do_lwip_clean_sock(int32_t fd);
void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock);
struct pbuf *do_lwip_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 6638984..8e2e807 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -46,6 +46,7 @@ struct protocol_stack {
uint32_t stack_idx;
cpu_set_t idle_cpuset; /* idle cpu in numa of stack, app thread bind to it */
int32_t epollfd; /* kernel event thread epoll fd */
+ volatile enum rte_lcore_state_t state;
struct rte_mempool *rxtx_mbuf_pool;
struct rte_ring *rx_ring;
@@ -114,6 +115,7 @@ struct protocol_stack *get_bind_protocol_stack(void);
struct protocol_stack_group *get_protocol_stack_group(void);
int32_t stack_group_init(void);
+void stack_group_exit(void);
int32_t stack_setup_thread(void);
int32_t stack_setup_app_thread(void);
@@ -176,6 +178,8 @@ void stack_replenish_sendring(struct rpc_msg *msg);
void stack_get_conntable(struct rpc_msg *msg);
void stack_get_connnum(struct rpc_msg *msg);
void stack_recvlist_count(struct rpc_msg *msg);
-void stack_polling(uint32_t wakeup_tick);
+void stack_exit_by_rpc(struct rpc_msg *msg);
+
+int stack_polling(uint32_t wakeup_tick);
void kni_handle_tx(struct rte_mbuf *mbuf);
#endif
diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h
index ca8a510..633ef93 100644
--- a/src/lstack/include/lstack_thread_rpc.h
+++ b/src/lstack/include/lstack_thread_rpc.h
@@ -62,7 +62,7 @@ struct protocol_stack;
struct rte_mbuf;
struct wakeup_poll;
struct lwip_sock;
-void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num);
+int poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num);
void rpc_call_clean_epoll(struct protocol_stack *stack, struct wakeup_poll *wakeup);
int32_t rpc_call_msgcnt(struct protocol_stack *stack);
int32_t rpc_call_shadow_fd(struct protocol_stack *stack, int32_t fd, const struct sockaddr *addr, socklen_t addrlen);
@@ -89,6 +89,7 @@ int32_t rpc_call_ioctl(int fd, long cmd, void *argp);
int32_t rpc_call_replenish(struct protocol_stack *stack, struct lwip_sock *sock);
int32_t rpc_call_mbufpoolsize(struct protocol_stack *stack);
int32_t rpc_call_rpcpool_size(struct protocol_stack *stack);
+int32_t rpc_call_stack_exit(struct protocol_stack *stack);
static inline __attribute__((always_inline)) void rpc_call(lockless_queue *queue, struct rpc_msg *msg)
{
diff --git a/src/lstack/include/lstack_wrap.h b/src/lstack/include/lstack_wrap.h
index 80e5f3c..dab5222 100644
--- a/src/lstack/include/lstack_wrap.h
+++ b/src/lstack/include/lstack_wrap.h
@@ -14,6 +14,7 @@
#define _LSTACK_WRAP_H_
void wrap_api_init(void);
+void wrap_api_set_dummy(void);
#endif
--
2.27.0

View File

@ -0,0 +1,52 @@
From a72ed2effc718023316452866632f52058ae8218 Mon Sep 17 00:00:00 2001
From: compile_success <980965867@qq.com>
Date: Mon, 11 Dec 2023 13:24:33 +0000
Subject: [PATCH] fix EPOLLIN event error
---
src/lstack/api/lstack_epoll.c | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c
index 7825bed..7dbef9d 100644
--- a/src/lstack/api/lstack_epoll.c
+++ b/src/lstack/api/lstack_epoll.c
@@ -61,6 +61,17 @@ void add_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
if (wakeup == NULL || wakeup->type == WAKEUP_CLOSE || (event & sock->epoll_events) == 0) {
return;
}
+
+ if (!get_global_cfg_params()->stack_mode_rtc) {
+ if (event == EPOLLIN && !NETCONN_IS_DATAIN(sock) && !NETCONN_IS_ACCEPTIN(sock)) {
+ return;
+ }
+
+ if (event == EPOLLOUT && !NETCONN_IS_OUTIDLE(sock)) {
+ return;
+ }
+ }
+
sock->events |= (event == EPOLLERR) ? (EPOLLIN | EPOLLERR) : (event & sock->epoll_events);
if (list_is_null(&sock->event_list)) {
list_add_node(&wakeup->event_list, &sock->event_list);
@@ -88,7 +99,16 @@ void add_sock_event(struct lwip_sock *sock, uint32_t event)
void del_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
{
- sock->events &= ~event;
+ if (get_global_cfg_params()->stack_mode_rtc) {
+ sock->events &= ~event;
+ } else {
+ if ((event & EPOLLOUT) && !NETCONN_IS_OUTIDLE(sock)) {
+ sock->events &= ~EPOLLOUT;
+ }
+ if ((event & EPOLLIN) && !NETCONN_IS_DATAIN(sock) && !NETCONN_IS_ACCEPTIN(sock)) {
+ sock->events &= ~EPOLLIN;
+ }
+ }
if (sock->events == 0) {
list_del_node_null(&sock->event_list);
--
2.27.0

View File

@ -2,7 +2,7 @@
Name: gazelle
Version: 1.0.2
Release: 18
Release: 19
Summary: gazelle is a high performance user-mode stack
License: MulanPSL-2.0
URL: https://gitee.com/openeuler/gazelle
@ -94,6 +94,12 @@ Patch9077: 0077-log-optimize-lstack-log.patch
Patch9078: 0078-support-show-nic-offload-and-features.patch
Patch9079: 0079-Fixed-an-issue-where-no-packet-is-sent-or-received-w.patch
Patch9080: 0080-fix-example-print-error.patch
Patch9081: 0081-dfx-fix-kernel_events-stat.patch
Patch9082: 0082-add-keep-alive-info.patch
Patch9083: 0083-fix-close-can-t-exit.patch
Patch9084: 0084-mod-unix-time-stamp-to-local-time.patch
Patch9085: 0085-optimize-gazelle-exit-process.patch
Patch9086: 0086-fix-EPOLLIN-event-error.patch
%description
%{name} is a high performance user-mode stack.
@ -135,6 +141,14 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b
%config(noreplace) %{conf_path}/ltran.conf
%changelog
* Sat Dec 16 2023 yinbin6 <yinbin8@huawei.com> - 1.0.2-19
- fix EPOLLIN event error
- optimize gazelle exit process 1. close all fds 2. lstack thread exits, then gazelle process exits
- mod unix time stamp to local time
- fix close can't exit
- add keep-alive info
- dfx: fix kernel_events stat
* Sat Dec 9 2023 yinbin6 <yinbin8@huawei.com> - 1.0.2-18
- Fixed an issue where no packet is sent or received when UDP traffic is sent
- support show nic offload and features