From dd9a270402cf8fe7f2adf9d6bad3a3ac1a2b7289 Mon Sep 17 00:00:00 2001 From: jiangheng12 Date: Fri, 5 May 2023 16:42:35 +0800 Subject: [PATCH] clean code --- src/common/gazelle_dfx_msg.h | 1 - src/common/gazelle_opt.h | 3 +- src/common/gazelle_parse_config.c | 8 +- src/lstack/api/lstack_epoll.c | 1 - src/lstack/api/lstack_wrap.c | 85 +++--- src/lstack/core/lstack_cfg.c | 31 +- src/lstack/core/lstack_control_plane.c | 9 +- src/lstack/core/lstack_dpdk.c | 21 +- src/lstack/core/lstack_init.c | 63 ++-- src/lstack/core/lstack_lwip.c | 310 ++++++++++---------- src/lstack/core/lstack_protocol_stack.c | 42 +-- src/lstack/core/lstack_stack_stat.c | 10 +- src/lstack/include/lstack_dpdk.h | 3 +- src/lstack/include/lstack_ethdev.h | 4 +- src/lstack/include/lstack_lockless_queue.h | 6 +- src/lstack/include/lstack_protocol_stack.h | 2 +- src/lstack/include/lstack_vdev.h | 3 +- src/lstack/netif/lstack_ethdev.c | 318 +++++++++++---------- src/lstack/netif/lstack_vdev.c | 16 +- src/ltran/ltran_dfx.c | 10 +- src/ltran/ltran_monitor.c | 2 +- src/ltran/ltran_param.c | 2 +- src/ltran/main.c | 4 +- 23 files changed, 489 insertions(+), 465 deletions(-) diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h index e4da687..a89b82c 100644 --- a/src/common/gazelle_dfx_msg.h +++ b/src/common/gazelle_dfx_msg.h @@ -15,7 +15,6 @@ #include #include -#include #define GAZELLE_CLIENT_NUM_MIN 1 #define GAZELLE_LOG_LEVEL_MAX 10 diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h index fe0483b..7b6aa19 100644 --- a/src/common/gazelle_opt.h +++ b/src/common/gazelle_opt.h @@ -47,7 +47,8 @@ #define TCP_CONN_COUNT 1500 #define MBUF_COUNT_PER_CONN 170 -#define RXTX_NB_MBUF_DEFAULT (MBUF_COUNT_PER_CONN * TCP_CONN_COUNT) /* mbuf per connect * connect num. size of mbuf is 2536 Byte */ +/* mbuf per connect * connect num. size of mbuf is 2536 Byte */ +#define RXTX_NB_MBUF_DEFAULT (MBUF_COUNT_PER_CONN * TCP_CONN_COUNT) #define STACK_THREAD_DEFAULT 4 #define STACK_NIC_READ_DEFAULT 128 diff --git a/src/common/gazelle_parse_config.c b/src/common/gazelle_parse_config.c index ce6a3f8..42e12b5 100644 --- a/src/common/gazelle_parse_config.c +++ b/src/common/gazelle_parse_config.c @@ -55,7 +55,7 @@ int32_t separate_str_to_array(char *args, uint32_t *array, int32_t array_size, i return -1; } errno = 0; - /* prefix 0x,0X indicate hexdecimal */ + /* 2: prefix 0x,0X indicate hexdecimal */ if (strncmp(args, "0x", 2) == 0 || strncmp(args, "0X", 2) == 0) { idx = strtol(args, &end, 16); /* 16: hexdecimal */ } else { @@ -116,13 +116,15 @@ int32_t filename_check(const char* args) } if (strlen(args) <= 0 || strlen(args) > GAZELLE_SOCK_FILENAME_MAXLEN - 1) { - COMMON_ERR("socket_filename_check: invalid unix sock name %s, filename exceeds the limit %d.\n", args, GAZELLE_SOCK_FILENAME_MAXLEN); + COMMON_ERR("socket_filename_check: invalid unix sock name %s, filename exceeds the limit %d.\n", + args, GAZELLE_SOCK_FILENAME_MAXLEN); return 1; } char* sensitive_chars = strpbrk(args, "|;&$><`\\!\n"); if (sensitive_chars != NULL) { - COMMON_ERR("socket_filename_check: invalid unix sock name %s, filename contains sensitive characters.\n", args); + COMMON_ERR("socket_filename_check: invalid unix sock name %s, filename contains sensitive characters.\n", + args); return 1; } diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c index d4b4be7..06bd71b 100644 --- a/src/lstack/api/lstack_epoll.c +++ b/src/lstack/api/lstack_epoll.c @@ -320,7 +320,6 @@ int32_t lstack_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct epoll_even return posix_api->epoll_ctl_fn(epfd, op, fd, event); } - struct wakeup_poll *wakeup = epoll_sock->wakeup; struct lwip_sock *sock = get_socket(fd); if (sock == NULL) { diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c index 1b12822..98632c0 100644 --- a/src/lstack/api/lstack_wrap.c +++ b/src/lstack/api/lstack_wrap.c @@ -199,7 +199,7 @@ static int get_addr(struct sockaddr_in *sin, char *interface) memset_s(&ifr, sizeof(ifr), 0, sizeof(ifr)); snprintf_s(ifr.ifr_name, sizeof(ifr.ifr_name), (sizeof(ifr.ifr_name) - 1), "%s", interface); - if(posix_api->ioctl_fn(sockfd, SIOCGIFADDR, &ifr) < 0){ + if (posix_api->ioctl_fn(sockfd, SIOCGIFADDR, &ifr) < 0) { posix_api->close_fn(sockfd); return -1; } @@ -243,11 +243,15 @@ bool is_dst_ip_localhost(const struct sockaddr *addr) struct sockaddr_in* sin = malloc(sizeof(struct sockaddr_in)); while (getdelim(&line, &linel, '\n', ifh) > 0) { - if (linenum++ < 2) continue; + /* 2: skip the first two lines, which are not nic name */ + if (linenum++ < 2) { + continue; + } p = line; - while (isspace(*p)) - ++p; + while (isspace(*p)) { + ++p; + } int n = strcspn(p, ": \t"); char interface[20] = {0}; /* 20: nic name len */ @@ -256,7 +260,7 @@ bool is_dst_ip_localhost(const struct sockaddr *addr) memset_s(sin, sizeof(struct sockaddr_in), 0, sizeof(struct sockaddr_in)); int ret = get_addr(sin, interface); if (ret == 0) { - if(sin->sin_addr.s_addr == servaddr->sin_addr.s_addr){ + if (sin->sin_addr.s_addr == servaddr->sin_addr.s_addr) { return 1; } } @@ -289,13 +293,14 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name int32_t ret = 0; char listen_ring_name[RING_NAME_LEN]; int remote_port = htons(((struct sockaddr_in *)name)->sin_port); - snprintf_s(listen_ring_name, sizeof(listen_ring_name), sizeof(listen_ring_name) - 1, "listen_rx_ring_%d", remote_port); + snprintf_s(listen_ring_name, sizeof(listen_ring_name), sizeof(listen_ring_name) - 1, + "listen_rx_ring_%d", remote_port); if (is_dst_ip_localhost(name) && rte_ring_lookup(listen_ring_name) == NULL) { ret = posix_api->connect_fn(s, name, namelen); - SET_CONN_TYPE_HOST(sock->conn); + SET_CONN_TYPE_HOST(sock->conn); } else { ret = rpc_call_connect(s, name, namelen); - SET_CONN_TYPE_LIBOS(sock->conn); + SET_CONN_TYPE_LIBOS(sock->conn); } return ret; @@ -435,26 +440,26 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len) static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt) { - struct lwip_sock *sock = NULL; - if (select_path(s, &sock) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) != PATH_LWIP) { return posix_api->readv_fn(s, iov, iovcnt); - } - - struct msghdr msg; - - msg.msg_name = NULL; - msg.msg_namelen = 0; - msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov); - msg.msg_iovlen = iovcnt; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = 0; - ssize_t result = recvmsg_from_stack(s, &msg, 0); - if(result == -1 && errno == EAGAIN){ + } + + struct msghdr msg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov); + msg.msg_iovlen = iovcnt; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + ssize_t result = recvmsg_from_stack(s, &msg, 0); + if (result == -1 && errno == EAGAIN) { errno = 0; - return 0; - } - return result; + return 0; + } + return result; } static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32_t flags) @@ -479,21 +484,21 @@ static inline ssize_t do_write(int32_t s, const void *mem, size_t size) static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt) { - struct lwip_sock *sock = NULL; - if (select_path(s, &sock) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) != PATH_LWIP) { return posix_api->writev_fn(s, iov, iovcnt); - } - - struct msghdr msg; - - msg.msg_name = NULL; - msg.msg_namelen = 0; - msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov); - msg.msg_iovlen = iovcnt; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = 0; - return sendmsg_to_stack(sock, s, &msg, 0); + } + + struct msghdr msg; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov); + msg.msg_iovlen = iovcnt; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + return sendmsg_to_stack(sock, s, &msg, 0); } static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flags) diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c index cdb0200..db35110 100644 --- a/src/lstack/core/lstack_cfg.c +++ b/src/lstack/core/lstack_cfg.c @@ -75,19 +75,19 @@ static int32_t parse_tuple_filter(void); const config_setting_t *_config_arg = NULL; \ _config_arg = config_lookup(&g_config, _arg_string); \ if (_config_arg == NULL) { \ - _arg = _default_val; \ - _ret = 0; \ + (_arg) = (_default_val); \ + (_ret) = 0; \ break; \ } \ int32_t _val = config_setting_get_int(_config_arg); \ - if (_val < _min_val || _val > _max_val) { \ + if (_val < (_min_val) || _val > (_max_val)) { \ LSTACK_PRE_LOG(LSTACK_ERR, "cfg %s %d invaild, range is [%d, %d].\n", \ - _arg_string, _val, _min_val, _max_val); \ - _ret = -EINVAL; \ + (_arg_string), _val, (_min_val), (_max_val)); \ + (_ret) = -EINVAL; \ break; \ } \ - _arg = _val; \ - _ret = 0; \ + (_arg) = _val; \ + (_ret) = 0; \ } while (0) struct config_vector_t { @@ -342,7 +342,8 @@ static int32_t parse_stack_cpu_number(void) } char *tmp_arg_send = strdup(args); - int32_t send_cpu_cnt = separate_str_to_array(tmp_arg_send, g_config_params.send_cpus, CFG_MAX_CPUS, CFG_MAX_CPUS); + int32_t send_cpu_cnt = separate_str_to_array(tmp_arg_send, g_config_params.send_cpus, + CFG_MAX_CPUS, CFG_MAX_CPUS); free(tmp_arg_send); // recv_num_cpus @@ -368,7 +369,8 @@ static int32_t parse_stack_cpu_number(void) } char *tmp_arg_recv = strdup(args); - int32_t recv_cpu_cnt = separate_str_to_array(tmp_arg_recv, g_config_params.recv_cpus, CFG_MAX_CPUS, CFG_MAX_CPUS); + int32_t recv_cpu_cnt = separate_str_to_array(tmp_arg_recv, g_config_params.recv_cpus, + CFG_MAX_CPUS, CFG_MAX_CPUS); free(tmp_arg_recv); if (send_cpu_cnt <= 0 || send_cpu_cnt > CFG_MAX_CPUS / 2 || send_cpu_cnt != recv_cpu_cnt) { @@ -444,7 +446,7 @@ int32_t init_stack_numa_cpuset(struct protocol_stack *stack) for (int32_t idx = 0; idx < cfg->num_cpu; ++idx) { if (!cfg->seperate_send_recv) { CPU_SET(cfg->cpus[idx], &stack_cpuset); - }else { + } else { CPU_SET(cfg->send_cpus[idx], &stack_cpuset); CPU_SET(cfg->recv_cpus[idx], &stack_cpuset); } @@ -494,7 +496,6 @@ static int32_t gazelle_parse_socket_mem(const char *arg, struct secondary_attach int32_t count = separate_str_to_array(socket_mem, sec_attach_arg->socket_per_size, GAZELLE_MAX_NUMA_NODES, INT32_MAX); - if (count < 0) { return -1; } @@ -726,7 +727,7 @@ static int32_t parse_dpdk_args(void) g_config_params.dpdk_argv[start_index + i] = p; const char *primary = "primary"; - if(strcmp(p, primary) == 0){ + if (strcmp(p, primary) == 0) { struct cfg_params *global_params = get_global_cfg_params(); global_params->is_primary = 1; } @@ -937,10 +938,8 @@ static int32_t parse_unix_prefix(void) } unix_prefix = config_lookup(&g_config, "unix_prefix"); - if (unix_prefix) { args = config_setting_get_string(unix_prefix); - if (filename_check(args)) { return -EINVAL; } @@ -979,7 +978,7 @@ static int32_t parse_num_process(void) num_process = config_lookup(&g_config, "num_process"); if (num_process == NULL) { g_config_params.num_process = 1; - }else { + } else { g_config_params.num_process = (uint8_t)config_setting_get_int(num_process); } @@ -1022,7 +1021,7 @@ static int parse_process_index(void) if (process_idx == NULL) { if (g_config_params.num_process == 1) { g_config_params.process_idx = 0; - }else { + } else { return -EINVAL; } } else { diff --git a/src/lstack/core/lstack_control_plane.c b/src/lstack/core/lstack_control_plane.c index e705cae..be156dc 100644 --- a/src/lstack/core/lstack_control_plane.c +++ b/src/lstack/core/lstack_control_plane.c @@ -466,14 +466,15 @@ void control_fd_close(void) struct cfg_params *global_params = get_global_cfg_params(); if (!global_params->use_ltran) { - int ret = unlink(global_params->unix_socket_filename); - if (ret == -1) { + int ret = unlink(global_params->unix_socket_filename); + if (ret == -1) { LSTACK_LOG(ERR, LSTACK, "unlink failed, just skip it\n"); - } + } } } -void delete_primary_path(void) { +void delete_primary_path(void) +{ if (!remove(GAZELLE_PRIMARY_START_PATH)) { LSTACK_LOG(ERR, LSTACK, "delete %s failed\n", GAZELLE_PRIMARY_START_PATH); } diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c index e386dfc..79d13ee 100644 --- a/src/lstack/core/lstack_dpdk.c +++ b/src/lstack/core/lstack_dpdk.c @@ -38,12 +38,12 @@ #include "lstack_log.h" #include "dpdk_common.h" -#include "lstack_dpdk.h" #include "lstack_lockless_queue.h" #include "lstack_protocol_stack.h" #include "lstack_thread_rpc.h" #include "lstack_lwip.h" #include "lstack_cfg.h" +#include "lstack_dpdk.h" struct eth_params { uint16_t port_id; @@ -116,7 +116,7 @@ int32_t dpdk_eal_init(void) ret = 0; } else { LSTACK_PRE_LOG(LSTACK_ERR, "rte_eal_init failed init, rte_errno %d\n", rte_errno); - return ret; + return ret; } } else { LSTACK_PRE_LOG(LSTACK_INFO, "dpdk_eal_init success\n"); @@ -155,7 +155,8 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, return pool; } -static struct rte_mempool* get_pktmbuf_mempool(const char *name, uint16_t queue_id){ +static struct rte_mempool* get_pktmbuf_mempool(const char *name, uint16_t queue_id) +{ int32_t ret; char pool_name[PATH_MAX]; struct rte_mempool *pool; @@ -169,9 +170,7 @@ static struct rte_mempool* get_pktmbuf_mempool(const char *name, uint16_t queue_ LSTACK_LOG(ERR, LSTACK, "look up %s pool rte_err=%d\n", pool_name, rte_errno); } - // rte_mempool_dump(stdout, pool) ; return pool; - } static struct reg_ring_msg *create_reg_mempool(const char *name, uint16_t queue_id) @@ -220,7 +219,8 @@ struct rte_ring *create_ring(const char *name, uint32_t count, uint32_t flags, i char ring_name[RTE_RING_NAMESIZE] = {0}; struct rte_ring *ring; - int32_t ret = snprintf_s(ring_name, sizeof(ring_name), RTE_RING_NAMESIZE - 1, "%s_%d_%d", name, get_global_cfg_params()->process_idx, queue_id); + int32_t ret = snprintf_s(ring_name, sizeof(ring_name), RTE_RING_NAMESIZE - 1, + "%s_%d_%d", name, get_global_cfg_params()->process_idx, queue_id); if (ret < 0) { return NULL; } @@ -521,10 +521,10 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i struct rte_mempool *rxtx_pktmbuf_pool = get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx]; uint16_t socket_id = 0; - struct cfg_params * cfg = get_global_cfg_params(); + struct cfg_params *cfg = get_global_cfg_params(); if (!cfg->use_ltran && cfg->num_process == 1) { socket_id = numa_node_of_cpu(cfg->cpus[idx]); - }else { + } else { socket_id = cfg->process_numa[idx]; } ret = rte_eth_rx_queue_setup(eth_params->port_id, idx, eth_params->nb_rx_desc, socket_id, @@ -568,8 +568,6 @@ int32_t dpdk_ethdev_start(void) int32_t dpdk_init_lstack_kni(void) { struct protocol_stack_group *stack_group = get_protocol_stack_group(); - - stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0); if (stack_group->kni_pktmbuf_pool == NULL) { return -1; @@ -651,7 +649,8 @@ void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id) if (len < 0) { return; } - if (len != rte_eth_xstats_get_names_by_id(port_id, (struct rte_eth_xstat_name *)dfx->data.nic_xstats.xstats_name, len, NULL)) { + if (len != rte_eth_xstats_get_names_by_id(port_id, + (struct rte_eth_xstat_name *)dfx->data.nic_xstats.xstats_name, len, NULL)) { dfx->data.nic_xstats.len = -1; return; } diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c index f17e4d2..4fbbc14 100644 --- a/src/lstack/core/lstack_init.c +++ b/src/lstack/core/lstack_init.c @@ -98,18 +98,20 @@ static int32_t preload_info_init(void) return 0; } -static void check_process_start(void) { +static void check_process_start(void) +{ if (get_global_cfg_params()->is_primary) { return; } while (!fopen(GAZELLE_PRIMARY_START_PATH, "r")) { - printf("please make sure the primary process start already!\n"); - sleep(1); + LSTACK_LOG(INFO, LSTACK, "please make sure the primary process start already!\n"); + sleep(1); } } -static int32_t set_process_start_flag(void) { +static int32_t set_process_start_flag(void) +{ if (!get_global_cfg_params()->is_primary) { return 0; } @@ -118,7 +120,7 @@ static int32_t set_process_start_flag(void) { fp = fopen(GAZELLE_PRIMARY_START_PATH, "w"); if (fp == NULL) { LSTACK_PRE_LOG(LSTACK_ERR, "set primary proceaa start flag failed!\n"); - return -1; + return -1; } (void)fclose(fp); return 0; @@ -199,9 +201,7 @@ static void create_control_thread(void) pthread_t tid; if (use_ltran()) { - /* - * The function call here should be in strict order. - */ + /* The function call here should be in strict order. */ dpdk_skip_nic_init(); if (control_init_client(false) != 0) { LSTACK_EXIT(1, "control_init_client failed\n"); @@ -235,8 +235,7 @@ static void gazelle_signal_init(void) LSTACK_EXIT(1, "signal SIGPIPE SIG_IGN\n"); } - /* - * register core sig handler func to dumped stack */ + /* register core sig handler func to dumped stack */ lstack_signal_init(); } @@ -264,15 +263,16 @@ static void set_kni_ip_mac() } set_ifr.ifr_hwaddr.sa_family = ARPHRD_ETHER; + /* 6: mac addr len */ for (int i = 0; i < 6; i++) { set_ifr.ifr_hwaddr.sa_data[i] = cfg->mac_addr[i]; } if (posix_api->ioctl_fn(fd, SIOCSIFHWADDR, &set_ifr) < 0) { LSTACK_LOG(ERR, LSTACK, "set kni macaddr=%hhx:%hhx:%hhx:%hhx:%hhx:%hhx fail\n", - cfg->mac_addr[0], cfg->mac_addr[1], - cfg->mac_addr[2], cfg->mac_addr[3], - cfg->mac_addr[4], cfg->mac_addr[5]); + cfg->mac_addr[0], cfg->mac_addr[1], + cfg->mac_addr[2], cfg->mac_addr[3], + cfg->mac_addr[4], cfg->mac_addr[5]); } if (posix_api->ioctl_fn(fd, SIOCGIFFLAGS, &set_ifr) < 0) { @@ -280,7 +280,7 @@ static void set_kni_ip_mac() } set_ifr.ifr_flags |= (IFF_RUNNING | IFF_UP); - if (posix_api->ioctl_fn(fd, SIOCSIFFLAGS, &set_ifr) < 0){ + if (posix_api->ioctl_fn(fd, SIOCSIFFLAGS, &set_ifr) < 0) { LSTACK_LOG(ERR, LSTACK, "set kni state fail\n"); } @@ -289,16 +289,14 @@ static void set_kni_ip_mac() __attribute__((constructor)) void gazelle_network_init(void) { - /* - * Init POSXI API and prelog */ + /* Init POSXI API and prelog */ lstack_prelog_init("LSTACK"); if (posix_api_init() != 0) { LSTACK_PRE_LOG(LSTACK_ERR, "posix_api_init failed\n"); LSTACK_EXIT(1, "failed\n"); } - /* - * Init LD_PRELOAD */ + /* Init LD_PRELOAD */ if (preload_info_init() < 0) { return; } @@ -306,35 +304,29 @@ __attribute__((constructor)) void gazelle_network_init(void) return; } - /* - * Read configure from lstack.cfg */ + /* Read configure from lstack.cfg */ if (cfg_init() != 0) { LSTACK_PRE_LOG(LSTACK_ERR, "cfg_init failed\n"); LSTACK_EXIT(1, "cfg_init failed\n"); } LSTACK_PRE_LOG(LSTACK_INFO, "cfg_init success\n"); - /* - * check primary process start */ + /* check primary process start */ check_process_start(); - /* - * check conflict */ + /* check conflict */ if (check_process_conflict() < 0) { LSTACK_PRE_LOG(LSTACK_INFO, "Have another same primary process. WARNING: Posix API will use kernel mode!\n"); return; } - /** - * check lstack num, and get process idx - */ + /* check lstack num, and get process idx */ if (check_params_from_primary() < 0) { LSTACK_PRE_LOG(LSTACK_ERR, "lstack num error, not same to primary process!\n"); LSTACK_EXIT(1, "lstack num error, not same to primary process!\n"); } - /* - * save initial affinity */ + /* save initial affinity */ if (!get_global_cfg_params()->main_thread_affinity) { if (thread_affinity_default() < 0) { LSTACK_PRE_LOG(LSTACK_ERR, "pthread_getaffinity_np failed\n"); @@ -342,17 +334,13 @@ __attribute__((constructor)) void gazelle_network_init(void) } } - // @todo, check process 2 dumped, resorce need to release. - gazelle_signal_init(); - /* - * Init control plane and dpdk init */ + /* Init control plane and dpdk init */ create_control_thread(); dpdk_restore_pci(); - /* - * cancel the core binding from DPDK initialization */ + /* cancel the core binding from DPDK initialization */ if (!get_global_cfg_params()->main_thread_affinity) { if (thread_affinity_default() < 0) { LSTACK_EXIT(1, "pthread_setaffinity_np failed\n"); @@ -366,16 +354,13 @@ __attribute__((constructor)) void gazelle_network_init(void) LSTACK_EXIT(1, "init_protocol_stack failed\n"); } - /* - * nic */ if (!use_ltran()) { if (init_dpdk_ethdev() != 0) { LSTACK_EXIT(1, "init_dpdk_ethdev failed\n"); } } - /* - * lwip initialization */ + /* lwip initialization */ lwip_sock_init(); /* wait stack thread and kernel_event thread init finish */ diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index 0535428..b6c5813 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -33,8 +33,8 @@ #include "posix/lstack_epoll.h" #include "lstack_thread_rpc.h" #include "dpdk_common.h" -#include "lstack_lwip.h" #include "lstack_cfg.h" +#include "lstack_lwip.h" static void free_ring_pbuf(struct rte_ring *ring) { @@ -263,7 +263,7 @@ struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8 return NULL; } if (pbuf->allow_in == 1) { - __sync_fetch_and_sub(&pbuf->allow_in, 1); + __sync_fetch_and_sub(&pbuf->allow_in, 1); } pthread_spin_unlock(&pbuf->pbuf_lock); @@ -300,7 +300,7 @@ struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8 pbuf->head = 1; return NULL; } - if(pbuf->allow_in == 1){ + if (pbuf->allow_in == 1) { __sync_fetch_and_sub(&pbuf->allow_in, 1); } pthread_spin_unlock(&pbuf->pbuf_lock); @@ -374,6 +374,9 @@ static ssize_t do_app_write(struct pbuf *pbufs[], void *buf, size_t len, uint32_ static inline ssize_t app_direct_write(struct protocol_stack *stack, struct lwip_sock *sock, void *buf, size_t len, uint32_t write_num) { + if (write_num == 0) { + return 0; + } struct pbuf **pbufs = (struct pbuf **)malloc(write_num * sizeof(struct pbuf *)); if (pbufs == NULL) { return 0; @@ -410,6 +413,9 @@ static inline ssize_t app_direct_write(struct protocol_stack *stack, struct lwip static inline ssize_t app_direct_attach(struct protocol_stack *stack, struct pbuf *attach_pbuf, void *buf, size_t len, uint32_t write_num) { + if (write_num == 0) { + return 0; + } struct pbuf **pbufs = (struct pbuf **)malloc(write_num * sizeof(struct pbuf *)); if (pbufs == NULL) { return 0; @@ -469,7 +475,7 @@ static inline struct pbuf *gazelle_ring_readlast(struct rte_ring *r) __rte_ring_dequeue_elems(r, last, (void **)&last_pbuf, sizeof(void *), 1); if (pthread_spin_trylock(&last_pbuf->pbuf_lock) != 0) { - return NULL; + return NULL; } if (last_pbuf->allow_in != 1) { pthread_spin_unlock(&last_pbuf->pbuf_lock); @@ -587,7 +593,8 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) /* send_ring have idle */ if (get_global_cfg_params()->expand_send_ring) { - send_len += (write_num <= write_avail) ? app_buff_write(sock, (char *)buf + send_len, len - send_len, write_num) : + send_len += (write_num <= write_avail) ? + app_buff_write(sock, (char *)buf + send_len, len - send_len, write_num) : app_direct_write(stack, sock, (char *)buf + send_len, len - send_len, write_num); } else { if (write_num > write_avail) { @@ -662,7 +669,7 @@ void stack_send(struct rpc_msg *msg) rpc_msg_free(msg); return; } else { - if(__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0){ + if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0) { rpc_call(&stack->rpc_queue, msg); __sync_fetch_and_add(&sock->call_num, 1); } else { @@ -741,7 +748,7 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags) sock->stack->stats.read_lwip_cnt += read_count; if (recv_len == 0) { - GAZELLE_RETURN(EAGAIN); + GAZELLE_RETURN(EAGAIN); } return recv_len; } @@ -797,11 +804,10 @@ ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags) static inline void notice_stack_send(struct lwip_sock *sock, int32_t fd, int32_t len, int32_t flags) { - if(__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) >= 2){ - ;; - } else { + // 2: call_num >= 2, don't need add new rpc send + if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) < 2) { while (rpc_call_send(fd, NULL, len, flags) < 0) { - usleep(1000); // wait 1ms to exec again + usleep(1000); // 1000: wait 1ms to exec again LSTACK_LOG(INFO, LSTACK, "rpc_call_send failed, try again\n"); } __sync_fetch_and_add(&sock->call_num, 1); @@ -838,10 +844,8 @@ ssize_t gazelle_same_node_ring_recv(struct lwip_sock *sock, const void *buf, siz act_len = -1; goto END; } - act_len = cur_end - index + 1; act_len = RTE_MIN(act_len, len); - if ((index & SAME_NODE_RING_MASK) + act_len > SAME_NODE_RING_LEN) { size_t act_len1 = SAME_NODE_RING_LEN - (index & SAME_NODE_RING_MASK); size_t act_len2 = act_len - act_len1; @@ -898,7 +902,8 @@ ssize_t gazelle_same_node_ring_send(struct lwip_sock *sock, const void *buf, siz PER_THREAD uint16_t stack_sock_num[GAZELLE_MAX_STACK_NUM] = {0}; PER_THREAD uint16_t max_sock_stack = 0; -static inline void thread_bind_stack(struct lwip_sock *sock) { +static inline void thread_bind_stack(struct lwip_sock *sock) +{ if (likely(sock->already_bind_numa || !sock->stack)) { return; } @@ -906,7 +911,7 @@ static inline void thread_bind_stack(struct lwip_sock *sock) { stack_sock_num[sock->stack->stack_idx]++; if (stack_sock_num[sock->stack->stack_idx] > max_sock_stack) { max_sock_stack = stack_sock_num[sock->stack->stack_idx]; - bind_to_stack_numa(sock->stack); + bind_to_stack_numa(sock->stack); } } @@ -960,7 +965,7 @@ ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr buflen += ret; if (ret < message->msg_iov[i].iov_len) { - break; + break; } } @@ -1104,7 +1109,6 @@ void read_recv_list(struct protocol_stack *stack, uint32_t max_num) ssize_t len = lwip_recv(sock->conn->socket, NULL, 0, 0); if (len == 0) { - /* FIXME: should use POLLRDHUP, when connection be closed. lwip event-callback no POLLRDHUP */ sock->errevent = 1; add_sock_event(sock, EPOLLERR); } else if (len > 0) { @@ -1125,7 +1129,7 @@ void gazelle_connected_callback(struct netconn *conn) return; } - if (sock->wakeup != NULL && sock->wakeup->epollfd > 0){ + if (sock->wakeup != NULL && sock->wakeup->epollfd > 0) { posix_api->epoll_ctl_fn(sock->wakeup->epollfd, EPOLL_CTL_DEL, fd, NULL); } @@ -1186,7 +1190,7 @@ static inline void clone_lwip_socket_opt(struct lwip_sock *dst_sock, struct lwip int32_t gazelle_socket(int domain, int type, int protocol) { - if (((type & SOCK_TYPE_MASK) & ~SOCK_STREAM) != 0){ + if (((type & SOCK_TYPE_MASK) & ~SOCK_STREAM) != 0) { LSTACK_LOG(ERR, LSTACK, "sock type error:%d, only support SOCK_STREAM \n", type); return -1; } @@ -1337,125 +1341,125 @@ void stack_recvlist_count(struct rpc_msg *msg) void netif_poll(struct netif *netif) { - struct tcp_pcb *pcb = NULL; - struct tcp_pcb_listen *pcbl = NULL; + struct tcp_pcb *pcb = NULL; + struct tcp_pcb_listen *pcbl = NULL; - for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) { + for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) { #define NETIF_POLL_READ_COUNT 32 - struct pbuf *pbufs[NETIF_POLL_READ_COUNT]; - int ret; - - if (pcb->client_rx_ring != NULL) { - ret = rte_ring_sc_dequeue_burst(pcb->client_rx_ring, (void **)pbufs, NETIF_POLL_READ_COUNT, NULL); - for (int i = 0; i < ret; i++) { - if (ip_input(pbufs[i], netif) != 0) { - LSTACK_LOG(INFO, LSTACK, "netif_poll: ip_input return err\n"); - pbuf_free(pbufs[i]); + struct pbuf *pbufs[NETIF_POLL_READ_COUNT]; + int ret; + + if (pcb->client_rx_ring != NULL) { + ret = rte_ring_sc_dequeue_burst(pcb->client_rx_ring, (void **)pbufs, NETIF_POLL_READ_COUNT, NULL); + for (int i = 0; i < ret; i++) { + if (ip_input(pbufs[i], netif) != 0) { + LSTACK_LOG(INFO, LSTACK, "ip_input return err\n"); + pbuf_free(pbufs[i]); + } + } } - } - } - } - for (pcbl = tcp_listen_pcbs.listen_pcbs; pcbl != NULL; pcbl = pcbl->next) { - if (pcbl->listen_rx_ring != NULL) { - struct pbuf *pbuf; - if (rte_ring_sc_dequeue(pcbl->listen_rx_ring, (void **)&pbuf) == 0) { - if (ip_input(pbuf, netif) != ERR_OK) { - pbuf_free(pbuf); + } + for (pcbl = tcp_listen_pcbs.listen_pcbs; pcbl != NULL; pcbl = pcbl->next) { + if (pcbl->listen_rx_ring != NULL) { + struct pbuf *pbuf; + if (rte_ring_sc_dequeue(pcbl->listen_rx_ring, (void **)&pbuf) == 0) { + if (ip_input(pbuf, netif) != ERR_OK) { + pbuf_free(pbuf); + } + } } - } } - } } /* processes on same node handshake packet use this function */ err_t netif_loop_output(struct netif *netif, struct pbuf *p) { - struct tcp_pcb *pcb = p->pcb; - struct pbuf *head = NULL; - - if (pcb == NULL || pcb->client_tx_ring == NULL) { - LSTACK_LOG(ERR, LSTACK, "netif_loop_output: pcb is null\n"); - return ERR_ARG; - } - - if (p->next != NULL) { - LSTACK_LOG(ERR, LSTACK, "netif_loop_output: not support chained pbuf\n"); - return ERR_ARG; - } - - struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)((char *)p->payload + sizeof(struct ip_hdr)); - uint8_t flags = TCPH_FLAGS(tcp_hdr); - - head = pbuf_alloc(0, p->len, PBUF_RAM); - if (head == NULL) { - LSTACK_LOG(ERR, LSTACK, "netif_loop_output: pbuf_alloc failed\n"); - return ERR_MEM; - } - head->ol_flags = p->ol_flags; - memcpy_s(head->payload, head->len, p->payload, p->len); - - if ((flags & TCP_SYN) && !(flags & TCP_ACK)) { - /* SYN packet, send to listen_ring */ - char ring_name[RING_NAME_LEN] = {0}; - snprintf_s(ring_name, sizeof(ring_name), sizeof(ring_name) - 1, "listen_rx_ring_%d", pcb->remote_port); - struct rte_ring *ring = rte_ring_lookup(ring_name); - if (ring == NULL) { - LSTACK_LOG(INFO, LSTACK, "netif_loop_output: cant find listen_rx_ring %d\n", pcb->remote_port); - pbuf_free(head); + struct tcp_pcb *pcb = p->pcb; + struct pbuf *head = NULL; + + if (pcb == NULL || pcb->client_tx_ring == NULL) { + LSTACK_LOG(ERR, LSTACK, "pcb is null\n"); + return ERR_ARG; + } + + if (p->next != NULL) { + LSTACK_LOG(ERR, LSTACK, "netif_loop_output: not support chained pbuf\n"); + return ERR_ARG; + } + + struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)((char *)p->payload + sizeof(struct ip_hdr)); + uint8_t flags = TCPH_FLAGS(tcp_hdr); + + head = pbuf_alloc(0, p->len, PBUF_RAM); + if (head == NULL) { + LSTACK_LOG(ERR, LSTACK, "netif_loop_output: pbuf_alloc failed\n"); + return ERR_MEM; + } + head->ol_flags = p->ol_flags; + memcpy_s(head->payload, head->len, p->payload, p->len); + + if ((flags & TCP_SYN) && !(flags & TCP_ACK)) { + /* SYN packet, send to listen_ring */ + char ring_name[RING_NAME_LEN] = {0}; + snprintf_s(ring_name, sizeof(ring_name), sizeof(ring_name) - 1, "listen_rx_ring_%d", pcb->remote_port); + struct rte_ring *ring = rte_ring_lookup(ring_name); + if (ring == NULL) { + LSTACK_LOG(INFO, LSTACK, "netif_loop_output: cant find listen_rx_ring %d\n", pcb->remote_port); + pbuf_free(head); + } else { + if (rte_ring_mp_enqueue(ring, head) != 0) { + LSTACK_LOG(INFO, LSTACK, "enqueue sync packet failed\n"); + pbuf_free(head); + } + } } else { - if (rte_ring_mp_enqueue(ring, head) != 0) { - LSTACK_LOG(INFO, LSTACK, "enqueue sync packet failed\n"); + /* send other type packet to tx_ring */ + if (rte_ring_sp_enqueue(pcb->client_tx_ring, head) != 0) { + LSTACK_LOG(INFO, LSTACK, "client tx ring full\n"); pbuf_free(head); } } - } else { - /* send other type packet to tx_ring */ - if (rte_ring_sp_enqueue(pcb->client_tx_ring, head) != 0) { - LSTACK_LOG(INFO, LSTACK, "client tx ring full\n"); - pbuf_free(head); - } - } - return ERR_OK; + return ERR_OK; } err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock) { - char name[RING_NAME_LEN]; - snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_rx_%u", pcb->remote_port); - if ((nsock->same_node_tx_ring_mz = rte_memzone_lookup(name)) == NULL) { - LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); - return -1; - } else { - LSTACK_LOG(INFO, LSTACK, "lookup %s success\n", name); - } - nsock->same_node_tx_ring = (struct same_node_ring *)nsock->same_node_tx_ring_mz->addr; - - snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_buf_rx_%u", pcb->remote_port); - if ((nsock->same_node_tx_ring->mz = rte_memzone_lookup(name)) == NULL) { - LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); - return -1; - } - - snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_tx_%u", pcb->remote_port); - if ((nsock->same_node_rx_ring_mz = rte_memzone_lookup(name)) == NULL) { - LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); - return -1; - } else { - LSTACK_LOG(INFO, LSTACK, "lookup %s success\n", name); - } - nsock->same_node_rx_ring = (struct same_node_ring *)nsock->same_node_rx_ring_mz->addr; - - snprintf_s(name, sizeof(name), sizeof(name) - 1,"rte_mz_buf_tx_%u", pcb->remote_port); - if ((nsock->same_node_rx_ring->mz = rte_memzone_lookup(name)) == NULL) { - LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); - return -1; - } - - /* rcvlink init in alloc_socket() */ - /* remove from g_rcv_process_list in free_socket */ - list_add_node(&nsock->stack->same_node_recv_list, &nsock->recv_list); - return 0; + char name[RING_NAME_LEN]; + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_rx_%u", pcb->remote_port); + if ((nsock->same_node_tx_ring_mz = rte_memzone_lookup(name)) == NULL) { + LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); + return -1; + } else { + LSTACK_LOG(INFO, LSTACK, "lookup %s success\n", name); + } + nsock->same_node_tx_ring = (struct same_node_ring *)nsock->same_node_tx_ring_mz->addr; + + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_buf_rx_%u", pcb->remote_port); + if ((nsock->same_node_tx_ring->mz = rte_memzone_lookup(name)) == NULL) { + LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); + return -1; + } + + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_tx_%u", pcb->remote_port); + if ((nsock->same_node_rx_ring_mz = rte_memzone_lookup(name)) == NULL) { + LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); + return -1; + } else { + LSTACK_LOG(INFO, LSTACK, "lookup %s success\n", name); + } + nsock->same_node_rx_ring = (struct same_node_ring *)nsock->same_node_rx_ring_mz->addr; + + snprintf_s(name, sizeof(name), sizeof(name) - 1,"rte_mz_buf_tx_%u", pcb->remote_port); + if ((nsock->same_node_rx_ring->mz = rte_memzone_lookup(name)) == NULL) { + LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); + return -1; + } + + /* rcvlink init in alloc_socket() */ + /* remove from g_rcv_process_list in free_socket */ + list_add_node(&nsock->stack->same_node_recv_list, &nsock->recv_list); + return 0; } err_t same_node_memzone_create(const struct rte_memzone **zone, int size, int port, char *name, char *rx) @@ -1469,7 +1473,8 @@ err_t same_node_memzone_create(const struct rte_memzone **zone, int size, int po return ERR_MEM; } - LSTACK_LOG(INFO, LSTACK, "lstack id %d, reserve %s(%p) success, addr is %p, size is %u\n", rte_socket_id(), mem_name, *zone, (*zone)->addr, size); + LSTACK_LOG(INFO, LSTACK, "lstack id %d, reserve %s(%p) success, addr is %p, size is %u\n", + rte_socket_id(), mem_name, *zone, (*zone)->addr, size); return ERR_OK; } @@ -1496,16 +1501,16 @@ err_t same_node_ring_create(struct rte_ring **ring, int size, int port, char *na static void init_same_node_ring(struct tcp_pcb *pcb) { - struct netconn *netconn = (struct netconn *)pcb->callback_arg; - struct lwip_sock *sock = get_socket(netconn->socket); - - pcb->client_rx_ring = NULL; - pcb->client_tx_ring = NULL; - pcb->free_ring = 0; - sock->same_node_rx_ring = NULL; - sock->same_node_rx_ring_mz = NULL; - sock->same_node_tx_ring = NULL; - sock->same_node_tx_ring_mz = NULL; + struct netconn *netconn = (struct netconn *)pcb->callback_arg; + struct lwip_sock *sock = get_socket(netconn->socket); + + pcb->client_rx_ring = NULL; + pcb->client_tx_ring = NULL; + pcb->free_ring = 0; + sock->same_node_rx_ring = NULL; + sock->same_node_rx_ring_mz = NULL; + sock->same_node_tx_ring = NULL; + sock->same_node_tx_ring_mz = NULL; } #define CLIENT_RING_SIZE 512 @@ -1522,24 +1527,28 @@ err_t create_same_node_ring(struct tcp_pcb *pcb) } pcb->free_ring = 1; - if (same_node_memzone_create(&sock->same_node_rx_ring_mz, sizeof(struct same_node_ring), pcb->local_port, "rte_mz", "rx") != 0) { + if (same_node_memzone_create(&sock->same_node_rx_ring_mz, sizeof(struct same_node_ring), + pcb->local_port, "rte_mz", "rx") != 0) { goto END; } sock->same_node_rx_ring = (struct same_node_ring*)sock->same_node_rx_ring_mz->addr; - if (same_node_memzone_create(&sock->same_node_rx_ring->mz, SAME_NODE_RING_LEN, pcb->local_port, "rte_mz_buf", "rx") != 0) { + if (same_node_memzone_create(&sock->same_node_rx_ring->mz, SAME_NODE_RING_LEN, + pcb->local_port, "rte_mz_buf", "rx") != 0) { goto END; } sock->same_node_rx_ring->sndbegin = 0; sock->same_node_rx_ring->sndend = 0; - if (same_node_memzone_create(&sock->same_node_tx_ring_mz, sizeof(struct same_node_ring), pcb->local_port, "rte_mz", "tx") != 0) { + if (same_node_memzone_create(&sock->same_node_tx_ring_mz, sizeof(struct same_node_ring), + pcb->local_port, "rte_mz", "tx") != 0) { goto END; } sock->same_node_tx_ring = (struct same_node_ring*)sock->same_node_tx_ring_mz->addr; - if (same_node_memzone_create(&sock->same_node_tx_ring->mz, SAME_NODE_RING_LEN, pcb->local_port, "rte_mz_buf", "tx") != 0) { + if (same_node_memzone_create(&sock->same_node_tx_ring->mz, SAME_NODE_RING_LEN, + pcb->local_port, "rte_mz_buf", "tx") != 0) { goto END; } @@ -1560,20 +1569,21 @@ END: err_t find_same_node_ring(struct tcp_pcb *npcb) { - char name[RING_NAME_LEN] = {0}; - snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_tx_ring_%u", npcb->remote_port); - npcb->client_rx_ring = rte_ring_lookup(name); - memset_s(name, sizeof(name), 0, sizeof(name)); - snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_rx_ring_%u", npcb->remote_port); - npcb->client_tx_ring = rte_ring_lookup(name); - npcb->free_ring = 0; - if (npcb->client_tx_ring == NULL || - npcb->client_rx_ring == NULL) { - LSTACK_LOG(INFO, LSTACK, "lookup client rxtx ring failed, port is %d\n", npcb->remote_port); - tcp_abandon(npcb, 0); - return ERR_CONN; - } else { - LSTACK_LOG(INFO, LSTACK, "find client_tx_ring_%u and client_rx_ring_%u\n", npcb->remote_port, npcb->remote_port); - } - return 0; + char name[RING_NAME_LEN] = {0}; + snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_tx_ring_%u", npcb->remote_port); + npcb->client_rx_ring = rte_ring_lookup(name); + memset_s(name, sizeof(name), 0, sizeof(name)); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_rx_ring_%u", npcb->remote_port); + npcb->client_tx_ring = rte_ring_lookup(name); + npcb->free_ring = 0; + if (npcb->client_tx_ring == NULL || + npcb->client_rx_ring == NULL) { + LSTACK_LOG(INFO, LSTACK, "lookup client rxtx ring failed, port is %d\n", npcb->remote_port); + tcp_abandon(npcb, 0); + return ERR_CONN; + } else { + LSTACK_LOG(INFO, LSTACK, "find client_tx_ring_%u and client_rx_ring_%u\n", + npcb->remote_port, npcb->remote_port); + } + return 0; } diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index 7b1b994..d5523df 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -32,11 +32,11 @@ #include "lstack_ethdev.h" #include "lstack_vdev.h" #include "lstack_lwip.h" -#include "lstack_protocol_stack.h" #include "lstack_cfg.h" #include "lstack_control_plane.h" #include "posix/lstack_epoll.h" #include "lstack_stack_stat.h" +#include "lstack_protocol_stack.h" #define KERNEL_EVENT_100us 100 @@ -98,7 +98,6 @@ int get_min_conn_stack(struct protocol_stack_group *stack_group) min_conn_num = stack->conn_num; } } - } return min_conn_stk_idx; } @@ -236,7 +235,6 @@ static int32_t create_thread(void *arg, char *thread_name, stack_thread_func fun LSTACK_LOG(ERR, LSTACK, "set name failed\n"); return -1; } - } else { ret = sprintf_s(name, sizeof(name), "%s%02hu", thread_name, t_params->queue_id); if (ret < 0) { @@ -307,15 +305,16 @@ static int32_t init_stack_value(struct protocol_stack *stack, void *arg) int idx = t_params->idx; if (get_global_cfg_params()->seperate_send_recv) { + // 2: idx is even, stack is recv thread, idx is odd, stack is send thread if (idx % 2 == 0) { - stack->cpu_id = get_global_cfg_params()->recv_cpus[idx/2]; + stack->cpu_id = get_global_cfg_params()->recv_cpus[idx / 2]; stack->is_send_thread = 0; - }else { - stack->cpu_id = get_global_cfg_params()->send_cpus[idx/2]; + } else { + stack->cpu_id = get_global_cfg_params()->send_cpus[idx / 2]; stack->is_send_thread = 1; } - }else { - stack->cpu_id = get_global_cfg_params()->cpus[idx]; + } else { + stack->cpu_id = get_global_cfg_params()->cpus[idx]; } stack->socket_id = numa_node_of_cpu(stack->cpu_id); @@ -355,7 +354,7 @@ static int32_t create_affiliate_thread(void *arg) static struct protocol_stack *stack_thread_init(void *arg) { - struct protocol_stack_group *stack_group = get_protocol_stack_group(); + struct protocol_stack_group *stack_group = get_protocol_stack_group(); struct protocol_stack *stack = calloc(1, sizeof(*stack)); if (stack == NULL) { LSTACK_LOG(ERR, LSTACK, "malloc stack failed\n"); @@ -437,7 +436,7 @@ static void* gazelle_stack_thread(void *arg) uint16_t queue_id = t_params->queue_id; struct cfg_params *cfg = get_global_cfg_params(); - uint8_t use_ltran_flag = cfg->use_ltran;; + uint8_t use_ltran_flag = cfg->use_ltran; bool kni_switch = cfg->kni_switch; uint32_t read_connect_number = cfg->read_connect_number; uint32_t rpc_number = cfg->rpc_number; @@ -502,7 +501,7 @@ static void* gazelle_stack_thread(void *arg) static void libnet_listen_thread(void *arg) { - struct cfg_params * cfg_param = get_global_cfg_params(); + struct cfg_params *cfg_param = get_global_cfg_params(); recv_pkts_from_other_process(cfg_param->process_idx, arg); } @@ -542,7 +541,7 @@ int32_t init_protocol_stack(void) if (!get_global_cfg_params()->seperate_send_recv) { stack_group->stack_num = get_global_cfg_params()->num_cpu; - }else { + } else { stack_group->stack_num = get_global_cfg_params()->num_cpu * 2; } @@ -550,7 +549,6 @@ int32_t init_protocol_stack(void) pthread_spin_init(&stack_group->poll_list_lock, PTHREAD_PROCESS_PRIVATE); pthread_spin_init(&stack_group->socket_lock, PTHREAD_PROCESS_PRIVATE); - if (init_protocol_sem() != 0) { return -1; } @@ -559,9 +557,10 @@ int32_t init_protocol_stack(void) int process_index = get_global_cfg_params()->process_idx; if (get_global_cfg_params()->is_primary) { + uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count; for (uint16_t idx = 0; idx < get_global_cfg_params()->tot_queue_num; idx++) { struct rte_mempool* rxtx_mbuf = create_pktmbuf_mempool("rxtx_mbuf", - get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count / stack_group->stack_num, RXTX_CACHE_SZ, idx); + total_mbufs / stack_group->stack_num, RXTX_CACHE_SZ, idx); if (rxtx_mbuf == NULL) { return -1; } @@ -572,12 +571,12 @@ int32_t init_protocol_stack(void) for (uint32_t i = 0; i < queue_num; i++) { if (get_global_cfg_params()->seperate_send_recv) { if (i % 2 == 0) { - ret = sprintf_s(name, sizeof(name), "%s_%d_%d", LSTACK_RECV_THREAD_NAME, process_index, i/2); + ret = sprintf_s(name, sizeof(name), "%s_%d_%d", LSTACK_RECV_THREAD_NAME, process_index, i / 2); if (ret < 0) { return -1; } } else { - ret = sprintf_s(name, sizeof(name), "%s_%d_%d", LSTACK_SEND_THREAD_NAME, process_index, i/2); + ret = sprintf_s(name, sizeof(name), "%s_%d_%d", LSTACK_SEND_THREAD_NAME, process_index, i / 2); if (ret < 0) { return -1; } @@ -601,17 +600,18 @@ int32_t init_protocol_stack(void) wait_sem_value(&stack_group->thread_phase1, stack_group->stack_num); - for(int idx = 0; idx < queue_num; idx++){ + for (int idx = 0; idx < queue_num; idx++){ free(t_params[idx]); } - if (!use_ltran()) { + if (!use_ltran()) { ret = sem_init(&stack_group->sem_listen_thread, 0, 0); ret = sprintf_s(name, sizeof(name), "%s", "listen_thread"); - struct sys_thread *thread = sys_thread_new(name, libnet_listen_thread, (void*)(&stack_group->sem_listen_thread), 0, 0); + struct sys_thread *thread = sys_thread_new(name, libnet_listen_thread, + (void*)(&stack_group->sem_listen_thread), 0, 0); free(thread); sem_wait(&stack_group->sem_listen_thread); - } + } if (get_init_fail()) { return -1; @@ -888,7 +888,7 @@ int32_t stack_broadcast_listen(int32_t fd, int32_t backlog) if (min_conn_stk_idx == i) { get_socket_by_fd(clone_fd)->conn->is_master_fd = 1; - }else { + } else { get_socket_by_fd(clone_fd)->conn->is_master_fd = 0; } diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c index d1c749a..60832b4 100644 --- a/src/lstack/core/lstack_stack_stat.c +++ b/src/lstack/core/lstack_stack_stat.c @@ -25,9 +25,9 @@ #include "gazelle_dfx_msg.h" #include "lstack_thread_rpc.h" #include "lstack_protocol_stack.h" -#include "lstack_stack_stat.h" #include "posix/lstack_epoll.h" #include "lstack_dpdk.h" +#include "lstack_stack_stat.h" #define US_PER_SEC 1000000 @@ -55,7 +55,7 @@ void calculate_lstack_latency(struct gazelle_stack_latency *stack_latency, const const struct latency_timestamp *lt; if (pbuf == NULL) { - return; + return; } lt = &pbuf_to_private(pbuf)->lt; @@ -120,7 +120,8 @@ static void set_latency_start_flag(bool start) stack->latency.start_time = get_current_time(); stack->latency.lwip_latency.latency_min = ~((uint64_t)0); stack->latency.read_latency.latency_min = ~((uint64_t)0); - memset_s(&stack->aggregate_stats, sizeof(struct gazelle_stack_aggregate_stats), 0, sizeof(stack->aggregate_stats)); + memset_s(&stack->aggregate_stats, sizeof(struct gazelle_stack_aggregate_stats), + 0, sizeof(stack->aggregate_stats)); } } @@ -218,7 +219,8 @@ static void get_stack_dfx_data(struct gazelle_stack_dfx_data *dfx, struct protoc } break; case GAZELLE_STAT_LSTACK_SHOW_AGGREGATE: - ret = memcpy_s(&dfx->data.aggregate_stats, sizeof(dfx->data.aggregate_stats), &stack->aggregate_stats, sizeof(stack->aggregate_stats)); + ret = memcpy_s(&dfx->data.aggregate_stats, sizeof(dfx->data.aggregate_stats), + &stack->aggregate_stats, sizeof(stack->aggregate_stats)); if (ret != EOK) { LSTACK_LOG(ERR, LSTACK, "memcpy_s err ret=%d \n", ret); } diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h index a896903..a1262a5 100644 --- a/src/lstack/include/lstack_dpdk.h +++ b/src/lstack/include/lstack_dpdk.h @@ -53,7 +53,8 @@ int32_t dpdk_init_lstack_kni(void); void dpdk_restore_pci(void); bool port_in_stack_queue(uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); uint16_t get_port_id(void); -struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,uint32_t mbuf_cache_size, uint16_t queue_id); +struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, + uint32_t mbuf_cache_size, uint16_t queue_id); void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id); #endif /* GAZELLE_DPDK_H */ diff --git a/src/lstack/include/lstack_ethdev.h b/src/lstack/include/lstack_ethdev.h index 7f944eb..326bd1b 100644 --- a/src/lstack/include/lstack_ethdev.h +++ b/src/lstack/include/lstack_ethdev.h @@ -20,10 +20,10 @@ enum port_type { PORT_CONNECT, }; -enum PACKET_TRANSFER_TYPE{ +enum PACKET_TRANSFER_TYPE { TRANSFER_KERNEL = -1, TRANSFER_OTHER_THREAD, - TRANSFER_CURRENT_THREAD, + TRANSFER_CURRENT_THREAD, }; enum TRANSFER_MESSAGE_RESULT { diff --git a/src/lstack/include/lstack_lockless_queue.h b/src/lstack/include/lstack_lockless_queue.h index c70b56a..c6f6f32 100644 --- a/src/lstack/include/lstack_lockless_queue.h +++ b/src/lstack/include/lstack_lockless_queue.h @@ -70,8 +70,9 @@ static inline lockless_queue_node* lockless_queue_mpsc_pop(lockless_queue* queue lockless_queue_node *next = tail->next; if (tail == &queue->stub) { - if (next == NULL) + if (next == NULL) { return NULL; + } queue->tail = next; tail = next; next = next->next; @@ -83,8 +84,9 @@ static inline lockless_queue_node* lockless_queue_mpsc_pop(lockless_queue* queue } lockless_queue_node *head = queue->head; - if (tail != head) + if (tail != head) { return NULL; + } lockless_queue_mpsc_push(queue, &queue->stub); diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h index c75161f..3a447dc 100644 --- a/src/lstack/include/lstack_protocol_stack.h +++ b/src/lstack/include/lstack_protocol_stack.h @@ -138,7 +138,7 @@ void stack_broadcast_clean_epoll(struct wakeup_poll *wakeup); void stack_send_pkts(struct protocol_stack *stack); struct rpc_msg; -struct thread_params{ +struct thread_params { uint16_t queue_id; uint16_t idx; }; diff --git a/src/lstack/include/lstack_vdev.h b/src/lstack/include/lstack_vdev.h index 540a31a..007eec7 100644 --- a/src/lstack/include/lstack_vdev.h +++ b/src/lstack/include/lstack_vdev.h @@ -23,7 +23,8 @@ int vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple); int recv_pkts_from_other_process(int process_index, void* arg); void transfer_delete_rule_info_to_process0(uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); -void transfer_create_rule_info_to_process0(uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); +void transfer_create_rule_info_to_process0(uint16_t queue_id, uint32_t src_ip, + uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); void transfer_add_or_delete_listen_port_to_process0(uint16_t listen_port, uint8_t process_idx, uint8_t is_add); void init_listen_and_user_ports(); diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index 0a91f79..2bae2f1 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -21,7 +21,7 @@ #include #include #include -#include "lwip/tcp.h" +#include #include #include @@ -36,37 +36,41 @@ #include "lstack_lwip.h" #include "dpdk_common.h" #include "lstack_protocol_stack.h" -#include "lstack_ethdev.h" #include "lstack_thread_rpc.h" +#include "lstack_ethdev.h" /* FRAME_MTU + 14byte header */ -#define MBUF_MAX_LEN 1514 -#define MAX_PATTERN_NUM 4 -#define MAX_ACTION_NUM 2 -#define FULL_MASK 0xffffffff /* full mask */ -#define EMPTY_MASK 0x0 /* empty mask */ -#define LSTACK_MBUF_LEN 64 -#define TRANSFER_TCP_MUBF_LEN LSTACK_MBUF_LEN + 3 -#define DELETE_FLOWS_PARAMS_NUM 3 -#define DELETE_FLOWS_PARAMS_LENGTH 30 -#define CREATE_FLOWS_PARAMS_NUM 6 -#define CREATE_FLOWS_PARAMS_LENGTH 60 +#define MBUF_MAX_LEN 1514 +#define MAX_PATTERN_NUM 4 +#define MAX_ACTION_NUM 2 +#define FULL_MASK 0xffffffff /* full mask */ +#define EMPTY_MASK 0x0 /* empty mask */ +#define LSTACK_MBUF_LEN 64 +#define TRANSFER_TCP_MUBF_LEN (LSTACK_MBUF_LEN + 3) +#define DELETE_FLOWS_PARAMS_NUM 3 +#define DELETE_FLOWS_PARAMS_LENGTH 30 +#define CREATE_FLOWS_PARAMS_NUM 6 +#define CREATE_FLOWS_PARAMS_LENGTH 60 #define ADD_OR_DELETE_LISTEN_PORT_PARAMS_LENGTH 25 -#define ADD_OR_DELETE_LISTEN_PORT_PARAMS_NUM 3 -#define REPLY_LEN 10 -#define SUCCESS_REPLY "success" -#define ERROR_REPLY "error" -#define PACKET_READ_SIZE 32 +#define ADD_OR_DELETE_LISTEN_PORT_PARAMS_NUM 3 +#define REPLY_LEN 10 +#define SUCCESS_REPLY "success" +#define ERROR_REPLY "error" +#define PACKET_READ_SIZE 32 + +#define GET_LSTACK_NUM 14 +#define GET_LSTACK_NUM_STRING "get_lstack_num" + +#define SERVER_PATH "/var/run/gazelle/server.socket" +#define SPLIT_DELIM "," -#define GET_LSTACK_NUM 14 -#define GET_LSTACK_NUM_STRING "get_lstack_num" +#define UNIX_TCP_PORT_MAX 65535 -char *client_path = "/var/run/gazelle/client.socket"; -char *server_path = "/var/run/gazelle/server.socket"; -const char *split_delim = ","; +#define IPV4_VERSION_OFFSET 4 +#define IPV4_VERSION 4 -uint8_t g_user_ports[65535] = {INVAILD_PROCESS_IDX,}; -uint8_t g_listen_ports[65535] = {INVAILD_PROCESS_IDX,}; +static uint8_t g_user_ports[UNIX_TCP_PORT_MAX] = {INVAILD_PROCESS_IDX, }; +static uint8_t g_listen_ports[UNIX_TCP_PORT_MAX] = {INVAILD_PROCESS_IDX, }; void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack) { @@ -173,9 +177,9 @@ void add_rule(char* rule_key, struct rte_flow *flow) struct flow_rule *rule; HASH_FIND_STR(g_flow_rules, rule_key, rule); if (rule == NULL) { - rule = (struct flow_rule*)malloc(sizeof(struct flow_rule)); - strcpy_s(rule->rule_key, RULE_KEY_LEN, rule_key); - HASH_ADD_STR(g_flow_rules, rule_key, rule); + rule = (struct flow_rule*)malloc(sizeof(struct flow_rule)); + strcpy_s(rule->rule_key, RULE_KEY_LEN, rule_key); + HASH_ADD_STR(g_flow_rules, rule_key, rule); } rule->flow = flow; } @@ -185,8 +189,8 @@ void delete_rule(char* rule_key) struct flow_rule *rule = NULL; HASH_FIND_STR(g_flow_rules, rule_key, rule); if (rule == NULL) { - HASH_DEL(g_flow_rules, rule); - free(rule); + HASH_DEL(g_flow_rules, rule); + free(rule); } } @@ -204,15 +208,14 @@ int transfer_pkt_to_other_process(char *buf, int process_index, int write_len, b int ret = 0; sockfd = posix_api->socket_fn(AF_UNIX, SOCK_STREAM, 0); - - memset_s(&serun, sizeof(serun), 0, sizeof(serun)); + memset_s(&serun, sizeof(serun), 0, sizeof(serun)); serun.sun_family = AF_UNIX; - sprintf_s(serun.sun_path, PATH_MAX,"%s%d", server_path, process_index); + sprintf_s(serun.sun_path, PATH_MAX, "%s%d", SERVER_PATH, process_index); int32_t len = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); - if (posix_api->connect_fn(sockfd, (struct sockaddr *)&serun, len) < 0){ - return CONNECT_ERROR; + if (posix_api->connect_fn(sockfd, (struct sockaddr *)&serun, len) < 0) { + return CONNECT_ERROR; } - posix_api->write_fn(sockfd, buf, write_len); + posix_api->write_fn(sockfd, buf, write_len); if (need_reply) { char reply_message[REPLY_LEN]; int32_t read_result = posix_api->read_fn(sockfd, reply_message, REPLY_LEN); @@ -233,7 +236,8 @@ int transfer_pkt_to_other_process(char *buf, int process_index, int write_len, b return ret; } -int32_t check_params_from_primary(void){ +int32_t check_params_from_primary(void) +{ struct cfg_params *cfg = get_global_cfg_params(); if (cfg->is_primary) { return 0; @@ -248,39 +252,40 @@ int32_t check_params_from_primary(void){ return 0; } -struct rte_flow * -create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, - uint16_t src_port, uint16_t dst_port, struct rte_flow_error *error) +struct rte_flow *create_flow_director(uint16_t port_id, uint16_t queue_id, + uint32_t src_ip, uint32_t dst_ip, + uint16_t src_port, uint16_t dst_port, + struct rte_flow_error *error) { struct rte_flow_attr attr; - struct rte_flow_item pattern[MAX_PATTERN_NUM]; - struct rte_flow_action action[MAX_ACTION_NUM]; - struct rte_flow *flow = NULL; - struct rte_flow_action_queue queue = { .index = queue_id }; - struct rte_flow_item_ipv4 ip_spec; - struct rte_flow_item_ipv4 ip_mask; + struct rte_flow_item pattern[MAX_PATTERN_NUM]; + struct rte_flow_action action[MAX_ACTION_NUM]; + struct rte_flow *flow = NULL; + struct rte_flow_action_queue queue = { .index = queue_id }; + struct rte_flow_item_ipv4 ip_spec; + struct rte_flow_item_ipv4 ip_mask; struct rte_flow_item_tcp tcp_spec; struct rte_flow_item_tcp tcp_mask; - int res; + int res; - memset_s(pattern, sizeof(pattern), 0, sizeof(pattern)); - memset_s(action, sizeof(action), 0, sizeof(action)); + memset_s(pattern, sizeof(pattern), 0, sizeof(pattern)); + memset_s(action, sizeof(action), 0, sizeof(action)); /* - * set the rule attribute. - * in this case only ingress packets will be checked. - */ - memset_s(&attr, sizeof(struct rte_flow_attr), 0, sizeof(struct rte_flow_attr)); - attr.ingress = 1; + * set the rule attribute. + * in this case only ingress packets will be checked. + */ + memset_s(&attr, sizeof(struct rte_flow_attr), 0, sizeof(struct rte_flow_attr)); + attr.ingress = 1; /* - * create the action sequence. - * one action only, move packet to queue - */ - action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; - action[0].conf = &queue; - action[1].type = RTE_FLOW_ACTION_TYPE_END; + * create the action sequence. + * one action only, move packet to queue + */ + action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; + action[0].conf = &queue; + action[1].type = RTE_FLOW_ACTION_TYPE_END; // not limit eth header pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; @@ -299,7 +304,7 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 // tcp header, full mask 0xffff memset_s(&tcp_spec, sizeof(struct rte_flow_item_tcp), 0, sizeof(struct rte_flow_item_tcp)); memset_s(&tcp_mask, sizeof(struct rte_flow_item_tcp), 0, sizeof(struct rte_flow_item_tcp)); - pattern[2].type = RTE_FLOW_ITEM_TYPE_TCP; + pattern[2].type = RTE_FLOW_ITEM_TYPE_TCP; // 2: pattern 2 is tcp header tcp_spec.hdr.src_port = src_port; tcp_spec.hdr.dst_port = dst_port; tcp_mask.hdr.src_port = rte_flow_item_tcp_mask.hdr.src_port; @@ -308,35 +313,39 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 pattern[2].mask = &tcp_mask; /* the final level must be always type end */ - pattern[3].type = RTE_FLOW_ITEM_TYPE_END; + pattern[3].type = RTE_FLOW_ITEM_TYPE_END; res = rte_flow_validate(port_id, &attr, pattern, action, error); - if (!res){ + if (!res) { flow = rte_flow_create(port_id, &attr, pattern, action, error); - }else { - LSTACK_LOG(ERR, PORT,"rte_flow_create.rte_flow_validate error, res %d \n", res); + } else { + LSTACK_LOG(ERR, PORT, "rte_flow_create.rte_flow_validate error, res %d \n", res); } - return flow; + return flow; } -void config_flow_director(uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port){ - +void config_flow_director(uint16_t queue_id, uint32_t src_ip, + uint32_t dst_ip, uint16_t src_port, uint16_t dst_port) +{ uint16_t port_id = get_port_id(); char rule_key[RULE_KEY_LEN] = {0}; sprintf_s(rule_key, sizeof(rule_key), "%u_%u_%u", src_ip, src_port, dst_port); struct flow_rule *fl_exist = find_rule(rule_key); - if(fl_exist != NULL){ + if (fl_exist != NULL) { return; } - LSTACK_LOG(INFO, LSTACK, "config_flow_director, flow queue_id %u, src_ip %u,src_port_ntohs:%u, dst_port_ntohs :%u \n", - queue_id, src_ip,ntohs(src_port), ntohs(dst_port) ); + LSTACK_LOG(INFO, LSTACK, + "config_flow_director, flow queue_id %u, src_ip %u,src_port_ntohs:%u, dst_port_ntohs:%u\n", + queue_id, src_ip, ntohs(src_port), ntohs(dst_port)); struct rte_flow_error error; struct rte_flow *flow = create_flow_director(port_id, queue_id, src_ip, dst_ip, src_port, dst_port, &error); if (!flow) { - LSTACK_LOG(ERR, LSTACK,"config_flow_director, flow can not be created. queue_id %u, src_ip %u, src_port %u, dst_port %u, dst_port_ntohs :%u, type %d. message: %s\n", - queue_id, src_ip,src_port,dst_port,ntohs(dst_port), error.type, error.message ? error.message : "(no stated reason)"); + LSTACK_LOG(ERR, LSTACK,"flow can not be created. queue_id %u, src_ip %u, src_port %u," + "dst_port %u, dst_port_ntohs :%u, type %d. message: %s\n", + queue_id, src_ip, src_port, dst_port, ntohs(dst_port), + error.type, error.message ? error.message : "(no stated reason)"); return; } __sync_fetch_and_add(&g_flow_num, 1); @@ -354,59 +363,61 @@ void delete_flow_director(uint32_t dst_ip, uint16_t src_port, uint16_t dst_port) struct rte_flow_error error; int ret = rte_flow_destroy(port_id, fl->flow, &error); if(ret != 0){ - LSTACK_LOG(ERR, PORT,"Flow can't be delete %d message: %s\n",error.type,error.message ? error.message : "(no stated reason)"); + LSTACK_LOG(ERR, PORT, "Flow can't be delete %d message: %s\n", + error.type, error.message ? error.message : "(no stated reason)"); } delete_rule(rule_key); __sync_fetch_and_sub(&g_flow_num, 1); } } -/* - * delete flows - * if process 0, delete directly, else transfer 'dst_ip,src_port,dst_port' to process 0. - */ +/* if process 0, delete directly, else transfer 'dst_ip,src_port,dst_port' to process 0. */ void transfer_delete_rule_info_to_process0(uint32_t dst_ip, uint16_t src_port, uint16_t dst_port) { - if (get_global_cfg_params()->is_primary){ + if (get_global_cfg_params()->is_primary) { delete_flow_director(dst_ip, src_port, dst_port); - }else { + } else { char process_server_path[DELETE_FLOWS_PARAMS_LENGTH]; - sprintf_s(process_server_path, DELETE_FLOWS_PARAMS_LENGTH, "%u%s%u%s%u", dst_ip,split_delim, src_port,split_delim,dst_port); + sprintf_s(process_server_path, DELETE_FLOWS_PARAMS_LENGTH, "%u%s%u%s%u", + dst_ip, SPLIT_DELIM, src_port, SPLIT_DELIM, dst_port); int ret = transfer_pkt_to_other_process(process_server_path, 0, DELETE_FLOWS_PARAMS_LENGTH, false); if(ret != TRANSFER_SUCESS){ - LSTACK_LOG(ERR, LSTACK,"transfer_delete_rule_info_to_process0 error. tid %d. dst_ip %u, src_port: %u, dst_port %u\n", + LSTACK_LOG(ERR, LSTACK, "error. tid %d. dst_ip %u, src_port: %u, dst_port %u\n", rte_gettid(), dst_ip, src_port, dst_port); } } } -/* - * add flows - * if process 0, add directly, else transfer 'src_ip,dst_ip，src_port，dst_port,queue_id' to process 0. - */ -void transfer_create_rule_info_to_process0(uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port) +// if process 0, add directly, else transfer 'src_ip,dst_ip，src_port，dst_port,queue_id' to process 0. +void transfer_create_rule_info_to_process0(uint16_t queue_id, uint32_t src_ip, + uint32_t dst_ip, uint16_t src_port, + uint16_t dst_port) { char process_server_path[CREATE_FLOWS_PARAMS_LENGTH]; /* exchage src_ip and dst_ip, src_port and dst_port */ uint8_t process_idx = get_global_cfg_params()->process_idx; - sprintf_s(process_server_path, CREATE_FLOWS_PARAMS_LENGTH, "%u%s%u%s%u%s%u%s%u%s%u", - dst_ip,split_delim,src_ip,split_delim, dst_port,split_delim,src_port, split_delim,queue_id,split_delim,process_idx); + sprintf_s(process_server_path, CREATE_FLOWS_PARAMS_LENGTH, "%u%s%u%s%u%s%u%s%u%s%u", + dst_ip, SPLIT_DELIM, src_ip, SPLIT_DELIM, + dst_port, SPLIT_DELIM, src_port, SPLIT_DELIM, + queue_id, SPLIT_DELIM, process_idx); int ret = transfer_pkt_to_other_process(process_server_path, 0, CREATE_FLOWS_PARAMS_LENGTH, true); - if(ret != TRANSFER_SUCESS){ - LSTACK_LOG(ERR, LSTACK,"transfer_create_rule_info_to_process0 error. tid %d. src_ip %u, dst_ip %u, src_port: %u, dst_port %u, queue_id %u, process_idx %u\n", - rte_gettid(), src_ip, dst_ip, src_port, dst_port, queue_id, process_idx); + if (ret != TRANSFER_SUCESS) { + LSTACK_LOG(ERR, LSTACK, "error. tid %d. src_ip %u, dst_ip %u, src_port: %u, dst_port %u," + "queue_id %u, process_idx %u\n", + rte_gettid(), src_ip, dst_ip, src_port, dst_port, queue_id, process_idx); } } void transfer_add_or_delete_listen_port_to_process0(uint16_t listen_port, uint8_t process_idx, uint8_t is_add) { char process_server_path[ADD_OR_DELETE_LISTEN_PORT_PARAMS_LENGTH]; - sprintf_s(process_server_path, ADD_OR_DELETE_LISTEN_PORT_PARAMS_LENGTH, "%u%s%u%s%u", listen_port,split_delim,process_idx, split_delim, is_add); + sprintf_s(process_server_path, ADD_OR_DELETE_LISTEN_PORT_PARAMS_LENGTH, + "%u%s%u%s%u", listen_port, SPLIT_DELIM, process_idx, SPLIT_DELIM, is_add); int ret = transfer_pkt_to_other_process(process_server_path, 0, ADD_OR_DELETE_LISTEN_PORT_PARAMS_LENGTH, true); if(ret != TRANSFER_SUCESS) { - LSTACK_LOG(ERR, LSTACK,"transfer_add_or_delete_listen_port_to_process0 error. tid %d. listen_port %u, process_idx %u\n", - rte_gettid(), listen_port, process_idx); - } + LSTACK_LOG(ERR, LSTACK, "error. tid %d. listen_port %u, process_idx %u\n", + rte_gettid(), listen_port, process_idx); + } } static int str_to_array(char *args, uint32_t *array, int size) @@ -417,7 +428,7 @@ static int str_to_array(char *args, uint32_t *array, int size) char *next_token = NULL; memset_s(array, sizeof(*array) * size, 0, sizeof(*array) * size); - elem = strtok_s((char *)args, split_delim, &next_token); + elem = strtok_s((char *)args, SPLIT_DELIM, &next_token); while (elem != NULL) { if (cnt >= size) { return -1; @@ -429,7 +440,7 @@ static int str_to_array(char *args, uint32_t *array, int size) array[cnt] = (uint32_t)val; cnt++; - elem = strtok_s(NULL, split_delim, &next_token); + elem = strtok_s(NULL, SPLIT_DELIM, &next_token); } return cnt; @@ -445,18 +456,20 @@ void parse_and_delete_rule(char* buf) delete_flow_director(dst_ip, src_port, dst_port); } -void add_user_process_port(uint16_t dst_port, uint8_t process_idx, enum port_type type){ +void add_user_process_port(uint16_t dst_port, uint8_t process_idx, enum port_type type) +{ if (type == PORT_LISTEN) { - g_listen_ports[dst_port] = process_idx; - }else { + g_listen_ports[dst_port] = process_idx; + } else { g_user_ports[dst_port] = process_idx; } } -void delete_user_process_port(uint16_t dst_port, enum port_type type){ +void delete_user_process_port(uint16_t dst_port, enum port_type type) +{ if (type == PORT_LISTEN) { g_listen_ports[dst_port] = INVAILD_PROCESS_IDX; - }else { + } else { g_user_ports[dst_port] = INVAILD_PROCESS_IDX; } } @@ -483,8 +496,8 @@ void parse_and_add_or_delete_listen_port(char* buf) uint8_t process_idx = array[1]; uint8_t is_add = array[2]; if (is_add == 1) { - add_user_process_port(listen_port,process_idx, PORT_LISTEN); - }else { + add_user_process_port(listen_port, process_idx, PORT_LISTEN); + } else { delete_user_process_port(listen_port, PORT_LISTEN); } @@ -498,9 +511,9 @@ void transfer_arp_to_other_process(struct rte_mbuf *mbuf) char arp_mbuf[LSTACK_MBUF_LEN] = {0}; sprintf_s(arp_mbuf, sizeof(arp_mbuf), "%lu", mbuf); int result = transfer_pkt_to_other_process(arp_mbuf, i, LSTACK_MBUF_LEN, false); - if(result == CONNECT_ERROR){ + if (result == CONNECT_ERROR) { LSTACK_LOG(INFO, LSTACK,"connect process %d failed, ensure the process is started.\n", i); - }else if (result == REPLY_ERROR) { + } else if (result == REPLY_ERROR) { LSTACK_LOG(ERR, LSTACK,"transfer arp pakages to process %d error. %m\n", i); } } @@ -544,9 +557,9 @@ void parse_arp_and_transefer(char* buf) void parse_tcp_and_transefer(char* buf) { char *next_token = NULL; - char *elem = strtok_s(buf, split_delim, &next_token); + char *elem = strtok_s(buf, SPLIT_DELIM, &next_token); struct rte_mbuf *mbuf = (struct rte_mbuf *) atoll(elem); - elem = strtok_s(NULL, split_delim, &next_token); + elem = strtok_s(NULL, SPLIT_DELIM, &next_token); uint16_t queue_id = atoll(elem); struct protocol_stack_group *stack_group = get_protocol_stack_group(); @@ -566,8 +579,9 @@ void parse_tcp_and_transefer(char* buf) transfer_tcp_to_thread(mbuf_copy, stk_index); } -int recv_pkts_from_other_process(int process_index, void* arg){ - struct sockaddr_un serun, cliun; +int recv_pkts_from_other_process(int process_index, void* arg) +{ + struct sockaddr_un serun, cliun; socklen_t cliun_len; int listenfd, connfd, size; char buf[132]; @@ -580,23 +594,23 @@ int recv_pkts_from_other_process(int process_index, void* arg){ memset_s(&serun, sizeof(serun), 0, sizeof(serun)); serun.sun_family = AF_UNIX; char process_server_path[PATH_MAX]; - sprintf_s(process_server_path, sizeof(process_server_path), "%s%d", server_path, process_index); - strcpy_s(serun.sun_path, sizeof(serun.sun_path),process_server_path); + sprintf_s(process_server_path, sizeof(process_server_path), "%s%d", SERVER_PATH, process_index); + strcpy_s(serun.sun_path, sizeof(serun.sun_path), process_server_path); size = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); unlink(process_server_path); - if (posix_api->bind_fn(listenfd, (struct sockaddr *)&serun, size) < 0) { + if (posix_api->bind_fn(listenfd, (struct sockaddr *)&serun, size) < 0) { perror("bind error"); return -1; } if (posix_api->listen_fn(listenfd, 20) < 0) { /* 20: max backlog */ - perror("listen error"); + perror("listen error"); return -1; - } + } sem_post((sem_t *)arg); /* block */ - while(1) { + while(1) { cliun_len = sizeof(cliun); - if ((connfd = posix_api->accept_fn(listenfd, (struct sockaddr *)&cliun, &cliun_len)) < 0){ + if ((connfd = posix_api->accept_fn(listenfd, (struct sockaddr *)&cliun, &cliun_len)) < 0) { perror("accept error"); continue; } @@ -605,11 +619,11 @@ int recv_pkts_from_other_process(int process_index, void* arg){ if (n < 0) { perror("read error"); break; - } else if(n == 0) { + } else if (n == 0) { break; } - if(n == LSTACK_MBUF_LEN){ + if(n == LSTACK_MBUF_LEN) { /* arp */ parse_arp_and_transefer(buf); } else if (n == TRANSFER_TCP_MUBF_LEN) { @@ -618,17 +632,17 @@ int recv_pkts_from_other_process(int process_index, void* arg){ } else if (n == DELETE_FLOWS_PARAMS_LENGTH) { /* delete rule */ parse_and_delete_rule(buf); - }else if(n == CREATE_FLOWS_PARAMS_LENGTH){ + } else if(n == CREATE_FLOWS_PARAMS_LENGTH) { /* add rule */ parse_and_create_rule(buf); char reply_buf[REPLY_LEN]; sprintf_s(reply_buf, sizeof(reply_buf), "%s", SUCCESS_REPLY); posix_api->write_fn(connfd, reply_buf, REPLY_LEN); - }else if (n == GET_LSTACK_NUM) { + } else if (n == GET_LSTACK_NUM) { char reply_buf[REPLY_LEN]; sprintf_s(reply_buf, sizeof(reply_buf), "%d", get_global_cfg_params()->num_cpu); posix_api->write_fn(connfd, reply_buf, REPLY_LEN); - }else{ + } else { /* add port */ parse_and_add_or_delete_listen_port(buf); char reply_buf[REPLY_LEN]; @@ -643,59 +657,62 @@ int recv_pkts_from_other_process(int process_index, void* arg){ return 0; } -void concat_mbuf_and_queue_id(struct rte_mbuf *mbuf, uint16_t queue_id, char* mbuf_and_queue_id, int write_len){ - - sprintf_s(mbuf_and_queue_id, write_len, "%lu%s%u", mbuf,split_delim,queue_id); +void concat_mbuf_and_queue_id(struct rte_mbuf *mbuf, uint16_t queue_id, + char* mbuf_and_queue_id, int write_len) +{ + sprintf_s(mbuf_and_queue_id, write_len, "%lu%s%u", mbuf, SPLIT_DELIM, queue_id); } -const int32_t ipv4_version_offset = 4; -const int32_t ipv4_version = 4; - int distribute_pakages(struct rte_mbuf *mbuf) { struct rte_ipv4_hdr *iph = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); - uint8_t ip_version = (iph->version_ihl & 0xf0) >> ipv4_version_offset; - if (likely(ip_version == ipv4_version)) { + uint8_t ip_version = (iph->version_ihl & 0xf0) >> IPV4_VERSION_OFFSET; + if (likely(ip_version == IPV4_VERSION)) { if (likely(iph->next_proto_id == IPPROTO_TCP)) { int each_process_queue_num = get_global_cfg_params()->num_queue; - struct rte_tcp_hdr *tcp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, sizeof(struct rte_ether_hdr) + - sizeof(struct rte_ipv4_hdr)); + struct rte_tcp_hdr *tcp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); uint16_t dst_port = tcp_hdr->dst_port; + uint32_t user_process_idx; - int user_process_idx = (g_listen_ports[dst_port] != INVAILD_PROCESS_IDX) ? g_listen_ports[dst_port] : g_user_ports[dst_port]; + if (g_listen_ports[dst_port] != INVAILD_PROCESS_IDX) { + user_process_idx = g_listen_ports[dst_port]; + } else { + user_process_idx = g_user_ports[dst_port]; + } if (user_process_idx == INVAILD_PROCESS_IDX) { return TRANSFER_KERNEL; } - if(unlikely(tcp_hdr->tcp_flags == TCP_SYN)){ + if (unlikely(tcp_hdr->tcp_flags == TCP_SYN)) { uint32_t src_ip = iph->src_addr; uint16_t src_port = tcp_hdr->src_port; - uint32_t index = rte_jhash_3words(src_ip, src_port | (dst_port) << 16, 0, 0) % each_process_queue_num; + uint32_t index = rte_jhash_3words(src_ip, src_port | ((dst_port) << 16), 0, 0); + index = index % each_process_queue_num; uint16_t queue_id = 0; if (get_global_cfg_params()->seperate_send_recv) { - queue_id = user_process_idx * each_process_queue_num + (index/2) * 2; - }else { + queue_id = user_process_idx * each_process_queue_num + (index / 2) * 2; + } else { queue_id = user_process_idx * each_process_queue_num + index; } - if(queue_id != 0){ - if(user_process_idx == 0){ + if (queue_id != 0) { + if (user_process_idx == 0) { transfer_tcp_to_thread(mbuf, queue_id); - }else { + } else { char mbuf_and_queue_id[TRANSFER_TCP_MUBF_LEN]; concat_mbuf_and_queue_id(mbuf, queue_id, mbuf_and_queue_id, TRANSFER_TCP_MUBF_LEN); - transfer_pkt_to_other_process(mbuf_and_queue_id, user_process_idx, TRANSFER_TCP_MUBF_LEN, false); + transfer_pkt_to_other_process(mbuf_and_queue_id, user_process_idx, + TRANSFER_TCP_MUBF_LEN, false); } return TRANSFER_OTHER_THREAD; - }else { + } else { return TRANSFER_CURRENT_THREAD; } - }else { + } else { return TRANSFER_CURRENT_THREAD; - } + } } - }else { - return TRANSFER_KERNEL; } return TRANSFER_KERNEL; } @@ -706,7 +723,7 @@ void kni_handle_rx(uint16_t port_id) struct rte_kni* kni = get_gazelle_kni(); uint32_t nb_kni_rx = 0; if (kni) { - nb_kni_rx = rte_kni_rx_burst(kni, pkts_burst, PACKET_READ_SIZE); + nb_kni_rx = rte_kni_rx_burst(kni, pkts_burst, PACKET_READ_SIZE); } if (nb_kni_rx > 0) { uint16_t nb_rx = rte_eth_tx_burst(port_id, 0, pkts_burst, nb_kni_rx); @@ -777,12 +794,11 @@ int32_t gazelle_eth_dev_poll(struct protocol_stack *stack, uint8_t use_ltran_fla if (likely(transfer_type == TRANSFER_CURRENT_THREAD)) { eth_dev_recv(stack->pkts[i], stack); - } else if (transfer_type == TRANSFER_KERNEL) { kni_handle_tx(stack->pkts[i]); } else { - /*transfer to other thread*/ - } + /* transfer to other thread */ + } } stack->stats.rx += nr_pkts; diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c index aef6035..5ea1f31 100644 --- a/src/lstack/netif/lstack_vdev.c +++ b/src/lstack/netif/lstack_vdev.c @@ -43,7 +43,7 @@ #define IPV4_MASK (0xf0) #define IPV4_VERION (0x40) -#define TCP_HDR_LEN(tcp_hdr) ((tcp_hdr->data_off & 0xf0) >> 2) +#define TCP_HDR_LEN(tcp_hdr) (((tcp_hdr)->data_off & 0xf0) >> 2) static uint32_t ltran_rx_poll(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t max_mbuf) { @@ -156,7 +156,8 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) if (get_global_cfg_params()->is_primary) { delete_user_process_port(qtuple->src_port, PORT_LISTEN); } else { - transfer_add_or_delete_listen_port_to_process0(qtuple->src_port,get_global_cfg_params()->process_idx, 0); + transfer_add_or_delete_listen_port_to_process0(qtuple->src_port, + get_global_cfg_params()->process_idx, 0); } } @@ -174,13 +175,15 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) if (type == REG_RING_TCP_CONNECT) { uint16_t queue_id = get_protocol_stack()->queue_id; - if (get_global_cfg_params()->is_primary){ + if (get_global_cfg_params()->is_primary) { add_user_process_port(qtuple->src_port, get_global_cfg_params()->process_idx, PORT_CONNECT); if (queue_id != 0) { - transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, + qtuple->src_port, qtuple->dst_port); } } else { - transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, + qtuple->src_port, qtuple->dst_port); } } @@ -188,7 +191,8 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) if (get_global_cfg_params()->is_primary) { add_user_process_port(qtuple->src_port, get_global_cfg_params()->process_idx, PORT_LISTEN); } else { - transfer_add_or_delete_listen_port_to_process0(qtuple->src_port, get_global_cfg_params()->process_idx, 1); + transfer_add_or_delete_listen_port_to_process0(qtuple->src_port, + get_global_cfg_params()->process_idx, 1); } } return 0; diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c index 944675e..e41c6db 100644 --- a/src/ltran/ltran_dfx.c +++ b/src/ltran/ltran_dfx.c @@ -138,19 +138,17 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg static const char *nic_stats_border = "########################"; printf("###### NIC extended statistics for port %-2d #########\n", xstats->port_id); - printf("%s############################\n",nic_stats_border); + printf("%s############################\n", nic_stats_border); if (xstats->len <= 0 || xstats->len > RTE_ETH_XSTATS_MAX_LEN) { printf("xstats item(%d) num error!\n", xstats->len); return; } for (uint32_t i = 0; i < xstats->len; i++) { - printf("%s: %"PRIu64"\n", xstats->xstats_name[i].name, - xstats->values[i]); + printf("%s: %"PRIu64"\n", xstats->xstats_name[i].name, xstats->values[i]); } - printf("%s############################\n", - nic_stats_border); + printf("%s############################\n", nic_stats_border); } static void gazelle_print_ltran_conn(void *buf, const struct gazelle_stat_msg_request *req_msg) @@ -1110,7 +1108,7 @@ static void gazelle_print_lstack_aggregate(void *buf, const struct gazelle_stat_ break; } ret = dfx_stat_read_from_ltran(buf, sizeof(struct gazelle_stack_dfx_data), req_msg->stat_mode); - } while(true); + } while (true); } static int32_t parse_dfx_ltran_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg) diff --git a/src/ltran/ltran_monitor.c b/src/ltran/ltran_monitor.c index bd047fc..d163a47 100644 --- a/src/ltran/ltran_monitor.c +++ b/src/ltran/ltran_monitor.c @@ -31,8 +31,8 @@ #include "ltran_instance.h" #include "gazelle_dfx_msg.h" #include "gazelle_base_func.h" -#include "ltran_monitor.h" #include "ltran_param.h" +#include "ltran_monitor.h" #define GAZELLE_LISTEN_BACKLOG 5 diff --git a/src/ltran/ltran_param.c b/src/ltran/ltran_param.c index 1eb4f48..e16e648 100644 --- a/src/ltran/ltran_param.c +++ b/src/ltran/ltran_param.c @@ -20,12 +20,12 @@ #include #include -#include "ltran_param.h" #include "ltran_errno.h" #include "ltran_base.h" #include "ltran_log.h" #include "gazelle_dfx_msg.h" #include "gazelle_base_func.h" +#include "ltran_param.h" #define HEX_BASE 16 diff --git a/src/ltran/main.c b/src/ltran/main.c index 03b3ad5..87f1e14 100644 --- a/src/ltran/main.c +++ b/src/ltran/main.c @@ -61,11 +61,11 @@ static void sig_default_handler(int32_t sig) int ret = 0; ret = unlink(get_ltran_config()->unix_socket_filename); if (ret) { - LTRAN_WARN("unlink %s ERROR. errn: %d. ret=%d\n", get_ltran_config()->unix_socket_filename, errno, ret); + LTRAN_WARN("unlink %s ERROR. errn: %d. ret=%d\n", get_ltran_config()->unix_socket_filename, errno, ret); } ret = unlink(get_ltran_config()->dfx_socket_filename); if (ret) { - LTRAN_WARN("unlink %s ERROR. errn: %d. ret=%d\n", get_ltran_config()->dfx_socket_filename, errno, ret); + LTRAN_WARN("unlink %s ERROR. errn: %d. ret=%d\n", get_ltran_config()->dfx_socket_filename, errno, ret); } kill(getpid(), sig); } -- 2.33.0