From fa4c8d298f37e7dad01d91cec6834238841d81b3 Mon Sep 17 00:00:00 2001 From: jiangheng12 Date: Fri, 24 Mar 2023 11:12:34 +0800 Subject: [PATCH] check primary process idx and secondary lstack num --- src/common/gazelle_dfx_msg.h | 5 +- src/lstack/api/lstack_wrap.c | 9 ++- src/lstack/core/lstack_cfg.c | 5 ++ src/lstack/core/lstack_init.c | 8 ++ src/lstack/core/lstack_lwip.c | 20 ++--- src/lstack/include/lstack_cfg.h | 1 + src/lstack/include/lstack_ethdev.h | 1 + src/lstack/netif/lstack_ethdev.c | 126 ++++++++++++++++++----------- src/ltran/ltran_dfx.c | 4 +- 9 files changed, 112 insertions(+), 67 deletions(-) diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h index 1863837..9105871 100644 --- a/src/common/gazelle_dfx_msg.h +++ b/src/common/gazelle_dfx_msg.h @@ -200,13 +200,14 @@ struct gazelle_stat_low_power_info { }; #define RTE_ETH_XSTATS_NAME_SIZE 64 +#define RTE_ETH_XSTATS_MAX_LEN 128 struct nic_eth_xstats_name { char name[RTE_ETH_XSTATS_NAME_SIZE]; }; struct nic_eth_xstats { - struct nic_eth_xstats_name xstats_name[128]; - uint64_t values[128]; + struct nic_eth_xstats_name xstats_name[RTE_ETH_XSTATS_MAX_LEN]; + uint64_t values[RTE_ETH_XSTATS_MAX_LEN]; uint32_t len; uint16_t port_id; }; diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c index 5234c19..5132ee9 100644 --- a/src/lstack/api/lstack_wrap.c +++ b/src/lstack/api/lstack_wrap.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -187,8 +188,8 @@ static int get_addr(struct sockaddr_in *sin, char *interface) if ((sockfd = posix_api->socket_fn(AF_INET, SOCK_STREAM, 0)) < 0) return -1; - memset(&ifr, 0, sizeof(ifr)); - snprintf(ifr.ifr_name, (sizeof(ifr.ifr_name) - 1), "%s", interface); + memset_s(&ifr, sizeof(ifr), 0, sizeof(ifr)); + snprintf_s(ifr.ifr_name, sizeof(ifr.ifr_name), (sizeof(ifr.ifr_name) - 1), "%s", interface); if(posix_api->ioctl_fn(sockfd, SIOCGIFADDR, &ifr) < 0){ posix_api->close_fn(sockfd); @@ -243,7 +244,7 @@ bool is_dst_ip_localhost(const struct sockaddr *addr) char interface[20] = {0}; strncpy(interface, p, n); - memset(sin, 0, sizeof(struct sockaddr_in)); + memset_s(sin, sizeof(struct sockaddr_in), 0, sizeof(struct sockaddr_in)); int ret = get_addr(sin, interface); if (ret == 0) { if(sin->sin_addr.s_addr == servaddr->sin_addr.s_addr){ @@ -278,7 +279,7 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name int32_t ret = 0; char listen_ring_name[RING_NAME_LEN]; int remote_port = htons(((struct sockaddr_in *)name)->sin_port); - snprintf(listen_ring_name, sizeof(listen_ring_name), "listen_rx_ring_%u", remote_port); + snprintf_s(listen_ring_name, sizeof(listen_ring_name), sizeof(listen_ring_name) - 1, "listen_rx_ring_%u", remote_port); if (is_dst_ip_localhost(name) && rte_ring_lookup(listen_ring_name) == NULL) { ret = posix_api->connect_fn(s, name, namelen); SET_CONN_TYPE_HOST(sock->conn); diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c index f033fa7..95590e2 100644 --- a/src/lstack/core/lstack_cfg.c +++ b/src/lstack/core/lstack_cfg.c @@ -1040,5 +1040,10 @@ static int parse_tuple_filter(void) if (g_config_params.use_ltran || g_config_params.listen_shadow) { return -EINVAL; } + + // check primary process_idx + if (g_config_params.is_primary && g_config_params.process_idx != 0) { + return -EINVAL; + } return 0; } diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c index c40938d..76dd384 100644 --- a/src/lstack/core/lstack_init.c +++ b/src/lstack/core/lstack_init.c @@ -324,6 +324,14 @@ __attribute__((constructor)) void gazelle_network_init(void) return; } + /** + * check lstack num, and get process idx + */ + if (check_params_from_primary() < 0) { + LSTACK_PRE_LOG(LSTACK_ERR, "lstack num error, not same to primary process!\n"); + LSTACK_EXIT(1, "lstack num error, not same to primary process!\n"); + } + /* * save initial affinity */ if (!get_global_cfg_params()->main_thread_affinity) { diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index b4a08ea..a4e6e0b 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -1386,7 +1386,7 @@ err_t netif_loop_output(struct netif *netif, struct pbuf *p) if ((flags & TCP_SYN) && !(flags & TCP_ACK)) { /* SYN packet, send to listen_ring */ char ring_name[RING_NAME_LEN] = {0}; - snprintf(ring_name, sizeof(ring_name), "listen_rx_ring_%d", pcb->remote_port); + snprintf_s(ring_name, sizeof(ring_name), sizeof(ring_name) - 1, "listen_rx_ring_%d", pcb->remote_port); struct rte_ring *ring = rte_ring_lookup(ring_name); if (ring == NULL) { LSTACK_LOG(INFO, LSTACK, "netif_loop_output: cant find listen_rx_ring %d\n", pcb->remote_port); @@ -1411,7 +1411,7 @@ err_t netif_loop_output(struct netif *netif, struct pbuf *p) err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock) { char name[RING_NAME_LEN]; - snprintf(name, sizeof(name), "rte_mz_rx_%u", pcb->remote_port); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_rx_%u", pcb->remote_port); if ((nsock->same_node_tx_ring_mz = rte_memzone_lookup(name)) == NULL) { LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); return -1; @@ -1420,13 +1420,13 @@ err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock) } nsock->same_node_tx_ring = (struct same_node_ring *)nsock->same_node_tx_ring_mz->addr; - snprintf(name, sizeof(name), "rte_mz_buf_rx_%u", pcb->remote_port); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_buf_rx_%u", pcb->remote_port); if ((nsock->same_node_tx_ring->mz = rte_memzone_lookup(name)) == NULL) { LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); return -1; } - snprintf(name, sizeof(name), "rte_mz_tx_%u", pcb->remote_port); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "rte_mz_tx_%u", pcb->remote_port); if ((nsock->same_node_rx_ring_mz = rte_memzone_lookup(name)) == NULL) { LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); return -1; @@ -1435,7 +1435,7 @@ err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock) } nsock->same_node_rx_ring = (struct same_node_ring *)nsock->same_node_rx_ring_mz->addr; - snprintf(name, sizeof(name), "rte_mz_buf_tx_%u", pcb->remote_port); + snprintf_s(name, sizeof(name), sizeof(name) - 1,"rte_mz_buf_tx_%u", pcb->remote_port); if ((nsock->same_node_rx_ring->mz = rte_memzone_lookup(name)) == NULL) { LSTACK_LOG(INFO, LSTACK, "lwip_accept: can't find %s\n",name); return -1; @@ -1450,7 +1450,7 @@ err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock) err_t same_node_memzone_create(const struct rte_memzone **zone, int size, int port, char *name, char *rx) { char mem_name[RING_NAME_LEN] = {0}; - snprintf(mem_name, sizeof(mem_name), "%s_%s_%u", name, rx, port); + snprintf_s(mem_name, sizeof(mem_name), sizeof(mem_name) - 1, "%s_%s_%u", name, rx, port); *zone = rte_memzone_reserve_aligned(mem_name, size, rte_socket_id(), 0, RTE_CACHE_LINE_SIZE); if (*zone == NULL) { @@ -1473,7 +1473,7 @@ err_t same_node_ring_create(struct rte_ring **ring, int size, int port, char *na flags = RING_F_SP_ENQ | RING_F_SC_DEQ; } - snprintf(ring_name, sizeof(ring_name), "%s_%s_ring_%u", name, rx, port); + snprintf_s(ring_name, sizeof(ring_name), sizeof(ring_name) - 1, "%s_%s_ring_%u", name, rx, port); *ring = rte_ring_create(ring_name, size, rte_socket_id(), flags); if (*ring == NULL) { LSTACK_LOG(ERR, LSTACK, "cannot create rte_ring %s, errno is %d\n", ring_name, rte_errno); @@ -1550,10 +1550,10 @@ END: err_t find_same_node_ring(struct tcp_pcb *npcb) { char name[RING_NAME_LEN] = {0}; - snprintf(name, sizeof(name), "client_tx_ring_%u", npcb->remote_port); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_tx_ring_%u", npcb->remote_port); npcb->client_rx_ring = rte_ring_lookup(name); - memset(name, 0, sizeof(name)); - snprintf(name, sizeof(name), "client_rx_ring_%u", npcb->remote_port); + memset_s(name, sizeof(name), 0, sizeof(name)); + snprintf_s(name, sizeof(name), sizeof(name) - 1, "client_rx_ring_%u", npcb->remote_port); npcb->client_tx_ring = rte_ring_lookup(name); npcb->free_ring = 0; if (npcb->client_tx_ring == NULL || diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h index 16f37b4..a4170ca 100644 --- a/src/lstack/include/lstack_cfg.h +++ b/src/lstack/include/lstack_cfg.h @@ -34,6 +34,7 @@ #define LOG_DIR_PATH PATH_MAX #define LOG_LEVEL_LEN 16 #define GAZELLE_MAX_NUMA_NODES 8 +#define MAX_PROCESS_NUM 32 /* Default value of low power mode parameters */ #define LSTACK_LPM_DETECT_MS_MIN (5 * 1000) diff --git a/src/lstack/include/lstack_ethdev.h b/src/lstack/include/lstack_ethdev.h index 39a972d..7f944eb 100644 --- a/src/lstack/include/lstack_ethdev.h +++ b/src/lstack/include/lstack_ethdev.h @@ -45,6 +45,7 @@ int32_t gazelle_eth_dev_poll(struct protocol_stack *stack, uint8_t use_ltran_fla void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack); int recv_pkts_from_other_process(int process_index, void* arg); +int32_t check_params_from_primary(void); void kni_handle_rx(uint16_t port_id); void delete_user_process_port(uint16_t dst_port, enum port_type type); void add_user_process_port(uint16_t dst_port, uint8_t process_idx, enum port_type type); diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index b35852b..8d05bdd 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -58,6 +58,9 @@ #define ERROR_REPLY "error" #define PACKET_READ_SIZE 32 +#define GET_LSTACK_NUM 14 +#define GET_LSTACK_NUM_STRING "get_lstack_num" + char *client_path = "/var/run/gazelle/client.socket"; char *server_path = "/var/run/gazelle/server.socket"; const char *split_delim = ","; @@ -171,7 +174,7 @@ void add_rule(char* rule_key, struct rte_flow *flow) HASH_FIND_STR(g_flow_rules, rule_key, rule); if (rule == NULL) { rule = (struct flow_rule*)malloc(sizeof(struct flow_rule)); - strcpy(rule->rule_key, rule_key); + strcpy_s(rule->rule_key, RULE_KEY_LEN, rule_key); HASH_ADD_STR(g_flow_rules, rule_key, rule); } rule->flow = flow; @@ -202,20 +205,26 @@ int transfer_pkt_to_other_process(char *buf, int process_index, int write_len, b sockfd = posix_api->socket_fn(AF_UNIX, SOCK_STREAM, 0); - memset(&serun, 0, sizeof(serun)); + memset_s(&serun, sizeof(serun), 0, sizeof(serun)); serun.sun_family = AF_UNIX; sprintf_s(serun.sun_path, PATH_MAX,"%s%d", server_path, process_index); - int len = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); + int32_t len = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); if (posix_api->connect_fn(sockfd, (struct sockaddr *)&serun, len) < 0){ return CONNECT_ERROR; } posix_api->write_fn(sockfd, buf, write_len); if (need_reply) { char reply_message[REPLY_LEN]; - posix_api->read_fn(sockfd, reply_message, REPLY_LEN); - if (strcmp(reply_message, SUCCESS_REPLY) == 0) { - ret = TRANSFER_SUCESS; - }else { + int32_t read_result = posix_api->read_fn(sockfd, reply_message, REPLY_LEN); + if (read_result > 0) { + if (strcmp(reply_message, SUCCESS_REPLY) == 0) { + ret = TRANSFER_SUCESS; + } else if (strcmp(reply_message, ERROR_REPLY) == 0) { + ret = REPLY_ERROR; + } else { + ret = atoi(reply_message); + } + } else { ret = REPLY_ERROR; } } @@ -224,6 +233,21 @@ int transfer_pkt_to_other_process(char *buf, int process_index, int write_len, b return ret; } +int32_t check_params_from_primary(void){ + struct cfg_params *cfg = get_global_cfg_params(); + if (cfg->is_primary) { + return 0; + } + // check lstack num + char get_lstack_num[GET_LSTACK_NUM]; + sprintf_s(get_lstack_num, GET_LSTACK_NUM, "%s", GET_LSTACK_NUM_STRING); + int32_t ret = transfer_pkt_to_other_process(get_lstack_num, 0, GET_LSTACK_NUM, true); + if (ret != cfg->num_cpu) { + return -1; + } + return 0; +} + struct rte_flow * create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port, struct rte_flow_error *error) @@ -240,14 +264,14 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 struct rte_flow_item_tcp tcp_mask; int res; - memset(pattern, 0, sizeof(pattern)); - memset(action, 0, sizeof(action)); + memset_s(pattern, sizeof(pattern), 0, sizeof(pattern)); + memset_s(action, sizeof(action), 0, sizeof(action)); /* * set the rule attribute. * in this case only ingress packets will be checked. */ - memset(&attr, 0, sizeof(struct rte_flow_attr)); + memset_s(&attr, sizeof(struct rte_flow_attr), 0, sizeof(struct rte_flow_attr)); attr.ingress = 1; /* @@ -262,8 +286,8 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; // ip header - memset(&ip_spec, 0, sizeof(struct rte_flow_item_ipv4)); - memset(&ip_mask, 0, sizeof(struct rte_flow_item_ipv4)); + memset_s(&ip_spec, sizeof(struct rte_flow_item_ipv4), 0, sizeof(struct rte_flow_item_ipv4)); + memset_s(&ip_mask, sizeof(struct rte_flow_item_ipv4), 0, sizeof(struct rte_flow_item_ipv4)); ip_spec.hdr.dst_addr = dst_ip; ip_mask.hdr.dst_addr = FULL_MASK; ip_spec.hdr.src_addr = src_ip; @@ -273,8 +297,8 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 pattern[1].mask = &ip_mask; // tcp header, full mask 0xffff - memset(&tcp_spec, 0, sizeof(struct rte_flow_item_tcp)); - memset(&tcp_mask, 0, sizeof(struct rte_flow_item_tcp)); + memset_s(&tcp_spec, sizeof(struct rte_flow_item_tcp), 0, sizeof(struct rte_flow_item_tcp)); + memset_s(&tcp_mask, sizeof(struct rte_flow_item_tcp), 0, sizeof(struct rte_flow_item_tcp)); pattern[2].type = RTE_FLOW_ITEM_TYPE_TCP; tcp_spec.hdr.src_port = src_port; tcp_spec.hdr.dst_port = dst_port; @@ -392,7 +416,7 @@ static int str_to_array(char *args, uint32_t *array, int size) char *elem = NULL; char *next_token = NULL; - memset(array, 0, sizeof(*array) * size); + memset_s(array, sizeof(*array) * size, 0, sizeof(*array) * size); elem = strtok_s((char *)args, split_delim, &next_token); while (elem != NULL) { if (cnt >= size) { @@ -548,50 +572,50 @@ int recv_pkts_from_other_process(int process_index, void* arg){ int listenfd, connfd, size; char buf[132]; /* socket */ - if ((listenfd = posix_api->socket_fn(AF_UNIX, SOCK_STREAM, 0)) < 0) { - perror("socket error"); - return -1; - } + if ((listenfd = posix_api->socket_fn(AF_UNIX, SOCK_STREAM, 0)) < 0) { + perror("socket error"); + return -1; + } /* bind */ - memset(&serun, 0, sizeof(serun)); - serun.sun_family = AF_UNIX; + memset_s(&serun, sizeof(serun), 0, sizeof(serun)); + serun.sun_family = AF_UNIX; char process_server_path[PATH_MAX]; sprintf_s(process_server_path, sizeof(process_server_path), "%s%d", server_path, process_index); - strcpy(serun.sun_path, process_server_path); - size = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); - unlink(process_server_path); + strcpy_s(serun.sun_path, sizeof(serun.sun_path),process_server_path); + size = offsetof(struct sockaddr_un, sun_path) + strlen(serun.sun_path); + unlink(process_server_path); if (posix_api->bind_fn(listenfd, (struct sockaddr *)&serun, size) < 0) { - perror("bind error"); - return -1; + perror("bind error"); + return -1; } - if (posix_api->listen_fn(listenfd, 20) < 0) { + if (posix_api->listen_fn(listenfd, 20) < 0) { /* 20: max backlog */ perror("listen error"); - return -1; + return -1; } sem_post((sem_t *)arg); /* block */ while(1) { - cliun_len = sizeof(cliun); - if ((connfd = posix_api->accept_fn(listenfd, (struct sockaddr *)&cliun, &cliun_len)) < 0){ - perror("accept error"); - continue; - } - while(1) { - int n = posix_api->read_fn(connfd, buf, sizeof(buf)); - if (n < 0) { - perror("read error"); - break; - } else if(n == 0) { - break; + cliun_len = sizeof(cliun); + if ((connfd = posix_api->accept_fn(listenfd, (struct sockaddr *)&cliun, &cliun_len)) < 0){ + perror("accept error"); + continue; + } + while(1) { + int n = posix_api->read_fn(connfd, buf, sizeof(buf)); + if (n < 0) { + perror("read error"); + break; + } else if(n == 0) { + break; } if(n == LSTACK_MBUF_LEN){ /* arp */ parse_arp_and_transefer(buf); - }else if(n == TRANSFER_TCP_MUBF_LEN) { + } else if (n == TRANSFER_TCP_MUBF_LEN) { /* tcp. lstack_mbuf_queue_id */ parse_tcp_and_transefer(buf); - }else if (n == DELETE_FLOWS_PARAMS_LENGTH) { + } else if (n == DELETE_FLOWS_PARAMS_LENGTH) { /* delete rule */ parse_and_delete_rule(buf); }else if(n == CREATE_FLOWS_PARAMS_LENGTH){ @@ -599,20 +623,24 @@ int recv_pkts_from_other_process(int process_index, void* arg){ parse_and_create_rule(buf); char reply_buf[REPLY_LEN]; sprintf_s(reply_buf, sizeof(reply_buf), "%s", SUCCESS_REPLY); - posix_api->write_fn(connfd, reply_buf, REPLY_LEN); - }else { + posix_api->write_fn(connfd, reply_buf, REPLY_LEN); + }else if (n == GET_LSTACK_NUM) { + char reply_buf[REPLY_LEN]; + sprintf_s(reply_buf, sizeof(reply_buf), "%d", get_global_cfg_params()->num_cpu); + posix_api->write_fn(connfd, reply_buf, REPLY_LEN); + }else{ /* add port */ parse_and_add_or_delete_listen_port(buf); char reply_buf[REPLY_LEN]; sprintf_s(reply_buf, sizeof(reply_buf), "%s", SUCCESS_REPLY); - posix_api->write_fn(connfd, reply_buf, REPLY_LEN); + posix_api->write_fn(connfd, reply_buf, REPLY_LEN); } - } - posix_api->close_fn(connfd); + } + posix_api->close_fn(connfd); } - posix_api->close_fn(listenfd); - return 0; + posix_api->close_fn(listenfd); + return 0; } void concat_mbuf_and_queue_id(struct rte_mbuf *mbuf, uint16_t queue_id, char* mbuf_and_queue_id, int write_len){ diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c index 331a23c..9180e89 100644 --- a/src/ltran/ltran_dfx.c +++ b/src/ltran/ltran_dfx.c @@ -139,8 +139,8 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg printf("###### NIC extended statistics for port %-2d #########\n", 0); printf("%s############################\n",nic_stats_border); - if (xstats->len <= 0) { - printf("Cannot get xstats\n"); + if (xstats->len <= 0 || xstats->len > RTE_ETH_XSTATS_MAX_LEN) { + printf("xstats item(%d) num error!\n", xstats->len); return; } -- 2.23.0