From 59c658fa0f75748fa7b49170b9fc34267ef1797c Mon Sep 17 00:00:00 2001 From: jiangheng12 Date: Mon, 3 Apr 2023 14:42:53 +0800 Subject: [PATCH] kni down not stop nic --- src/common/dpdk_common.c | 37 +++++++++++-------------- src/common/dpdk_common.h | 2 +- src/lstack/core/lstack_cfg.c | 8 +++--- src/lstack/core/lstack_protocol_stack.c | 10 ++++--- src/lstack/netif/lstack_ethdev.c | 8 ++++-- src/ltran/ltran_forward.c | 5 ++-- 6 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/common/dpdk_common.c b/src/common/dpdk_common.c index f5a20dd..23c96d6 100644 --- a/src/common/dpdk_common.c +++ b/src/common/dpdk_common.c @@ -34,16 +34,12 @@ #define COMMON_INFO(fmt, ...) LSTACK_LOG(INFO, LSTACK, fmt, ##__VA_ARGS__) #endif -static pthread_mutex_t g_kni_mutex = PTHREAD_MUTEX_INITIALIZER; struct rte_kni *g_pkni = NULL; +static volatile bool g_kni_started = false; -/* - * lock for preventing data race between tx thread and down operation. - * Don't need to add lock on rx because down operation and rx are in the same thread - */ -pthread_mutex_t *get_kni_mutex(void) +bool get_kni_started(void) { - return &g_kni_mutex; + return g_kni_started; } struct rte_kni* get_gazelle_kni(void) @@ -62,23 +58,18 @@ static int32_t kni_config_network_interface(uint16_t port_id, uint8_t if_up) } if (if_up != 0) { /* Configure network interface up */ - if (!g_bond_dev_started) { - pthread_mutex_lock(&g_kni_mutex); - ret = rte_eth_dev_start(port_id); - pthread_mutex_unlock(&g_kni_mutex); - if (ret < 0) { - COMMON_ERR("Failed to start port %hu ret=%d\n", port_id, ret); + if (!g_kni_started) { + g_kni_started = true; + if (!g_bond_dev_started) { + rte_eth_dev_start(port_id); + g_bond_dev_started = true; } - g_bond_dev_started = true; } else { COMMON_INFO("trying to start a started dev. \n"); } } else { /* Configure network interface down */ - if (g_bond_dev_started) { - pthread_mutex_lock(&g_kni_mutex); - rte_eth_dev_stop(port_id); - pthread_mutex_unlock(&g_kni_mutex); - g_bond_dev_started = false; + if (g_kni_started) { + g_kni_started = false; } else { COMMON_INFO("trying to stop a stopped dev. \n"); } @@ -201,6 +192,12 @@ void dpdk_kni_release(void) int32_t kni_process_tx(struct rte_mbuf **pkts_burst, uint32_t count) { uint32_t i; + if (!g_kni_started) { + for (i = 0; i < count; i++) { + rte_pktmbuf_free(pkts_burst[i]); + } + return 0; + } for (i = 0; i < count; ++i) { struct rte_ipv4_hdr * ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkts_burst[i], char*) @@ -227,9 +224,7 @@ void kni_process_rx(uint16_t port) nb_kni_rx = rte_kni_rx_burst(g_pkni, pkts_burst, GAZELLE_KNI_READ_SIZE); if (nb_kni_rx > 0) { - pthread_mutex_lock(&g_kni_mutex); nb_rx = rte_eth_tx_burst(port, 0, pkts_burst, nb_kni_rx); - pthread_mutex_unlock(&g_kni_mutex); for (i = nb_rx; i < nb_kni_rx; ++i) { rte_pktmbuf_free(pkts_burst[i]); diff --git a/src/common/dpdk_common.h b/src/common/dpdk_common.h index 6b107ae..2f0e8d1 100644 --- a/src/common/dpdk_common.h +++ b/src/common/dpdk_common.h @@ -94,7 +94,7 @@ static __rte_always_inline void time_stamp_into_mbuf(uint32_t rx_count, struct r } } -pthread_mutex_t *get_kni_mutex(void); +bool get_kni_started(void); struct rte_kni* get_gazelle_kni(void); int32_t dpdk_kni_init(uint16_t port, struct rte_mempool *pool); int32_t kni_process_tx(struct rte_mbuf **pkts_burst, uint32_t count); diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c index 168aa49..8558121 100644 --- a/src/lstack/core/lstack_cfg.c +++ b/src/lstack/core/lstack_cfg.c @@ -1026,6 +1026,10 @@ static int parse_process_index(void) } } else { g_config_params.process_idx = (uint8_t)config_setting_get_int(process_idx); + if (g_config_params.is_primary && g_config_params.process_idx != 0 || + !g_config_params.is_primary && g_config_params.process_idx == 0) { + return -EINVAL; + } } return 0; @@ -1045,9 +1049,5 @@ static int parse_tuple_filter(void) return -EINVAL; } - // check primary process_idx - if (g_config_params.is_primary && g_config_params.process_idx != 0) { - return -EINVAL; - } return 0; } diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index 76914f8..a858b37 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -483,7 +483,9 @@ static void* gazelle_stack_thread(void *arg) * so processing KNI requests only in the thread with queue_id No.0 is sufficient. */ if (kni_switch && !queue_id && !(wakeup_tick & 0xfff)) { rte_kni_handle_request(get_gazelle_kni()); - kni_handle_rx(get_port_id()); + if (get_kni_started()) { + kni_handle_rx(get_port_id()); + } } wakeup_tick++; @@ -557,9 +559,9 @@ int32_t init_protocol_stack(void) if (get_global_cfg_params()->is_primary) { for (uint16_t idx = 0; idx < get_global_cfg_params()->tot_queue_num; idx++) { - struct rte_mempool* rxtx_mbuf = create_pktmbuf_mempool("rxtx_mbuf", - get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count / stack_group->stack_num, RXTX_CACHE_SZ, idx); - get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx] = rxtx_mbuf; + struct rte_mempool* rxtx_mbuf = create_pktmbuf_mempool("rxtx_mbuf", + get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count / stack_group->stack_num, RXTX_CACHE_SZ, idx); + get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx] = rxtx_mbuf; } } diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index e26fe30..4103f22 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -719,7 +719,9 @@ void kni_handle_rx(uint16_t port_id) void kni_handle_tx(struct rte_mbuf *mbuf) { - if (!get_global_cfg_params()->kni_switch) { + if (!get_global_cfg_params()->kni_switch || + !get_kni_started()) { + rte_pktmbuf_free(mbuf); return; } struct rte_ipv4_hdr *ipv4_hdr; @@ -776,9 +778,9 @@ int32_t gazelle_eth_dev_poll(struct protocol_stack *stack, uint8_t use_ltran_fla if (likely(transfer_type == TRANSFER_CURRENT_THREAD)) { eth_dev_recv(stack->pkts[i], stack); - }else if (transfer_type == TRANSFER_KERNEL) { + } else if (transfer_type == TRANSFER_KERNEL) { kni_handle_tx(stack->pkts[i]); - }else { + } else { /*transfer to other thread*/ } } diff --git a/src/ltran/ltran_forward.c b/src/ltran/ltran_forward.c index 8629acb..b41e1e2 100644 --- a/src/ltran/ltran_forward.c +++ b/src/ltran/ltran_forward.c @@ -690,7 +690,6 @@ static __rte_always_inline void downstream_forward_one(struct gazelle_stack *sta /* send packets anyway. */ tx_pkts = 0; - pthread_mutex_lock(get_kni_mutex()); while (tx_pkts < used_cnt) { tx_pkts += rte_eth_tx_burst(port_id, queue_id, (struct rte_mbuf **)(&dst_bufs[tx_pkts]), @@ -702,7 +701,6 @@ static __rte_always_inline void downstream_forward_one(struct gazelle_stack *sta } } } - pthread_mutex_unlock(get_kni_mutex()); get_statistics()->port_stats[g_port_index].tx_bytes += tx_bytes; get_statistics()->port_stats[g_port_index].tx += tx_pkts; @@ -737,7 +735,8 @@ int32_t downstream_forward(uint16_t *port) while (get_ltran_stop_flag() != GAZELLE_TRUE) { /* kni rx means read from kni and send to nic */ - if (get_ltran_config()->dpdk.kni_switch == GAZELLE_ON) { + if (get_ltran_config()->dpdk.kni_switch == GAZELLE_ON && + get_kni_started()) { kni_process_rx(g_port_index); } -- 2.33.0