From ac2c6340d9e08b94bf99682e12ab5d5836d2140d Mon Sep 17 00:00:00 2001 From: wu-changsheng Date: Mon, 31 Oct 2022 22:07:26 +0800 Subject: [PATCH 5/7] optimize variable access --- src/common/gazelle_opt.h | 1 + src/lstack/api/lstack_wrap.c | 8 +- src/lstack/core/lstack_lwip.c | 102 +++++++++------------ src/lstack/core/lstack_protocol_stack.c | 24 +++-- src/lstack/core/lstack_thread_rpc.c | 2 + src/lstack/include/lstack_protocol_stack.h | 1 + src/lstack/netif/lstack_ethdev.c | 5 +- src/lstack/netif/lstack_vdev.c | 7 +- 8 files changed, 67 insertions(+), 83 deletions(-) diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h index 011553c..8ab40ed 100644 --- a/src/common/gazelle_opt.h +++ b/src/common/gazelle_opt.h @@ -31,6 +31,7 @@ #define GAZELLE_MBUFF_PRIV_SIZE (sizeof(uint64_t) * 2) #define DEFAULT_RING_SIZE (512) +#define DEFAULT_RING_MASK (511) #define DEFAULT_BACKUP_RING_SIZE_FACTOR (16) #define VDEV_RX_QUEUE_SZ DEFAULT_RING_SIZE diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c index 9672d3d..1c7a722 100644 --- a/src/lstack/api/lstack_wrap.c +++ b/src/lstack/api/lstack_wrap.c @@ -46,6 +46,10 @@ bool select_thread_path(void); static enum KERNEL_LWIP_PATH select_path(int fd) { + if (!select_thread_path()) { + return PATH_KERNEL; + } + if (unlikely(posix_api == NULL)) { /* posix api maybe call before gazelle init */ if (posix_api_init() != 0) { @@ -54,10 +58,6 @@ static enum KERNEL_LWIP_PATH select_path(int fd) return PATH_KERNEL; } - if (!select_thread_path()) { - return PATH_KERNEL; - } - if (unlikely(posix_api->ues_posix)) { return PATH_KERNEL; } diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index 3a1eb81..f924ee7 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -37,8 +37,6 @@ #define HALF_DIVISOR (2) #define USED_IDLE_WATERMARK (VDEV_IDLE_QUEUE_SZ >> 2) -static int32_t lwip_alloc_pbufs(pbuf_layer layer, uint16_t length, pbuf_type type, void **pbufs, uint32_t num); - static void free_ring_pbuf(struct rte_ring *ring) { void *pbufs[SOCK_RECV_RING_SIZE]; @@ -85,17 +83,38 @@ static void reset_sock_data(struct lwip_sock *sock) sock->recv_lastdata = NULL; } -static void replenish_send_idlembuf(struct rte_ring *ring) +static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, uint16_t length, pbuf_type type) +{ + struct pbuf_custom *pbuf_custom = mbuf_to_pbuf(mbuf); + pbuf_custom->custom_free_function = gazelle_free_pbuf; + + void *data = rte_pktmbuf_mtod(mbuf, void *); + struct pbuf *pbuf = pbuf_alloced_custom(layer, length, type, pbuf_custom, data, MAX_PACKET_SZ); + if (pbuf) { + pbuf->ol_flags = 0; + pbuf->l2_len = 0; + pbuf->l3_len = 0; + } + + return pbuf; +} + +static void replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring) { void *pbuf[SOCK_SEND_RING_SIZE]; uint32_t replenish_cnt = gazelle_ring_free_count(ring); uint32_t alloc_num = LWIP_MIN(replenish_cnt, RING_SIZE(SOCK_SEND_RING_SIZE)); - if (lwip_alloc_pbufs(PBUF_TRANSPORT, TCP_MSS, PBUF_RAM, (void **)pbuf, alloc_num) != 0) { + if (rte_pktmbuf_alloc_bulk(stack->tx_pktmbuf_pool, (struct rte_mbuf **)pbuf, alloc_num) != 0) { + stack->stats.tx_allocmbuf_fail++; return; } + for (uint32_t i = 0; i < alloc_num; i++) { + pbuf[i] = init_mbuf_to_pbuf(pbuf[i], PBUF_TRANSPORT, TCP_MSS, PBUF_RAM); + } + uint32_t num = gazelle_ring_sp_enqueue(ring, pbuf, alloc_num); for (uint32_t i = num; i < alloc_num; i++) { pbuf_free(pbuf[i]); @@ -126,7 +145,7 @@ void gazelle_init_sock(int32_t fd) LSTACK_LOG(ERR, LSTACK, "sock_send create failed. errno: %d.\n", rte_errno); return; } - replenish_send_idlembuf(sock->send_ring); + replenish_send_idlembuf(stack, sock->send_ring); sock->stack = stack; sock->stack->conn_num++; @@ -183,46 +202,17 @@ int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, return 0; } -static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, uint16_t length, pbuf_type type) -{ - struct pbuf_custom *pbuf_custom = mbuf_to_pbuf(mbuf); - pbuf_custom->custom_free_function = gazelle_free_pbuf; - - void *data = rte_pktmbuf_mtod(mbuf, void *); - struct pbuf *pbuf = pbuf_alloced_custom(layer, length, type, pbuf_custom, data, MAX_PACKET_SZ); - if (pbuf) { - pbuf->ol_flags = 0; - pbuf->l2_len = 0; - pbuf->l3_len = 0; - } - - return pbuf; -} - -static int32_t lwip_alloc_pbufs(pbuf_layer layer, uint16_t length, pbuf_type type, void **bufs, uint32_t num) -{ - int32_t ret = rte_pktmbuf_alloc_bulk(get_protocol_stack()->tx_pktmbuf_pool, (struct rte_mbuf **)bufs, num); - if (ret != 0) { - get_protocol_stack()->stats.tx_allocmbuf_fail++; - return -1; - } - - for (uint32_t i = 0; i < num; i++) { - bufs[i] = init_mbuf_to_pbuf(bufs[i], layer, length, type); - } - - return 0; -} - struct pbuf *lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type) { - struct pbuf *pbuf; + struct rte_mbuf *mbuf; + struct protocol_stack *stack = get_protocol_stack(); - if (lwip_alloc_pbufs(layer, length, type, (void **)&pbuf, 1) != 0) { + if (rte_pktmbuf_alloc_bulk(stack->tx_pktmbuf_pool, &mbuf, 1) != 0) { + stack->stats.tx_allocmbuf_fail++; return NULL; } - return pbuf; + return init_mbuf_to_pbuf(mbuf, layer, length, type); } struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags) @@ -324,7 +314,7 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len) return send_len; } -static void do_lwip_send(int32_t fd, struct lwip_sock *sock, int32_t flags) +static void do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock, int32_t flags) { /* send all send_ring, so len set lwip send max. */ ssize_t len = lwip_send(fd, sock, UINT16_MAX, flags); @@ -335,7 +325,7 @@ static void do_lwip_send(int32_t fd, struct lwip_sock *sock, int32_t flags) } if (gazelle_ring_readable_count(sock->send_ring) < SOCK_SEND_REPLENISH_THRES) { - replenish_send_idlembuf(sock->send_ring); + replenish_send_idlembuf(stack, sock->send_ring); } if ((sock->epoll_events & EPOLLOUT) && NETCONN_IS_OUTIDLE(sock)) { @@ -347,8 +337,7 @@ void stack_send(struct rpc_msg *msg) { int32_t fd = msg->args[MSG_ARG_0].i; int32_t flags = msg->args[MSG_ARG_2].i; - - struct protocol_stack *stack = get_protocol_stack(); + struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p; struct lwip_sock *sock = get_socket(fd); if (sock == NULL) { @@ -363,7 +352,7 @@ void stack_send(struct rpc_msg *msg) return; } - do_lwip_send(fd, sock, flags); + do_lwip_send(stack, fd, sock, flags); /* have remain data add sendlist */ if (NETCONN_IS_DATAOUT(sock)) { @@ -392,7 +381,7 @@ void send_stack_list(struct protocol_stack *stack, uint32_t send_max) continue; } - do_lwip_send(sock->conn->socket, sock, 0); + do_lwip_send(stack, sock->conn->socket, sock, 0); if (!NETCONN_IS_DATAOUT(sock)) { list_del_node_null(&sock->send_list); @@ -542,11 +531,7 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags) return 0; } - struct lwip_sock *sock = get_socket(fd); - if (sock == NULL) { - GAZELLE_RETURN(EINVAL); - } - + struct lwip_sock *sock = get_socket_by_fd(fd); ssize_t send = write_stack_data(sock, buf, len); if (send <= 0) { return send; @@ -562,11 +547,7 @@ ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags) int32_t ret; int32_t i; ssize_t buflen = 0; - - struct lwip_sock *sock = get_socket(s); - if (sock == NULL) { - GAZELLE_RETURN(EINVAL); - } + struct lwip_sock *sock = get_socket_by_fd(s); if (check_msg_vaild(message)) { GAZELLE_RETURN(EINVAL); @@ -635,17 +616,16 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags) struct pbuf *pbuf = NULL; ssize_t recvd = 0; uint16_t copy_len; - - struct lwip_sock *sock = get_socket(fd); - if (sock == NULL) { - LSTACK_LOG(ERR, LSTACK, "get_socket null fd %d.\n", fd); - GAZELLE_RETURN(EINVAL); - } + struct lwip_sock *sock = get_socket_by_fd(fd); if (sock->errevent > 0 && !NETCONN_IS_DATAIN(sock)) { return 0; } + if (recv_left > UINT16_MAX) { + recv_left = UINT16_MAX; + } + while (recv_left > 0) { if (sock->recv_lastdata) { pbuf = sock->recv_lastdata; diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index 41dda89..2759d7d 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -36,9 +36,9 @@ #include "posix/lstack_epoll.h" #include "lstack_stack_stat.h" -#define READ_LIST_MAX 128 -#define SEND_LIST_MAX 128 -#define HANDLE_RPC_MSG_MAX 128 +#define READ_LIST_MAX 32 +#define SEND_LIST_MAX 32 +#define HANDLE_RPC_MSG_MAX 32 #define KERNEL_EVENT_100us 100 static PER_THREAD struct protocol_stack *g_stack_p = NULL; @@ -164,7 +164,7 @@ void low_power_idling(struct protocol_stack *stack) last_cycle_ts = sys_now(); } - uint64_t now_pkts = get_protocol_stack()->stats.rx; + uint64_t now_pkts = stack->stats.rx; uint32_t now_ts = sys_now(); if (((now_ts - last_cycle_ts) > LSTACK_LPM_DETECT_MS) || ((now_pkts - last_cycle_pkts) >= LSTACK_LPM_PKTS_IN_DETECT)) { @@ -258,7 +258,9 @@ static void* gazelle_kernelevent_thread(void *arg) uint16_t queue_id = *(uint16_t *)arg; struct protocol_stack *stack = get_protocol_stack_group()->stacks[queue_id]; - bind_to_stack_numa(stack); + if (get_global_cfg_params()->app_bind_numa) { + bind_to_stack_numa(stack); + } LSTACK_LOG(INFO, LSTACK, "kernelevent_%02hu start\n", queue_id); @@ -420,6 +422,7 @@ static void* gazelle_stack_thread(void *arg) { uint16_t queue_id = *(uint16_t *)arg; bool use_ltran_flag = use_ltran(); + uint32_t wakeup_tick = 0; struct protocol_stack *stack = stack_thread_init(queue_id); if (stack == NULL) { @@ -442,9 +445,11 @@ static void* gazelle_stack_thread(void *arg) send_stack_list(stack, SEND_LIST_MAX); - wakeup_kernel_event(stack); - - wakeup_stack_epoll(stack); + if ((wakeup_tick & 0xf) == 0) { + wakeup_kernel_event(stack); + wakeup_stack_epoll(stack); + } + wakeup_tick++; sys_timer_run(); @@ -523,8 +528,9 @@ int32_t init_protocol_stack(void) void stack_arp(struct rpc_msg *msg) { struct rte_mbuf *mbuf = (struct rte_mbuf *)msg->args[MSG_ARG_0].p; + struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_1].p; - eth_dev_recv(mbuf, NULL); + eth_dev_recv(mbuf, stack); } void stack_socket(struct rpc_msg *msg) diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c index db1de5a..295baf3 100644 --- a/src/lstack/core/lstack_thread_rpc.c +++ b/src/lstack/core/lstack_thread_rpc.c @@ -234,6 +234,7 @@ int32_t rpc_call_arp(struct protocol_stack *stack, struct rte_mbuf *mbuf) msg->self_release = 0; msg->args[MSG_ARG_0].p = mbuf; + msg->args[MSG_ARG_1].p = stack; rpc_call(&stack->rpc_queue, msg); @@ -451,6 +452,7 @@ int32_t rpc_call_send(int fd, const void *buf, size_t len, int flags) msg->args[MSG_ARG_0].i = fd; msg->args[MSG_ARG_1].size = len; msg->args[MSG_ARG_2].i = flags; + msg->args[MSG_ARG_3].p = stack; msg->self_release = 0; rpc_call(&stack->rpc_queue, msg); diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h index cc2cfb9..fed1882 100644 --- a/src/lstack/include/lstack_protocol_stack.h +++ b/src/lstack/include/lstack_protocol_stack.h @@ -49,6 +49,7 @@ struct protocol_stack { struct rte_ring *reg_ring; struct rte_ring *wakeup_ring; struct reg_ring_msg *reg_buf; + uint32_t reg_head; volatile bool low_power; lockless_queue rpc_queue __rte_cache_aligned; diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index 5ddc0db..3abed5e 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -39,9 +39,6 @@ void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack) struct pbuf *prev = NULL; struct pbuf *head = NULL; struct pbuf_custom *pc = NULL; - if (!stack) { - stack = get_protocol_stack(); - } struct rte_mbuf *m = mbuf; uint16_t len, pkt_len; @@ -81,7 +78,7 @@ void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack) } } -#define READ_PKTS_MAX 128 +#define READ_PKTS_MAX 32 int32_t eth_dev_poll(void) { uint32_t nr_pkts; diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c index f9fa5a3..1c148e1 100644 --- a/src/lstack/netif/lstack_vdev.c +++ b/src/lstack/netif/lstack_vdev.c @@ -113,7 +113,6 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) uint32_t sent_pkts = 0; void *free_buf[VDEV_REG_QUEUE_SZ]; struct reg_ring_msg *tmp_buf = NULL; - static PER_THREAD uint32_t head = 0; const uint32_t tbegin = sys_now(); struct protocol_stack *stack = get_protocol_stack(); @@ -124,6 +123,7 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) } } + uint32_t reg_index = stack->reg_head++ & DEFAULT_RING_MASK; do { (void)gazelle_ring_sc_dequeue(stack->reg_ring, free_buf, VDEV_REG_QUEUE_SZ); @@ -131,7 +131,7 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) continue; } - tmp_buf = &stack->reg_buf[head]; + tmp_buf = &stack->reg_buf[reg_index]; tmp_buf->type = type; tmp_buf->tid = get_stack_tid(); ret = memcpy_s(&tmp_buf->qtuple, sizeof(*qtuple), qtuple, sizeof(struct gazelle_quintuple)); @@ -144,9 +144,6 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) sent_pkts = gazelle_ring_sp_enqueue(stack->reg_ring, free_buf, 1); } while ((sent_pkts < 1) && (ENQUEUE_RING_RETRY_TIMEOUT > sys_now() - tbegin) && get_register_state()); - if (sent_pkts == 1) { - head = (head + 1) % VDEV_REG_QUEUE_SZ; - } return (int32_t)sent_pkts; } -- 2.23.0