From bf872f6a1dfbe1da9aec8e82cd7b89c0c743c2d7 Mon Sep 17 00:00:00 2001 From: Lemmy Huang Date: Tue, 23 May 2023 19:55:21 +0800 Subject: [PATCH] cleancode: refactor lwipsock.h Signed-off-by: Lemmy Huang --- src/lstack/api/lstack_epoll.c | 22 ++++++++-------- src/lstack/api/lstack_wrap.c | 12 ++++----- src/lstack/core/lstack_init.c | 3 --- src/lstack/core/lstack_lwip.c | 35 ++++++++++++++----------- src/lstack/core/lstack_protocol_stack.c | 18 ++++++------- src/lstack/include/lstack_lwip.h | 10 ++++++- 6 files changed, 55 insertions(+), 45 deletions(-) diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c index a25b0de..2fba2ee 100644 --- a/src/lstack/api/lstack_epoll.c +++ b/src/lstack/api/lstack_epoll.c @@ -120,7 +120,7 @@ static uint32_t update_events(struct lwip_sock *sock) if ((sock->epoll_events & EPOLLOUT) && NETCONN_IS_OUTIDLE(sock)) { /* lwip_netconn_do_connected set LIBOS FLAGS when connected */ - if (sock->conn && CONN_TYPE_IS_LIBOS(sock->conn)) { + if (POSIX_IS_TYPE(sock, POSIX_LWIP)) { event |= EPOLLOUT; } } @@ -142,7 +142,7 @@ static void raise_pending_events(struct wakeup_poll *wakeup, struct lwip_sock *s if (NETCONN_IS_OUTIDLE(sock)) { /* lwip_netconn_do_connected set LIBOS FLAGS when connected */ - if (sock->conn && CONN_TYPE_IS_LIBOS(sock->conn)) { + if (POSIX_IS_TYPE(sock, POSIX_LWIP)) { event |= EPOLLOUT; } } @@ -168,7 +168,7 @@ int32_t lstack_do_epoll_create(int32_t fd) return fd; } - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { LSTACK_LOG(ERR, LSTACK, "fd=%d sock is NULL errno=%d\n", fd, errno); posix_api->close_fn(fd); @@ -231,7 +231,7 @@ int32_t lstack_epoll_create(int32_t flags) int32_t lstack_epoll_close(int32_t fd) { - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { LSTACK_LOG(ERR, LSTACK, "fd=%d sock is NULL errno=%d\n", fd, errno); GAZELLE_RETURN(EINVAL); @@ -315,18 +315,18 @@ int32_t lstack_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct epoll_even GAZELLE_RETURN(EINVAL); } - struct lwip_sock *epoll_sock = get_socket_by_fd(epfd); + struct lwip_sock *epoll_sock = lwip_get_socket_nouse(epfd); if (epoll_sock == NULL || epoll_sock->wakeup == NULL) { return posix_api->epoll_ctl_fn(epfd, op, fd, event); } struct wakeup_poll *wakeup = epoll_sock->wakeup; - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { return posix_api->epoll_ctl_fn(epfd, op, fd, event); } - if (CONN_TYPE_HAS_HOST(sock->conn)) { + if (POSIX_HAS_TYPE(sock, POSIX_KERNEL)) { int32_t ret = posix_api->epoll_ctl_fn(epfd, op, fd, event); if (ret < 0) { LSTACK_LOG(ERR, LSTACK, "fd=%d epfd=%d op=%d errno=%d\n", fd, epfd, op, errno); @@ -409,7 +409,7 @@ static int32_t poll_lwip_event(struct pollfd *fds, nfds_t nfds) for (uint32_t i = 0; i < nfds; i++) { /* sock->listen_next pointerto next stack listen */ int32_t fd = fds[i].fd; - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); while (sock && sock->conn) { uint32_t events = update_events(sock); if (events) { @@ -463,7 +463,7 @@ static void ms_to_timespec(struct timespec *timespec, int32_t timeout) int32_t lstack_epoll_wait(int32_t epfd, struct epoll_event* events, int32_t maxevents, int32_t timeout) { - struct lwip_sock *sock = get_socket_by_fd(epfd); + struct lwip_sock *sock = lwip_get_socket_nouse(epfd); if (sock == NULL || sock->wakeup == NULL) { return posix_api->epoll_wait_fn(epfd, events, maxevents, timeout); } @@ -627,7 +627,7 @@ static void poll_init(struct wakeup_poll *wakeup, struct pollfd *fds, nfds_t nfd for (uint32_t i = 0; i < nfds; i++) { int32_t fd = fds[i].fd; fds[i].revents = 0; - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (fd == wakeup->last_fds[i].fd && fds[i].events == wakeup->last_fds[i].events) { /* fd close then socket may get same fd. */ @@ -639,7 +639,7 @@ static void poll_init(struct wakeup_poll *wakeup, struct pollfd *fds, nfds_t nfd wakeup->last_fds[i].events = fds[i].events; poll_change = 1; - if (sock == NULL || sock->conn == NULL || CONN_TYPE_HAS_HOST(sock->conn)) { + if (sock == NULL || sock->conn == NULL || POSIX_HAS_TYPE(sock, POSIX_KERNEL)) { update_kernel_poll(wakeup, i, fds + i); } diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c index 4e83501..160cf8e 100644 --- a/src/lstack/api/lstack_wrap.c +++ b/src/lstack/api/lstack_wrap.c @@ -66,14 +66,14 @@ static inline enum KERNEL_LWIP_PATH select_path(int fd, struct lwip_sock **socke return PATH_KERNEL; } - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); /* AF_UNIX case */ - if (!sock || !sock->conn || CONN_TYPE_IS_HOST(sock->conn)) { + if (!sock || !sock->conn || POSIX_IS_TYPE(sock, POSIX_KERNEL)) { return PATH_KERNEL; } - if (likely(CONN_TYPE_IS_LIBOS(sock->conn))) { + if (likely(POSIX_IS_TYPE(sock, POSIX_LWIP))) { if (socket) { *socket = sock; } @@ -287,7 +287,7 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name return posix_api->connect_fn(s, name, namelen); } - sock = get_socket(s); + sock = lwip_get_socket_nouse(s); if (sock == NULL) { return posix_api->connect_fn(s, name, namelen); } @@ -303,10 +303,10 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name "listen_rx_ring_%d", remote_port); if (is_dst_ip_localhost(name) && rte_ring_lookup(listen_ring_name) == NULL) { ret = posix_api->connect_fn(s, name, namelen); - SET_CONN_TYPE_HOST(sock->conn); + POSIX_SET_TYPE(sock, POSIX_KERNEL); } else { ret = rpc_call_connect(s, name, namelen); - SET_CONN_TYPE_LIBOS(sock->conn); + POSIX_SET_TYPE(sock, POSIX_LWIP); } return ret; diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c index 7bb1a66..894f549 100644 --- a/src/lstack/core/lstack_init.c +++ b/src/lstack/core/lstack_init.c @@ -359,9 +359,6 @@ __attribute__((constructor)) void gazelle_network_init(void) } } - /* lwip initialization */ - lwip_sock_init(); - /* wait stack thread and kernel_event thread init finish */ wait_sem_value(&get_protocol_stack_group()->all_init, get_protocol_stack_group()->stack_num); if (g_init_fail) { diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index 1a82a2e..035311f 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -92,6 +92,7 @@ static void reset_sock_data(struct lwip_sock *sock) sock->send_pre_del = NULL; } + sock->type = 0; sock->stack = NULL; sock->wakeup = NULL; sock->listen_next = NULL; @@ -171,11 +172,16 @@ void gazelle_init_sock(int32_t fd) { static _Atomic uint32_t name_tick = 0; struct protocol_stack *stack = get_protocol_stack(); - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { return; } + sock->same_node_rx_ring = NULL; + sock->same_node_rx_ring_mz = NULL; + sock->same_node_tx_ring = NULL; + sock->same_node_tx_ring_mz = NULL; + reset_sock_data(sock); sock->recv_ring = create_ring("sock_recv", SOCK_RECV_RING_SIZE, RING_F_SP_ENQ | RING_F_SC_DEQ, @@ -202,7 +208,7 @@ void gazelle_init_sock(int32_t fd) void gazelle_clean_sock(int32_t fd) { - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL || sock->stack == NULL) { return; } @@ -680,7 +686,7 @@ void stack_send(struct rpc_msg *msg) struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p; bool replenish_again; - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { msg->result = -1; LSTACK_LOG(ERR, LSTACK, "stack_send: sock error!\n"); @@ -971,7 +977,7 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags, return 0; } - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); thread_bind_stack(sock); @@ -1047,7 +1053,7 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags, struct struct pbuf *pbuf = NULL; ssize_t recvd = 0; uint32_t copy_len; - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); bool latency_enable = get_protocol_stack_group()->latency_start; if (sock->errevent > 0 && !NETCONN_IS_DATAIN(sock)) { @@ -1113,7 +1119,7 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags, struct void add_recv_list(int32_t fd) { - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock && sock->stack && list_is_null(&sock->recv_list)) { list_add_node(&sock->stack->recv_list, &sock->recv_list); @@ -1179,7 +1185,7 @@ void gazelle_connected_callback(struct netconn *conn) } int32_t fd = conn->socket; - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL || sock->conn == NULL) { return; } @@ -1190,8 +1196,7 @@ void gazelle_connected_callback(struct netconn *conn) posix_api->shutdown_fn(fd, SHUT_RDWR); - SET_CONN_TYPE_LIBOS(conn); - + POSIX_SET_TYPE(sock, POSIX_LWIP); add_sock_event(sock, EPOLLOUT); } @@ -1217,7 +1222,7 @@ static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const s conn->recv_cnt = rte_ring_count(netconn->recvmbox->ring); conn->fd = netconn->socket; - struct lwip_sock *sock = get_socket(netconn->socket); + struct lwip_sock *sock = lwip_get_socket_nouse(netconn->socket); if (netconn->socket > 0 && sock != NULL && sock->recv_ring != NULL && sock->send_ring != NULL) { conn->recv_ring_cnt = gazelle_ring_readable_count(sock->recv_ring); conn->recv_ring_cnt += (sock->recv_lastdata) ? 1 : 0; @@ -1252,7 +1257,7 @@ int32_t gazelle_socket(int domain, int type, int protocol) gazelle_init_sock(fd); - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL || sock->stack == NULL) { lwip_close(fd); gazelle_clean_sock(fd); @@ -1276,8 +1281,8 @@ void create_shadow_fd(struct rpc_msg *msg) return; } - struct lwip_sock *sock = get_socket_by_fd(fd); - struct lwip_sock *clone_sock = get_socket_by_fd(clone_fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); + struct lwip_sock *clone_sock = lwip_get_socket_nouse(clone_fd); if (sock == NULL || clone_sock == NULL) { LSTACK_LOG(ERR, LSTACK, "get sock null fd=%d clone_fd=%d\n", fd, clone_fd); msg->result = -1; @@ -1552,7 +1557,7 @@ err_t same_node_ring_create(struct rte_ring **ring, int size, int port, char *na static void init_same_node_ring(struct tcp_pcb *pcb) { struct netconn *netconn = (struct netconn *)pcb->callback_arg; - struct lwip_sock *sock = get_socket(netconn->socket); + struct lwip_sock *sock = lwip_get_socket_nouse(netconn->socket); pcb->client_rx_ring = NULL; pcb->client_tx_ring = NULL; @@ -1567,7 +1572,7 @@ static void init_same_node_ring(struct tcp_pcb *pcb) err_t create_same_node_ring(struct tcp_pcb *pcb) { struct netconn *netconn = (struct netconn *)pcb->callback_arg; - struct lwip_sock *sock = get_socket(netconn->socket); + struct lwip_sock *sock = lwip_get_socket_nouse(netconn->socket); if (same_node_ring_create(&pcb->client_rx_ring, CLIENT_RING_SIZE, pcb->local_port, "client", "rx") != 0) { goto END; diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index f04f736..b7aeeae 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -109,7 +109,7 @@ struct protocol_stack *get_protocol_stack(void) struct protocol_stack *get_protocol_stack_by_fd(int32_t fd) { - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { return NULL; } @@ -666,7 +666,7 @@ void stack_listen(struct rpc_msg *msg) int32_t fd = msg->args[MSG_ARG_0].i; int32_t backlog = msg->args[MSG_ARG_1].i; - struct lwip_sock *sock = get_socket_by_fd(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { msg->result = -1; return; @@ -690,7 +690,7 @@ void stack_accept(struct rpc_msg *msg) return; } - struct lwip_sock *sock = get_socket(accept_fd); + struct lwip_sock *sock = lwip_get_socket_nouse(accept_fd); if (sock == NULL || sock->stack == NULL) { lwip_close(accept_fd); gazelle_clean_sock(accept_fd); @@ -820,7 +820,7 @@ void stack_clean_epoll(struct rpc_msg *msg) /* when fd is listenfd, listenfd of all protocol stack thread will be closed */ int32_t stack_broadcast_close(int32_t fd) { - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); int32_t ret = 0; if (sock == NULL) { @@ -857,7 +857,7 @@ int32_t stack_broadcast_listen(int32_t fd, int32_t backlog) socklen_t addr_len = sizeof(addr); int32_t ret, clone_fd; - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { LSTACK_LOG(ERR, LSTACK, "tid %ld, %d get sock null\n", get_stack_tid(), fd); GAZELLE_RETURN(EINVAL); @@ -887,9 +887,9 @@ int32_t stack_broadcast_listen(int32_t fd, int32_t backlog) } if (min_conn_stk_idx == i) { - get_socket_by_fd(clone_fd)->conn->is_master_fd = 1; + lwip_get_socket_nouse(clone_fd)->conn->is_master_fd = 1; } else { - get_socket_by_fd(clone_fd)->conn->is_master_fd = 0; + lwip_get_socket_nouse(clone_fd)->conn->is_master_fd = 0; } ret = rpc_call_listen(clone_fd, backlog); @@ -903,7 +903,7 @@ int32_t stack_broadcast_listen(int32_t fd, int32_t backlog) static struct lwip_sock *get_min_accept_sock(int32_t fd) { - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); struct lwip_sock *min_sock = NULL; while (sock) { @@ -941,7 +941,7 @@ int32_t stack_broadcast_accept4(int32_t fd, struct sockaddr *addr, socklen_t *ad { int32_t ret = -1; - struct lwip_sock *sock = get_socket(fd); + struct lwip_sock *sock = lwip_get_socket_nouse(fd); if (sock == NULL) { errno = EINVAL; return -1; diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h index 223ff93..d4f49e0 100644 --- a/src/lstack/include/lstack_lwip.h +++ b/src/lstack/include/lstack_lwip.h @@ -13,13 +13,21 @@ #ifndef __GAZELLE_LWIP_H__ #define __GAZELLE_LWIP_H__ +#include + +static inline unsigned same_node_ring_count(struct lwip_sock *sock) +{ + const unsigned long long cur_begin = __atomic_load_n(&sock->same_node_rx_ring->sndbegin, __ATOMIC_RELAXED); + const unsigned long long cur_end = __atomic_load_n(&sock->same_node_rx_ring->sndend, __ATOMIC_RELAXED); + return cur_end - cur_begin; +} + #define NETCONN_IS_ACCEPTIN(sock) (((sock)->conn->acceptmbox != NULL) && !sys_mbox_empty((sock)->conn->acceptmbox)) #define NETCONN_IS_DATAIN(sock) ((gazelle_ring_readable_count((sock)->recv_ring) || (sock)->recv_lastdata) || (sock->same_node_rx_ring != NULL && same_node_ring_count(sock))) #define NETCONN_IS_DATAOUT(sock) (gazelle_ring_readover_count((sock)->send_ring) || (sock)->send_lastdata || (sock)->send_pre_del) #define NETCONN_IS_OUTIDLE(sock) gazelle_ring_readable_count((sock)->send_ring) #define NETCONN_IS_UDP(sock) (NETCONNTYPE_GROUP(netconn_type((sock)->conn)) == NETCONN_UDP) -struct lwip_sock; struct rte_mempool; struct rpc_msg; struct rte_mbuf; -- 2.23.0