From ff2512a2ac5000b3e0bedc4fd194d5f8a2b07887 Mon Sep 17 00:00:00 2001 From: kircher Date: Fri, 24 Mar 2023 19:30:09 +0800 Subject: [PATCH] optimite select_path and pbuf_take --- src/lstack/api/lstack_wrap.c | 65 +++++++++++++++++++++----------- src/lstack/core/lstack_lwip.c | 22 ++++++++--- src/lstack/include/lstack_lwip.h | 2 +- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c index 5132ee9..1fe576f 100644 --- a/src/lstack/api/lstack_wrap.c +++ b/src/lstack/api/lstack_wrap.c @@ -50,7 +50,7 @@ enum KERNEL_LWIP_PATH { PATH_UNKNOW, }; -static inline enum KERNEL_LWIP_PATH select_path(int fd) +static inline enum KERNEL_LWIP_PATH select_path(int fd, struct lwip_sock **socket) { if (unlikely(posix_api == NULL)) { /* @@ -75,6 +75,7 @@ static inline enum KERNEL_LWIP_PATH select_path(int fd) } if (likely(CONN_TYPE_IS_LIBOS(sock->conn))) { + *socket = sock; return PATH_LWIP; } @@ -82,6 +83,7 @@ static inline enum KERNEL_LWIP_PATH select_path(int fd) /* after lwip connect, call send immediately, pcb->state is SYN_SENT, need return PATH_LWIP */ /* pcb->state default value is CLOSED when call socket, need return PATH_UNKNOW */ if (pcb != NULL && pcb->state <= ESTABLISHED && pcb->state >= LISTEN) { + *socket = sock return PATH_LWIP; } @@ -150,7 +152,8 @@ static inline int32_t do_epoll_wait(int32_t epfd, struct epoll_event* events, in static inline int32_t do_accept(int32_t s, struct sockaddr *addr, socklen_t *addrlen) { - if (select_path(s) == PATH_KERNEL) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->accept_fn(s, addr, addrlen); } @@ -168,7 +171,8 @@ static int32_t do_accept4(int32_t s, struct sockaddr *addr, socklen_t *addrlen, GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_KERNEL) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->accept4_fn(s, addr, addrlen, flags); } @@ -208,7 +212,8 @@ static int32_t do_bind(int32_t s, const struct sockaddr *name, socklen_t namelen GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_KERNEL) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->bind_fn(s, name, namelen); } @@ -263,11 +268,11 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_KERNEL) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->connect_fn(s, name, namelen); } - struct lwip_sock *sock = get_socket(s); if (sock == NULL) { return posix_api->connect_fn(s, name, namelen); } @@ -293,7 +298,8 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name static inline int32_t do_listen(int32_t s, int32_t backlog) { - if (select_path(s) == PATH_KERNEL) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->listen_fn(s, backlog); } @@ -317,7 +323,8 @@ static inline int32_t do_getpeername(int32_t s, struct sockaddr *name, socklen_t GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP) { return rpc_call_getpeername(s, name, namelen); } @@ -330,7 +337,8 @@ static inline int32_t do_getsockname(int32_t s, struct sockaddr *name, socklen_t GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP) { return rpc_call_getsockname(s, name, namelen); } @@ -351,7 +359,8 @@ static bool unsupport_optname(int32_t optname) static inline int32_t do_getsockopt(int32_t s, int32_t level, int32_t optname, void *optval, socklen_t *optlen) { - if (select_path(s) == PATH_LWIP && !unsupport_optname(optname)) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP && !unsupport_optname(optname)) { return rpc_call_getsockopt(s, level, optname, optval, optlen); } @@ -360,7 +369,8 @@ static inline int32_t do_getsockopt(int32_t s, int32_t level, int32_t optname, v static inline int32_t do_setsockopt(int32_t s, int32_t level, int32_t optname, const void *optval, socklen_t optlen) { - if (select_path(s) == PATH_KERNEL || unsupport_optname(optname)) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL || unsupport_optname(optname)) { return posix_api->setsockopt_fn(s, level, optname, optval, optlen); } @@ -393,7 +403,8 @@ static inline ssize_t do_recv(int32_t sockfd, void *buf, size_t len, int32_t fla return 0; } - if (select_path(sockfd) == PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(sockfd, &sock) == PATH_LWIP) { return read_stack_data(sockfd, buf, len, flags); } @@ -410,7 +421,8 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len) return 0; } - if (select_path(s) == PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP) { return read_stack_data(s, mem, len, 0); } return posix_api->read_fn(s, mem, len); @@ -418,7 +430,8 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len) static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt) { - if (select_path(s) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) != PATH_LWIP) { return posix_api->readv_fn(s, iov, iovcnt); } @@ -441,7 +454,8 @@ static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt) static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32_t flags) { - if (select_path(sockfd) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(sockfd, &sock) != PATH_LWIP) { return posix_api->send_fn(sockfd, buf, len, flags); } @@ -450,7 +464,8 @@ static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32 static inline ssize_t do_write(int32_t s, const void *mem, size_t size) { - if (select_path(s) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) != PATH_LWIP) { return posix_api->write_fn(s, mem, size); } @@ -459,7 +474,8 @@ static inline ssize_t do_write(int32_t s, const void *mem, size_t size) static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt) { - if (select_path(s) != PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) != PATH_LWIP) { return posix_api->writev_fn(s, iov, iovcnt); } @@ -472,7 +488,7 @@ static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt) msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = 0; - return sendmsg_to_stack(s, &msg, 0); + return sendmsg_to_stack(sock, s, &msg, 0); } static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flags) @@ -481,7 +497,8 @@ static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flag GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_LWIP) { + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP) { return recvmsg_from_stack(s, message, flags); } @@ -494,8 +511,9 @@ static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_ GAZELLE_RETURN(EINVAL); } - if (select_path(s) == PATH_LWIP) { - return sendmsg_to_stack(s, message, flags); + struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_LWIP) { + return sendmsg_to_stack(sock, s, message, flags); } return posix_api->send_msg(s, message, flags); @@ -508,7 +526,7 @@ static inline int32_t do_close(int32_t s) return lstack_epoll_close(s); } - if (select_path(s) == PATH_KERNEL) { + if (select_path(s, &sock) == PATH_KERNEL) { return posix_api->close_fn(s); } @@ -561,7 +579,8 @@ static int32_t do_sigaction(int32_t signum, const struct sigaction *act, struct va_start(ap, _cmd); \ val = va_arg(ap, typeof(val)); \ va_end(ap); \ - if (select_path(_fd) == PATH_KERNEL) \ + struct lwip_sock *sock = NULL; \ + if (select_path(_fd, &sock) == PATH_KERNEL) \ return _fcntl_fn(_fd, _cmd, val); \ int32_t ret = _fcntl_fn(_fd, _cmd, val); \ if (ret == -1) \ diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index a4e6e0b..71abbc6 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -344,19 +344,28 @@ static ssize_t do_app_write(struct pbuf *pbufs[], void *buf, size_t len, uint32_ { ssize_t send_len = 0; uint32_t i = 0; + uint32_t expand_send_ring = get_global_cfg_params()->expand_send_ring; for (i = 0; i < write_num - 1; i++) { rte_prefetch0(pbufs[i + 1]); rte_prefetch0(pbufs[i + 1]->payload); rte_prefetch0((char *)buf + send_len + MBUF_MAX_DATA_LEN); - pbuf_take(pbufs[i], (char *)buf + send_len, MBUF_MAX_DATA_LEN); + if (expand_send_ring) { + pbuf_take(pbufs[i], (char *)buf + send_len, MBUF_MAX_DATA_LEN); + } else { + rte_memcpy((char *)pbufs[i]->payload, (char *)buf + send_len, MBUF_MAX_DATA_LEN); + } pbufs[i]->tot_len = pbufs[i]->len = MBUF_MAX_DATA_LEN; send_len += MBUF_MAX_DATA_LEN; } /* reduce the branch in loop */ uint16_t copy_len = len - send_len; - pbuf_take(pbufs[i], (char *)buf + send_len, copy_len); + if (expand_send_ring) { + pbuf_take(pbufs[i], (char *)buf + send_len, copy_len); + } else { + rte_memcpy((char *)pbufs[i]->payload, (char *)buf + send_len, copy_len); + } pbufs[i]->tot_len = pbufs[i]->len = copy_len; send_len += copy_len; @@ -500,7 +509,11 @@ static inline size_t merge_data_lastpbuf(struct lwip_sock *sock, void *buf, size uint16_t offset = last_pbuf->len; last_pbuf->tot_len = last_pbuf->len = offset + send_len; - pbuf_take_at(last_pbuf, buf, send_len, offset); + if (get_global_cfg_params()->expand_send_ring) { + pbuf_take_at(last_pbuf, buf, send_len, offset); + } else { + rte_memcpy((char *)last_pbuf->payload + offset, buf, send_len); + } gazelle_ring_lastover(last_pbuf); @@ -924,12 +937,11 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags) return send; } -ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags) +ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr *message, int32_t flags) { int32_t ret; int32_t i; ssize_t buflen = 0; - struct lwip_sock *sock = get_socket_by_fd(s); if (check_msg_vaild(message)) { GAZELLE_RETURN(EINVAL); diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h index d52a06d..0b29e71 100644 --- a/src/lstack/include/lstack_lwip.h +++ b/src/lstack/include/lstack_lwip.h @@ -43,7 +43,7 @@ void stack_send(struct rpc_msg *msg); void app_rpc_write(struct rpc_msg *msg); int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num); void gazelle_free_pbuf(struct pbuf *pbuf); -ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags); +ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr *message, int32_t flags); ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags); ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags); void rpc_replenish(struct rpc_msg *msg); -- 2.23.0