From 04b8cb3f4577cd7bf62bc831ca32cdc8229ee5b8 Mon Sep 17 00:00:00 2001 From: Lemmy Huang Date: Sat, 27 May 2023 16:14:06 +0800 Subject: [PATCH 4/5] cleancode: refactor OFFLOAD_CHECKSUM GAZELLE_TCP_DATAACKS_REXMIT GAZELLE_TCP_NEW_PORT Signed-off-by: Lemmy Huang --- src/core/ipv4/icmp.c | 8 ++-- src/core/ipv4/ip4.c | 12 +++--- src/core/ipv4/ip4_frag.c | 14 +++---- src/core/pbuf.c | 2 +- src/core/tcp.c | 61 +++++++++++++++++++++--------- src/core/tcp_in.c | 54 +++++++++++++------------- src/core/tcp_out.c | 16 ++++---- src/core/udp.c | 2 +- src/include/gazelle_dpdk_offload.h | 43 ++++++++++----------- src/include/gazelle_tcp_priv.h | 53 ++++++++++++++++++++++---- src/include/lwip/pbuf.h | 2 +- src/include/lwip/priv/tcp_priv.h | 37 ------------------ src/include/lwip/tcp.h | 4 +- src/include/lwipopts.h | 27 ++++++------- src/netif/ethernet.c | 6 +-- 15 files changed, 183 insertions(+), 158 deletions(-) diff --git a/src/core/ipv4/icmp.c b/src/core/ipv4/icmp.c index 2b15ed4..9da81d9 100644 --- a/src/core/ipv4/icmp.c +++ b/src/core/ipv4/icmp.c @@ -51,7 +51,7 @@ #include -#if GAZELLE_ENABLE && CHECKSUM_GEN_IP_HW +#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_GEN_IP #include "gazelle_dpdk_offload.h" #endif @@ -240,11 +240,11 @@ icmp_input(struct pbuf *p, struct netif *inp) IPH_CHKSUM_SET(iphdr, 0); #if CHECKSUM_GEN_IP IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_GEN_IP) { -#if CHECKSUM_GEN_IP_HW +#if OFFLOAD_CHECKSUM_GEN_IP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { - iph_cksum_set(p, hlen, 1); + ol_chksum_gen_ip(p, hlen, 1); } else { - iph_cksum_set(p, hlen, 0); + ol_chksum_gen_ip(p, hlen, 0); IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, hlen)); } #else diff --git a/src/core/ipv4/ip4.c b/src/core/ipv4/ip4.c index 079716d..444f496 100644 --- a/src/core/ipv4/ip4.c +++ b/src/core/ipv4/ip4.c @@ -59,7 +59,7 @@ #include -#if GAZELLE_ENABLE && (CHECKSUM_CHECK_IP_HW || CHECKSUM_GEN_IP_HW) +#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_CHECK_IP || OFFLOAD_CHECKSUM_GEN_IP) #include "gazelle_dpdk_offload.h" #endif @@ -501,10 +501,10 @@ ip4_input(struct pbuf *p, struct netif *inp) /* verify checksum */ #if CHECKSUM_CHECK_IP IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_IP) { -#if CHECKSUM_CHECK_IP_HW +#if OFFLOAD_CHECKSUM_CHECK_IP u64_t ret; if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_IPV4_CKSUM) { - ret = is_cksum_ipbad(p); + ret = ol_chksum_check_ip(p); } else { ret = (u64_t)inet_chksum(iphdr, iphdr_hlen); } @@ -971,11 +971,11 @@ ip4_output_if_opt_src(struct pbuf *p, const ip4_addr_t *src, const ip4_addr_t *d IPH_CHKSUM_SET(iphdr, 0); #if CHECKSUM_GEN_IP IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) { -#if CHECKSUM_GEN_IP_HW +#if OFFLOAD_CHECKSUM_GEN_IP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { - iph_cksum_set(p, ip_hlen, 1); + ol_chksum_gen_ip(p, ip_hlen, 1); } else { - iph_cksum_set(p, ip_hlen, 0); + ol_chksum_gen_ip(p, ip_hlen, 0); IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, ip_hlen)); } #else diff --git a/src/core/ipv4/ip4_frag.c b/src/core/ipv4/ip4_frag.c index 4d697c8..4cc48e8 100644 --- a/src/core/ipv4/ip4_frag.c +++ b/src/core/ipv4/ip4_frag.c @@ -51,7 +51,7 @@ #include -#if GAZELLE_ENABLE && CHECKSUM_GEN_IP_HW +#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_GEN_IP #include "gazelle_dpdk_offload.h" #endif @@ -641,11 +641,11 @@ ip4_reass(struct pbuf *p) /* @todo: do we need to set/calculate the correct checksum? */ #if CHECKSUM_GEN_IP IF__NETIF_CHECKSUM_ENABLED(ip_current_input_netif(), NETIF_CHECKSUM_GEN_IP) { -#if CHECKSUM_GEN_IP_HW +#if OFFLOAD_CHECKSUM_GEN_IP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { - iph_cksum_set(p, IP_HLEN, 1); + ol_chksum_gen_ip(p, IP_HLEN, 1); } else { - iph_cksum_set(p, IP_HLEN, 0); + ol_chksum_gen_ip(p, IP_HLEN, 0); IPH_CHKSUM_SET(fraghdr, inet_chksum(fraghdr, IP_HLEN)); } #else @@ -884,11 +884,11 @@ ip4_frag(struct pbuf *p, struct netif *netif, const ip4_addr_t *dest) IPH_CHKSUM_SET(iphdr, 0); #if CHECKSUM_GEN_IP IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) { -#if CHECKSUM_GEN_IP_HW +#if OFFLOAD_CHECKSUM_GEN_IP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { - iph_cksum_set(p, IP_HLEN, 1); + ol_chksum_gen_ip(p, IP_HLEN, 1); } else { - iph_cksum_set(p, IP_HLEN, 0); + ol_chksum_gen_ip(p, IP_HLEN, 0); IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, IP_HLEN)); } diff --git a/src/core/pbuf.c b/src/core/pbuf.c index 16615c6..bc629b4 100644 --- a/src/core/pbuf.c +++ b/src/core/pbuf.c @@ -1030,7 +1030,7 @@ pbuf_copy_partial_pbuf(struct pbuf *p_to, const struct pbuf *p_from, u16_t copy_ len_calc = p_to->len - offset_to; } -#if GAZELLE_ENABLE && (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW) +#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP) p_to->l2_len = p_from->l2_len; p_to->l3_len = p_from->l3_len; p_to->ol_flags = p_from->ol_flags; diff --git a/src/core/tcp.c b/src/core/tcp.c index caba39d..e3355dc 100644 --- a/src/core/tcp.c +++ b/src/core/tcp.c @@ -189,19 +189,21 @@ PER_THREAD u8_t tcp_active_pcbs_changed; /** Timer counter to handle calling slow-timer from tcp_tmr() */ static PER_THREAD u8_t tcp_timer; static PER_THREAD u8_t tcp_timer_ctr; -#if GAZELLE_ENABLE + +#if GAZELLE_TCP_NEW_PORT static u16_t tcp_new_port(struct tcp_pcb *pcb); static pthread_mutex_t g_tcp_port_mutex = PTHREAD_MUTEX_INITIALIZER; static u8_t port_state[TCP_LOCAL_PORT_RANGE_END - TCP_LOCAL_PORT_RANGE_START + 1] = {0}; + void release_port(u16_t port) { if (port >= TCP_LOCAL_PORT_RANGE_START && port <= TCP_LOCAL_PORT_RANGE_END) { port_state[port - TCP_LOCAL_PORT_RANGE_START] = 0; } } -#else /* GAZELLE_ENABLE */ +#else /* GAZELLE_TCP_NEW_PORT */ static u16_t tcp_new_port(void); -#endif /* GAZELLE_ENABLE */ +#endif /* GAZELLE_TCP_NEW_PORT */ #if GAZELLE_TCP_PCB_HASH PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/lip/fip */ @@ -255,6 +257,8 @@ tcp_free(struct tcp_pcb *pcb) rte_memzone_free(sock->same_node_tx_ring_mz); } vdev_unreg_done(pcb); +#endif +#if GAZELLE_TCP_NEW_PORT release_port(pcb->local_port); #endif LWIP_ASSERT("tcp_free: LISTEN", pcb->state != LISTEN); @@ -755,7 +759,7 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port) #endif /* LWIP_IPV6 && LWIP_IPV6_SCOPES */ if (port == 0) { -#if GAZELLE_ENABLE +#if GAZELLE_TCP_NEW_PORT port = tcp_new_port(pcb); #else port = tcp_new_port(); @@ -986,11 +990,12 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) /* copy over ext_args to listening pcb */ memcpy(&lpcb->ext_args, &pcb->ext_args, sizeof(pcb->ext_args)); #endif -#if GAZELLE_ENABLE +#if GAZELLE_TCP_NEW_PORT /* pcb transfer to lpcb and reg into tcp_listen_pcbs. freeing pcb shouldn't release sock table in here. * local_port=0 avoid to release sock table in tcp_free */ pcb->local_port = 0; - +#endif /* GAZELLE_TCP_NEW_PORT */ +#if GAZELLE_ENABLE char name[RING_NAME_LEN]; snprintf(name, sizeof(name), "listen_rx_ring_%u", lpcb->local_port); if (rte_ring_lookup(name) != NULL) { @@ -999,7 +1004,7 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) } else { same_node_ring_create(&lpcb->listen_rx_ring, SAME_NODE_RING_SIZE, lpcb->local_port, "listen", "rx"); } -#endif +#endif /* GAZELLE_ENABLE */ tcp_free(pcb); #if LWIP_CALLBACK_API lpcb->accept = tcp_accept_null; @@ -1112,13 +1117,9 @@ tcp_recved(struct tcp_pcb *pcb, u16_t len) * * @return a new (free) local TCP port number */ -#if GAZELLE_ENABLE +#if GAZELLE_TCP_NEW_PORT static u16_t tcp_new_port(struct tcp_pcb *pcb) -#else -static u16_t -tcp_new_port(void) -#endif { u16_t n = 0; u16_t tmp_port = 0; @@ -1131,16 +1132,11 @@ tcp_new_port(void) } if (__atomic_load_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], __ATOMIC_ACQUIRE) == 0) { -#if GAZELLE_ENABLE if (port_in_stack_queue(pcb->remote_ip.addr, pcb->local_ip.addr, pcb->remote_port, tcp_port)) { tmp_port = tcp_port; __atomic_store_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], 1, __ATOMIC_RELEASE); break; } -#else - __atomic_store_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], 1, __ATOMIC_RELEASE); - break; -#endif } n++; @@ -1154,6 +1150,35 @@ tcp_new_port(void) return tmp_port; } +#else /* GAZELLE_TCP_NEW_PORT */ +static u16_t +tcp_new_port(void) +{ + u8_t i; + u16_t n = 0; + struct tcp_pcb *pcb; + +again: + tcp_port++; + if (tcp_port == TCP_LOCAL_PORT_RANGE_END) { + tcp_port = TCP_LOCAL_PORT_RANGE_START; + } + /* Check all PCB lists. */ + for (i = 0; i < NUM_TCP_PCB_LISTS; i++) { + for (pcb = *tcp_pcb_lists[i]; pcb != NULL; pcb = pcb->next) { + if (pcb->local_port == tcp_port) { + n++; + if (n > (TCP_LOCAL_PORT_RANGE_END - TCP_LOCAL_PORT_RANGE_START)) { + return 0; + } + goto again; + } + } + } + return tcp_port; +} +#endif /* GAZELLE_TCP_NEW_PORT */ + /** * @ingroup tcp_raw * Connects to another host. The function given as the "connected" @@ -1234,7 +1259,7 @@ tcp_connect(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port, old_local_port = pcb->local_port; if (pcb->local_port == 0) { -#if GAZELLE_ENABLE +#if GAZELLE_TCP_NEW_PORT pcb->local_port = tcp_new_port(pcb); #else pcb->local_port = tcp_new_port(); diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c index 575bf05..70c125a 100644 --- a/src/core/tcp_in.c +++ b/src/core/tcp_in.c @@ -65,9 +65,9 @@ #include -#if GAZELLE_ENABLE && CHECKSUM_CHECK_TCP_HW +#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_CHECK_TCP #include -#endif /* CHECKSUM_CHECK_TCP_HW */ +#endif /* OFFLOAD_CHECKSUM_CHECK_TCP */ #ifdef LWIP_HOOK_FILENAME #include LWIP_HOOK_FILENAME @@ -168,10 +168,10 @@ tcp_input(struct pbuf *p, struct netif *inp) #if CHECKSUM_CHECK_TCP IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_TCP) { /* Verify TCP checksum. */ -#if CHECKSUM_CHECK_TCP_HW +#if OFFLOAD_CHECKSUM_CHECK_TCP u64_t ret; if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_TCP_CKSUM) { - ret = is_cksum_tcpbad(p); + ret = ol_chksum_check_tcp(p); } else { ret = (u64_t)ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, ip_current_src_addr(), ip_current_dest_addr()); @@ -271,12 +271,9 @@ tcp_input(struct pbuf *p, struct netif *inp) #if GAZELLE_TCP_PCB_HASH pcb = NULL; - u32_t idx; struct hlist_head *head; - idx = TUPLE4_HASH_FN( ip_current_dest_addr()->addr, tcphdr->dest, - ip_current_src_addr()->addr, tcphdr->src) & - (tcp_active_htable->size - 1); - head = &tcp_active_htable->array[idx].chain; + u32_t idx = tcp_hash_tuple4_fn(ip_current_dest_addr()->addr, tcphdr->dest, ip_current_src_addr()->addr, tcphdr->src); + head = &tcp_active_htable->array[idx & (tcp_active_htable->size - 1)].chain; hlist_for_each_entry(pcb, head, tcp_node) { #else for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) { @@ -1252,7 +1249,6 @@ tcp_receive(struct tcp_pcb *pcb) s16_t m; u32_t right_wnd_edge; int found_dupack = 0; - int found_dataack = 0; LWIP_ASSERT("tcp_receive: invalid pcb", pcb != NULL); LWIP_ASSERT("tcp_receive: wrong state", pcb->state >= ESTABLISHED); @@ -1329,33 +1325,37 @@ tcp_receive(struct tcp_pcb *pcb) } } } - } else { - /* fast rexmit when receive too many acks with data */ - if (TCP_SEQ_LT(ackno + 1, pcb->snd_nxt)) { - if (pcb->snd_wl2 + pcb->snd_wnd == right_wnd_edge) { - if (pcb->rtime >= 0) { - if (pcb->lastack == ackno) { - found_dataack = 1; - ++pcb->dataacks; - if ((pcb->dataacks > GAZELLE_TCP_MAX_DATA_ACK_NUM) && (pcb->nrtx < (TCP_MAXRTX / 2))) { - if (tcp_rexmit(pcb) == ERR_OK) { - pcb->rtime = 0; - pcb->dataacks = 0; - } + } +#if GAZELLE_TCP_DATAACKS_REXMIT + /* fast rexmit when receive too many acks with data */ + /* if (tcplen != 0) */ + /* and skip (ackno + 1 == pcb->snd_nxt) */ + else if (TCP_SEQ_LT(ackno + 1, pcb->snd_nxt)) { + int found_dataack = 0; + if (pcb->snd_wl2 + pcb->snd_wnd == right_wnd_edge) { + if (pcb->rtime >= 0) { + if (pcb->lastack == ackno) { + found_dataack = 1; + ++pcb->dataacks; + if ((pcb->dataacks > GAZELLE_TCP_DATAACKS_NUM) && (pcb->nrtx < (TCP_MAXRTX / 2))) { + if (tcp_rexmit(pcb) == ERR_OK) { + pcb->rtime = 0; + pcb->dataacks = 0; } } } } } + if (!found_dataack) { + pcb->dataacks = 0; + } } +#endif /* GAZELLE_TCP_DATAACKS_REXMIT */ /* If Clause (1) or more is true, but not a duplicate ack, reset * count of consecutive duplicate acks */ if (!found_dupack) { pcb->dupacks = 0; } - if (!found_dataack) { - pcb->dataacks = 0; - } } else if (TCP_SEQ_BETWEEN(ackno, pcb->lastack + 1, pcb->snd_nxt)) { /* We come here when the ACK acknowledges new data. */ tcpwnd_size_t acked; @@ -1381,7 +1381,9 @@ tcp_receive(struct tcp_pcb *pcb) /* Reset the fast retransmit variables. */ pcb->dupacks = 0; pcb->lastack = ackno; +#if GAZELLE_TCP_DATAACKS_REXMIT pcb->dataacks = 0; +#endif /* Update the congestion control variables (cwnd and ssthresh). */ diff --git a/src/core/tcp_out.c b/src/core/tcp_out.c index fde3b8f..08859f5 100644 --- a/src/core/tcp_out.c +++ b/src/core/tcp_out.c @@ -83,7 +83,7 @@ #if GAZELLE_ENABLE #include "lwipsock.h" #include -#if CHECKSUM_GEN_TCP_HW +#if OFFLOAD_CHECKSUM_GEN_TCP #include "gazelle_dpdk_offload.h" #endif #endif @@ -1825,10 +1825,10 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif #if CHECKSUM_GEN_TCP IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) { -#if CHECKSUM_GEN_TCP_HW +#if OFFLOAD_CHECKSUM_GEN_TCP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) { - tcph_cksum_set(seg->p, TCPH_HDRLEN_BYTES(seg->tcphdr)); - seg->tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP,seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); + ol_chksum_gen_tcp(seg->p, TCPH_HDRLEN_BYTES(seg->tcphdr)); + seg->tcphdr->chksum = ol_ip_chksum_pseudo(IP_PROTO_TCP,seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); } else { #if TCP_CHECKSUM_ON_COPY u32_t acc; @@ -1901,7 +1901,7 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP, seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); #endif /* TCP_CHECKSUM_ON_COPY */ -#endif /* CHECKSUM_GEN_TCP_HW */ +#endif /* OFFLOAD_CHECKSUM_GEN_TCP */ } #endif /* CHECKSUM_GEN_TCP */ TCP_STATS_INC(tcp.xmit); @@ -2261,10 +2261,10 @@ tcp_output_control_segment(struct tcp_pcb *pcb, struct pbuf *p, #if CHECKSUM_GEN_TCP IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) { struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload; -#if CHECKSUM_GEN_TCP_HW +#if OFFLOAD_CHECKSUM_GEN_TCP if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) { - tcph_cksum_set(p, TCPH_HDRLEN_BYTES(tcphdr)); - tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP, p->tot_len, src, dst); + ol_chksum_gen_tcp(p, TCPH_HDRLEN_BYTES(tcphdr)); + tcphdr->chksum = ol_ip_chksum_pseudo(IP_PROTO_TCP, p->tot_len, src, dst); } else { tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, src, dst); diff --git a/src/core/udp.c b/src/core/udp.c index fc86ac5..170c911 100644 --- a/src/core/udp.c +++ b/src/core/udp.c @@ -911,7 +911,7 @@ udp_sendto_if_src_chksum(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *d LWIP_DEBUGF(UDP_DEBUG, ("udp_send: ip_output_if (,,,,0x%02"X16_F",)\n", (u16_t)ip_proto)); /* output to IP */ NETIF_SET_HINTS(netif, &(pcb->netif_hints)); - udph_cksum_set(q, UDP_HLEN); + ol_chksum_gen_udp(q, UDP_HLEN); err = ip_output_if_src(q, src_ip, dst_ip, ttl, pcb->tos, ip_proto, netif); NETIF_RESET_HINTS(netif); diff --git a/src/include/gazelle_dpdk_offload.h b/src/include/gazelle_dpdk_offload.h index 0548258..c905a31 100644 --- a/src/include/gazelle_dpdk_offload.h +++ b/src/include/gazelle_dpdk_offload.h @@ -34,60 +34,59 @@ #define _GAZELLE_DPDK_OFFLOAD_H_ #include "lwipopts.h" + #if GAZELLE_ENABLE #include - -#if CHECKSUM_OFFLOAD_ALL #include #include "lwip/pbuf.h" -#endif extern uint64_t get_eth_params_rx_ol(void); extern uint64_t get_eth_params_tx_ol(void); -#if CHECKSUM_CHECK_IP_HW -// for ip4_input -static inline u64_t is_cksum_ipbad(struct pbuf *p) { + +#if OFFLOAD_CHECKSUM_CHECK_IP +// replaces inet_chksum() for ip4_input +static inline u64_t ol_chksum_check_ip(struct pbuf *p) { return p->ol_flags & (RTE_MBUF_F_RX_IP_CKSUM_BAD); } -#endif /* CHECKSUM_CHECK_IP_HW */ +#endif /* OFFLOAD_CHECKSUM_CHECK_IP */ -#if CHECKSUM_CHECK_TCP_HW -// for tcp_input -static inline u64_t is_cksum_tcpbad(struct pbuf *p) { +#if OFFLOAD_CHECKSUM_CHECK_TCP +// replace ip_chksum_pseudo() for tcp_input +static inline u64_t ol_chksum_check_tcp(struct pbuf *p) { return p->ol_flags & (RTE_MBUF_F_RX_L4_CKSUM_BAD); } -#endif /* CHECKSUM_CHECK_TCP_HW */ +#endif /* OFFLOAD_CHECKSUM_CHECK_TCP */ -#if CHECKSUM_GEN_IP_HW -static inline void ethh_cksum_set(struct pbuf *p, u16_t len) { +#if OFFLOAD_CHECKSUM_GEN_IP +static inline void ol_chksum_gen_eth(struct pbuf *p, u16_t len) { p->l2_len = len; } -// replaces IPH_CHKSUM_SET -static inline void iph_cksum_set(struct pbuf *p, u16_t len, bool do_ipcksum) { +// replaces inet_chksum() for ip4_output +static inline void ol_chksum_gen_ip(struct pbuf *p, u16_t len, bool do_ipcksum) { p->ol_flags |= RTE_MBUF_F_TX_IPV4; if (do_ipcksum) { p->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; } p->l3_len = len; } -#endif /* CHECKSUM_GEN_IP_HW */ +#endif /* OFFLOAD_CHECKSUM_GEN_IP */ // replace ip_chksum_pseudo -#if CHECKSUM_GEN_TCP_HW +#if OFFLOAD_CHECKSUM_GEN_TCP #include -static inline void tcph_cksum_set(struct pbuf *p, u16_t len) { +// replace ip_chksum_pseudo() for tcp_output +static inline void ol_chksum_gen_tcp(struct pbuf *p, u16_t len) { p->l4_len = len; p->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; } -static inline void udph_cksum_set(struct pbuf *p, u16_t len) { +static inline void ol_chksum_gen_udp(struct pbuf *p, u16_t len) { p->l4_len = len; } -static inline u16_t ip_chksum_pseudo_offload(u8_t proto, u16_t proto_len, - const ip_addr_t *src, const ip_addr_t *dst) +static inline u16_t ol_ip_chksum_pseudo(u8_t proto, u16_t proto_len, const ip_addr_t *src, const ip_addr_t *dst) { struct ipv4_psd_header { uint32_t src_addr; /* IP address of source host. */ @@ -105,7 +104,7 @@ static inline u16_t ip_chksum_pseudo_offload(u8_t proto, u16_t proto_len, return rte_raw_cksum(&psd_hdr, sizeof(psd_hdr)); } -#endif /* CHECKSUM_GEN_TCP_HW */ +#endif /* OFFLOAD_CHECKSUM_GEN_TCP */ #endif /* GAZELLE_ENABLE */ #endif /* _GAZELLE_DPDK_OFFLOAD_H_ */ diff --git a/src/include/gazelle_tcp_priv.h b/src/include/gazelle_tcp_priv.h index 01d4953..fee9a0a 100644 --- a/src/include/gazelle_tcp_priv.h +++ b/src/include/gazelle_tcp_priv.h @@ -35,7 +35,6 @@ #include "lwip/opt.h" #include "lwip/sys.h" -#include "gazelle_hlist.h" #define __TCP_REG(pcbs, npcb) \ do { \ @@ -96,6 +95,10 @@ #endif /* LWIP_DEBUG */ #if GAZELLE_TCP_PCB_HASH +#include "gazelle_hlist.h" +#include +#define tcp_hash_tuple4_fn(lip, lport, rip, rport) rte_jhash_3words(lip, rip, lport|(rport<<16), 0) + struct tcp_hashbucket { sys_mutex_t mutex; struct hlist_head chain; @@ -106,17 +109,13 @@ struct tcp_hash_table { }; extern PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/lip/fip */ -#include -#define TUPLE4_HASH_FN(laddr, lport, faddr, fport) rte_jhash_3words(laddr, faddr, lport|(fport<<16), 0) - #define TCP_REG_HASH(pcbs, npcb) \ do { \ struct hlist_head *head; \ struct tcp_hash_table *htb = pcbs; \ - u32_t idx = TUPLE4_HASH_FN((npcb)->local_ip.addr, (npcb)->local_port, \ - (npcb)->remote_ip.addr, (npcb)->remote_port) & \ - (htb->size - 1); \ - head = &htb->array[idx].chain; \ + u32_t idx = tcp_hash_tuple4_fn((npcb)->local_ip.addr, (npcb)->local_port, \ + (npcb)->remote_ip.addr, (npcb)->remote_port); \ + head = &htb->array[idx & (htb->size - 1)].chain; \ hlist_add_head(&(npcb)->tcp_node, head); \ tcp_timer_needed(); \ } while (0) @@ -152,4 +151,42 @@ extern PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/ } while (0) #endif /* GAZELLE_TCP_REUSE_IPPORT */ + +#if GAZELLE_ENABLE +#include "gazelle_tcp_reg.h" +static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pcb *pcb) +{ + LWIP_ASSERT("Invalid parameter", pcb != NULL); + + struct gazelle_quintuple qtuple; + qtuple.protocol = 0; + qtuple.src_ip = pcb->local_ip.addr; + qtuple.src_port = lwip_htons(pcb->local_port); + qtuple.dst_ip = pcb->remote_ip.addr; + qtuple.dst_port = lwip_htons(pcb->remote_port); + +#if GAZELLE_TCP_REUSE_IPPORT + if (reg_type == REG_RING_TCP_CONNECT_CLOSE) { + struct tcp_pcb_listen* lpcb = pcb->listener; + if (lpcb != NULL) { + lpcb->connect_num--; + } + } +#endif + + return vdev_reg_xmit(reg_type, &qtuple); +} +static inline void vdev_unreg_done(const struct tcp_pcb *pcb) +{ + if (pcb->local_port == 0) { + return; + } + if (pcb->state == LISTEN) { + vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, pcb); + } else { + vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, pcb); + } +} +#endif + #endif /* __GAZELLE_TCP_PRIV_H__ */ \ No newline at end of file diff --git a/src/include/lwip/pbuf.h b/src/include/lwip/pbuf.h index fb21134..728c5e4 100644 --- a/src/include/lwip/pbuf.h +++ b/src/include/lwip/pbuf.h @@ -222,7 +222,7 @@ struct pbuf { /** For incoming packets, this contains the input netif's index */ u8_t if_idx; -#if GAZELLE_ENABLE && CHECKSUM_OFFLOAD_ALL +#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_ENABLE /** checksum offload ol_flags */ u64_t ol_flags; /* < L2 (MAC) Header Length for non-tunneling pkt. */ diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h index 7a04c99..074528e 100644 --- a/src/include/lwip/priv/tcp_priv.h +++ b/src/include/lwip/priv/tcp_priv.h @@ -340,43 +340,6 @@ extern PER_THREAD struct tcp_pcb *tcp_tw_pcbs; /* List of all TCP PCBs in T #define NUM_TCP_PCB_LISTS 4 extern PER_THREAD struct tcp_pcb ** tcp_pcb_lists[NUM_TCP_PCB_LISTS]; -#if GAZELLE_ENABLE -#include "gazelle_tcp_reg.h" -static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pcb *pcb) -{ - LWIP_ASSERT("Invalid parameter", pcb != NULL); - - struct gazelle_quintuple qtuple; - qtuple.protocol = 0; - qtuple.src_ip = pcb->local_ip.addr; - qtuple.src_port = lwip_htons(pcb->local_port); - qtuple.dst_ip = pcb->remote_ip.addr; - qtuple.dst_port = lwip_htons(pcb->remote_port); - -#if GAZELLE_TCP_REUSE_IPPORT - if (reg_type == REG_RING_TCP_CONNECT_CLOSE) { - struct tcp_pcb_listen* lpcb = pcb->listener; - if (lpcb != NULL) { - lpcb->connect_num--; - } - } -#endif - - return vdev_reg_xmit(reg_type, &qtuple); -} -static inline void vdev_unreg_done(const struct tcp_pcb *pcb) -{ - if (pcb->local_port == 0) { - return; - } - if (pcb->state == LISTEN) { - vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, pcb); - } else { - vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, pcb); - } -} -#endif - /* Axioms about the above lists: 1) Every TCP PCB that is not CLOSED is in one of the lists. 2) A PCB is only in one of the lists. diff --git a/src/include/lwip/tcp.h b/src/include/lwip/tcp.h index 432282f..ef51895 100644 --- a/src/include/lwip/tcp.h +++ b/src/include/lwip/tcp.h @@ -326,8 +326,10 @@ struct tcp_pcb { /* fast retransmit/recovery */ u8_t dupacks; - u32_t dataacks; u32_t lastack; /* Highest acknowledged seqno. */ +#if GAZELLE_TCP_DATAACKS_REXMIT + u32_t dataacks; +#endif /* congestion avoidance/control variables */ tcpwnd_size_t cwnd; diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h index f39386f..6b5a2d1 100644 --- a/src/include/lwipopts.h +++ b/src/include/lwipopts.h @@ -52,9 +52,12 @@ #define GAZELLE_RESERVED_CLIENTS (2000) #define GAZELLE_TCP_PCB_HASH 1 -#define GAZELLE_TCP_ACTIVE_HTABLE_SIZE (GAZELLE_MAX_CLIENTS >> 1) +#define GAZELLE_TCP_ACTIVE_HTABLE_SIZE 32768 -#define GAZELLE_TCP_MAX_DATA_ACK_NUM 256 +#define GAZELLE_TCP_NEW_PORT 1 + +#define GAZELLE_TCP_DATAACKS_REXMIT 1 +#define GAZELLE_TCP_DATAACKS_NUM 256 #define GAZELLE_TCP_MAX_PBUF_CHAIN_LEN 40 @@ -67,22 +70,16 @@ */ #define LWIP_CHECKSUM_CTRL_PER_NETIF 1 /* checksum ability check before checksum*/ -// rx cksum -#define CHECKSUM_CHECK_IP 1 /* master switch */ -#define CHECKSUM_CHECK_TCP 1 /* master switch */ -// tx cksum -#define CHECKSUM_GEN_IP 1 /* master switch */ -#define CHECKSUM_GEN_TCP 1 /* master switch */ - // rx offload cksum -#define CHECKSUM_CHECK_IP_HW (1 && CHECKSUM_CHECK_IP) /* hardware switch */ -#define CHECKSUM_CHECK_TCP_HW (1 && CHECKSUM_CHECK_TCP) /* hardware switch */ +#define OFFLOAD_CHECKSUM_CHECK_IP (1 && CHECKSUM_CHECK_IP) /* hardware switch */ +#define OFFLOAD_CHECKSUM_CHECK_TCP (1 && CHECKSUM_CHECK_TCP) /* hardware switch */ // tx offload cksum -#define CHECKSUM_GEN_IP_HW (1 && CHECKSUM_GEN_IP) /* hardware switch */ -#define CHECKSUM_GEN_TCP_HW (1 && CHECKSUM_GEN_TCP) /* hardware switch */ - -#define CHECKSUM_OFFLOAD_ALL (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW || CHECKSUM_CHECK_IP_HW || CHECKSUM_CHECK_TCP_HW) +#define OFFLOAD_CHECKSUM_GEN_IP (1 && CHECKSUM_GEN_IP) /* hardware switch */ +#define OFFLOAD_CHECKSUM_GEN_TCP (1 && CHECKSUM_GEN_TCP) /* hardware switch */ +#define OFFLOAD_CHECKSUM_ENABLE \ + (OFFLOAD_CHECKSUM_CHECK_IP || OFFLOAD_CHECKSUM_CHECK_TCP || \ + OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP) /* --------------------------------------- diff --git a/src/netif/ethernet.c b/src/netif/ethernet.c index deddf3f..8310485 100644 --- a/src/netif/ethernet.c +++ b/src/netif/ethernet.c @@ -56,7 +56,7 @@ #include "netif/ppp/pppoe.h" #endif /* PPPOE_SUPPORT */ -#if GAZELLE_ENABLE && (CHECKSUM_GEN_TCP_HW || CHECKSUM_GEN_IP_HW) +#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_GEN_TCP || OFFLOAD_CHECKSUM_GEN_IP) #include "gazelle_dpdk_offload.h" #endif @@ -312,8 +312,8 @@ ethernet_output(struct netif * netif, struct pbuf * p, LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("ethernet_output: sending packet %p\n", (void *)p)); -#if CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW - ethh_cksum_set(p, sizeof(*ethhdr)); +#if OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP + ol_chksum_gen_eth(p, sizeof(*ethhdr)); #endif /* send the packet */ -- 2.22.0.windows.1