packages/lwip
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
lwip/0072-cleancode-refactor-OFFLOAD_CHECKSUM-GAZELLE_TCP_DATA.patch
Lemmy Huang 02139c576e cleancode: refactor OFFLOAD_CHECKSUM GAZELLE_TCP_DATAACKS_REXMIT GAZELLE_TCP_NEW_PORT 
Signed-off-by: Lemmy Huang <huangliming5@huawei.com>
(cherry picked from commit d9cb60dd02211c6f69b6293783c3556e9fb4ddbb)
2023-05-29 21:01:58 +08:00

813 lines
28 KiB
Diff
Raw Blame History

From 04b8cb3f4577cd7bf62bc831ca32cdc8229ee5b8 Mon Sep 17 00:00:00 2001
From: Lemmy Huang <huangliming5@huawei.com>
Date: Sat, 27 May 2023 16:14:06 +0800
Subject: [PATCH 4/5] cleancode: refactor OFFLOAD_CHECKSUM
GAZELLE_TCP_DATAACKS_REXMIT GAZELLE_TCP_NEW_PORT
Signed-off-by: Lemmy Huang <huangliming5@huawei.com>
---
src/core/ipv4/icmp.c | 8 ++--
src/core/ipv4/ip4.c | 12 +++---
src/core/ipv4/ip4_frag.c | 14 +++----
src/core/pbuf.c | 2 +-
src/core/tcp.c | 61 +++++++++++++++++++++---------
src/core/tcp_in.c | 54 +++++++++++++-------------
src/core/tcp_out.c | 16 ++++----
src/core/udp.c | 2 +-
src/include/gazelle_dpdk_offload.h | 43 ++++++++++-----------
src/include/gazelle_tcp_priv.h | 53 ++++++++++++++++++++++----
src/include/lwip/pbuf.h | 2 +-
src/include/lwip/priv/tcp_priv.h | 37 ------------------
src/include/lwip/tcp.h | 4 +-
src/include/lwipopts.h | 27 ++++++-------
src/netif/ethernet.c | 6 +--
15 files changed, 183 insertions(+), 158 deletions(-)
diff --git a/src/core/ipv4/icmp.c b/src/core/ipv4/icmp.c
index 2b15ed4..9da81d9 100644
--- a/src/core/ipv4/icmp.c
+++ b/src/core/ipv4/icmp.c
@@ -51,7 +51,7 @@
#include <string.h>
-#if GAZELLE_ENABLE && CHECKSUM_GEN_IP_HW
+#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_GEN_IP
#include "gazelle_dpdk_offload.h"
#endif
@@ -240,11 +240,11 @@ icmp_input(struct pbuf *p, struct netif *inp)
IPH_CHKSUM_SET(iphdr, 0);
#if CHECKSUM_GEN_IP
IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_GEN_IP) {
-#if CHECKSUM_GEN_IP_HW
+#if OFFLOAD_CHECKSUM_GEN_IP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) {
- iph_cksum_set(p, hlen, 1);
+ ol_chksum_gen_ip(p, hlen, 1);
} else {
- iph_cksum_set(p, hlen, 0);
+ ol_chksum_gen_ip(p, hlen, 0);
IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, hlen));
}
#else
diff --git a/src/core/ipv4/ip4.c b/src/core/ipv4/ip4.c
index 079716d..444f496 100644
--- a/src/core/ipv4/ip4.c
+++ b/src/core/ipv4/ip4.c
@@ -59,7 +59,7 @@
#include <string.h>
-#if GAZELLE_ENABLE && (CHECKSUM_CHECK_IP_HW || CHECKSUM_GEN_IP_HW)
+#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_CHECK_IP || OFFLOAD_CHECKSUM_GEN_IP)
#include "gazelle_dpdk_offload.h"
#endif
@@ -501,10 +501,10 @@ ip4_input(struct pbuf *p, struct netif *inp)
/* verify checksum */
#if CHECKSUM_CHECK_IP
IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_IP) {
-#if CHECKSUM_CHECK_IP_HW
+#if OFFLOAD_CHECKSUM_CHECK_IP
u64_t ret;
if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_IPV4_CKSUM) {
- ret = is_cksum_ipbad(p);
+ ret = ol_chksum_check_ip(p);
} else {
ret = (u64_t)inet_chksum(iphdr, iphdr_hlen);
}
@@ -971,11 +971,11 @@ ip4_output_if_opt_src(struct pbuf *p, const ip4_addr_t *src, const ip4_addr_t *d
IPH_CHKSUM_SET(iphdr, 0);
#if CHECKSUM_GEN_IP
IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) {
-#if CHECKSUM_GEN_IP_HW
+#if OFFLOAD_CHECKSUM_GEN_IP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) {
- iph_cksum_set(p, ip_hlen, 1);
+ ol_chksum_gen_ip(p, ip_hlen, 1);
} else {
- iph_cksum_set(p, ip_hlen, 0);
+ ol_chksum_gen_ip(p, ip_hlen, 0);
IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, ip_hlen));
}
#else
diff --git a/src/core/ipv4/ip4_frag.c b/src/core/ipv4/ip4_frag.c
index 4d697c8..4cc48e8 100644
--- a/src/core/ipv4/ip4_frag.c
+++ b/src/core/ipv4/ip4_frag.c
@@ -51,7 +51,7 @@
#include <string.h>
-#if GAZELLE_ENABLE && CHECKSUM_GEN_IP_HW
+#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_GEN_IP
#include "gazelle_dpdk_offload.h"
#endif
@@ -641,11 +641,11 @@ ip4_reass(struct pbuf *p)
/* @todo: do we need to set/calculate the correct checksum? */
#if CHECKSUM_GEN_IP
IF__NETIF_CHECKSUM_ENABLED(ip_current_input_netif(), NETIF_CHECKSUM_GEN_IP) {
-#if CHECKSUM_GEN_IP_HW
+#if OFFLOAD_CHECKSUM_GEN_IP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) {
- iph_cksum_set(p, IP_HLEN, 1);
+ ol_chksum_gen_ip(p, IP_HLEN, 1);
} else {
- iph_cksum_set(p, IP_HLEN, 0);
+ ol_chksum_gen_ip(p, IP_HLEN, 0);
IPH_CHKSUM_SET(fraghdr, inet_chksum(fraghdr, IP_HLEN));
}
#else
@@ -884,11 +884,11 @@ ip4_frag(struct pbuf *p, struct netif *netif, const ip4_addr_t *dest)
IPH_CHKSUM_SET(iphdr, 0);
#if CHECKSUM_GEN_IP
IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) {
-#if CHECKSUM_GEN_IP_HW
+#if OFFLOAD_CHECKSUM_GEN_IP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) {
- iph_cksum_set(p, IP_HLEN, 1);
+ ol_chksum_gen_ip(p, IP_HLEN, 1);
} else {
- iph_cksum_set(p, IP_HLEN, 0);
+ ol_chksum_gen_ip(p, IP_HLEN, 0);
IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, IP_HLEN));
}
diff --git a/src/core/pbuf.c b/src/core/pbuf.c
index 16615c6..bc629b4 100644
--- a/src/core/pbuf.c
+++ b/src/core/pbuf.c
@@ -1030,7 +1030,7 @@ pbuf_copy_partial_pbuf(struct pbuf *p_to, const struct pbuf *p_from, u16_t copy_
len_calc = p_to->len - offset_to;
}
-#if GAZELLE_ENABLE && (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW)
+#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP)
p_to->l2_len = p_from->l2_len;
p_to->l3_len = p_from->l3_len;
p_to->ol_flags = p_from->ol_flags;
diff --git a/src/core/tcp.c b/src/core/tcp.c
index caba39d..e3355dc 100644
--- a/src/core/tcp.c
+++ b/src/core/tcp.c
@@ -189,19 +189,21 @@ PER_THREAD u8_t tcp_active_pcbs_changed;
/** Timer counter to handle calling slow-timer from tcp_tmr() */
static PER_THREAD u8_t tcp_timer;
static PER_THREAD u8_t tcp_timer_ctr;
-#if GAZELLE_ENABLE
+
+#if GAZELLE_TCP_NEW_PORT
static u16_t tcp_new_port(struct tcp_pcb *pcb);
static pthread_mutex_t g_tcp_port_mutex = PTHREAD_MUTEX_INITIALIZER;
static u8_t port_state[TCP_LOCAL_PORT_RANGE_END - TCP_LOCAL_PORT_RANGE_START + 1] = {0};
+
void release_port(u16_t port)
{
if (port >= TCP_LOCAL_PORT_RANGE_START && port <= TCP_LOCAL_PORT_RANGE_END) {
port_state[port - TCP_LOCAL_PORT_RANGE_START] = 0;
}
}
-#else /* GAZELLE_ENABLE */
+#else /* GAZELLE_TCP_NEW_PORT */
static u16_t tcp_new_port(void);
-#endif /* GAZELLE_ENABLE */
+#endif /* GAZELLE_TCP_NEW_PORT */
#if GAZELLE_TCP_PCB_HASH
PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/lip/fip */
@@ -255,6 +257,8 @@ tcp_free(struct tcp_pcb *pcb)
rte_memzone_free(sock->same_node_tx_ring_mz);
}
vdev_unreg_done(pcb);
+#endif
+#if GAZELLE_TCP_NEW_PORT
release_port(pcb->local_port);
#endif
LWIP_ASSERT("tcp_free: LISTEN", pcb->state != LISTEN);
@@ -755,7 +759,7 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port)
#endif /* LWIP_IPV6 && LWIP_IPV6_SCOPES */
if (port == 0) {
-#if GAZELLE_ENABLE
+#if GAZELLE_TCP_NEW_PORT
port = tcp_new_port(pcb);
#else
port = tcp_new_port();
@@ -986,11 +990,12 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
/* copy over ext_args to listening pcb */
memcpy(&lpcb->ext_args, &pcb->ext_args, sizeof(pcb->ext_args));
#endif
-#if GAZELLE_ENABLE
+#if GAZELLE_TCP_NEW_PORT
/* pcb transfer to lpcb and reg into tcp_listen_pcbs. freeing pcb shouldn't release sock table in here.
* local_port=0 avoid to release sock table in tcp_free */
pcb->local_port = 0;
-
+#endif /* GAZELLE_TCP_NEW_PORT */
+#if GAZELLE_ENABLE
char name[RING_NAME_LEN];
snprintf(name, sizeof(name), "listen_rx_ring_%u", lpcb->local_port);
if (rte_ring_lookup(name) != NULL) {
@@ -999,7 +1004,7 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
} else {
same_node_ring_create(&lpcb->listen_rx_ring, SAME_NODE_RING_SIZE, lpcb->local_port, "listen", "rx");
}
-#endif
+#endif /* GAZELLE_ENABLE */
tcp_free(pcb);
#if LWIP_CALLBACK_API
lpcb->accept = tcp_accept_null;
@@ -1112,13 +1117,9 @@ tcp_recved(struct tcp_pcb *pcb, u16_t len)
*
* @return a new (free) local TCP port number
*/
-#if GAZELLE_ENABLE
+#if GAZELLE_TCP_NEW_PORT
static u16_t
tcp_new_port(struct tcp_pcb *pcb)
-#else
-static u16_t
-tcp_new_port(void)
-#endif
{
u16_t n = 0;
u16_t tmp_port = 0;
@@ -1131,16 +1132,11 @@ tcp_new_port(void)
}
if (__atomic_load_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], __ATOMIC_ACQUIRE) == 0) {
-#if GAZELLE_ENABLE
if (port_in_stack_queue(pcb->remote_ip.addr, pcb->local_ip.addr, pcb->remote_port, tcp_port)) {
tmp_port = tcp_port;
__atomic_store_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], 1, __ATOMIC_RELEASE);
break;
}
-#else
- __atomic_store_n(&port_state[tcp_port - TCP_LOCAL_PORT_RANGE_START], 1, __ATOMIC_RELEASE);
- break;
-#endif
}
n++;
@@ -1154,6 +1150,35 @@ tcp_new_port(void)
return tmp_port;
}
+#else /* GAZELLE_TCP_NEW_PORT */
+static u16_t
+tcp_new_port(void)
+{
+ u8_t i;
+ u16_t n = 0;
+ struct tcp_pcb *pcb;
+
+again:
+ tcp_port++;
+ if (tcp_port == TCP_LOCAL_PORT_RANGE_END) {
+ tcp_port = TCP_LOCAL_PORT_RANGE_START;
+ }
+ /* Check all PCB lists. */
+ for (i = 0; i < NUM_TCP_PCB_LISTS; i++) {
+ for (pcb = *tcp_pcb_lists[i]; pcb != NULL; pcb = pcb->next) {
+ if (pcb->local_port == tcp_port) {
+ n++;
+ if (n > (TCP_LOCAL_PORT_RANGE_END - TCP_LOCAL_PORT_RANGE_START)) {
+ return 0;
+ }
+ goto again;
+ }
+ }
+ }
+ return tcp_port;
+}
+#endif /* GAZELLE_TCP_NEW_PORT */
+
/**
* @ingroup tcp_raw
* Connects to another host. The function given as the "connected"
@@ -1234,7 +1259,7 @@ tcp_connect(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port,
old_local_port = pcb->local_port;
if (pcb->local_port == 0) {
-#if GAZELLE_ENABLE
+#if GAZELLE_TCP_NEW_PORT
pcb->local_port = tcp_new_port(pcb);
#else
pcb->local_port = tcp_new_port();
diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c
index 575bf05..70c125a 100644
--- a/src/core/tcp_in.c
+++ b/src/core/tcp_in.c
@@ -65,9 +65,9 @@
#include <string.h>
-#if GAZELLE_ENABLE && CHECKSUM_CHECK_TCP_HW
+#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_CHECK_TCP
#include <gazelle_dpdk_offload.h>
-#endif /* CHECKSUM_CHECK_TCP_HW */
+#endif /* OFFLOAD_CHECKSUM_CHECK_TCP */
#ifdef LWIP_HOOK_FILENAME
#include LWIP_HOOK_FILENAME
@@ -168,10 +168,10 @@ tcp_input(struct pbuf *p, struct netif *inp)
#if CHECKSUM_CHECK_TCP
IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_TCP) {
/* Verify TCP checksum. */
-#if CHECKSUM_CHECK_TCP_HW
+#if OFFLOAD_CHECKSUM_CHECK_TCP
u64_t ret;
if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_TCP_CKSUM) {
- ret = is_cksum_tcpbad(p);
+ ret = ol_chksum_check_tcp(p);
} else {
ret = (u64_t)ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len,
ip_current_src_addr(), ip_current_dest_addr());
@@ -271,12 +271,9 @@ tcp_input(struct pbuf *p, struct netif *inp)
#if GAZELLE_TCP_PCB_HASH
pcb = NULL;
- u32_t idx;
struct hlist_head *head;
- idx = TUPLE4_HASH_FN( ip_current_dest_addr()->addr, tcphdr->dest,
- ip_current_src_addr()->addr, tcphdr->src) &
- (tcp_active_htable->size - 1);
- head = &tcp_active_htable->array[idx].chain;
+ u32_t idx = tcp_hash_tuple4_fn(ip_current_dest_addr()->addr, tcphdr->dest, ip_current_src_addr()->addr, tcphdr->src);
+ head = &tcp_active_htable->array[idx & (tcp_active_htable->size - 1)].chain;
hlist_for_each_entry(pcb, head, tcp_node) {
#else
for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) {
@@ -1252,7 +1249,6 @@ tcp_receive(struct tcp_pcb *pcb)
s16_t m;
u32_t right_wnd_edge;
int found_dupack = 0;
- int found_dataack = 0;
LWIP_ASSERT("tcp_receive: invalid pcb", pcb != NULL);
LWIP_ASSERT("tcp_receive: wrong state", pcb->state >= ESTABLISHED);
@@ -1329,33 +1325,37 @@ tcp_receive(struct tcp_pcb *pcb)
}
}
}
- } else {
- /* fast rexmit when receive too many acks with data */
- if (TCP_SEQ_LT(ackno + 1, pcb->snd_nxt)) {
- if (pcb->snd_wl2 + pcb->snd_wnd == right_wnd_edge) {
- if (pcb->rtime >= 0) {
- if (pcb->lastack == ackno) {
- found_dataack = 1;
- ++pcb->dataacks;
- if ((pcb->dataacks > GAZELLE_TCP_MAX_DATA_ACK_NUM) && (pcb->nrtx < (TCP_MAXRTX / 2))) {
- if (tcp_rexmit(pcb) == ERR_OK) {
- pcb->rtime = 0;
- pcb->dataacks = 0;
- }
+ }
+#if GAZELLE_TCP_DATAACKS_REXMIT
+ /* fast rexmit when receive too many acks with data */
+ /* if (tcplen != 0) */
+ /* and skip (ackno + 1 == pcb->snd_nxt) */
+ else if (TCP_SEQ_LT(ackno + 1, pcb->snd_nxt)) {
+ int found_dataack = 0;
+ if (pcb->snd_wl2 + pcb->snd_wnd == right_wnd_edge) {
+ if (pcb->rtime >= 0) {
+ if (pcb->lastack == ackno) {
+ found_dataack = 1;
+ ++pcb->dataacks;
+ if ((pcb->dataacks > GAZELLE_TCP_DATAACKS_NUM) && (pcb->nrtx < (TCP_MAXRTX / 2))) {
+ if (tcp_rexmit(pcb) == ERR_OK) {
+ pcb->rtime = 0;
+ pcb->dataacks = 0;
}
}
}
}
}
+ if (!found_dataack) {
+ pcb->dataacks = 0;
+ }
}
+#endif /* GAZELLE_TCP_DATAACKS_REXMIT */
/* If Clause (1) or more is true, but not a duplicate ack, reset
* count of consecutive duplicate acks */
if (!found_dupack) {
pcb->dupacks = 0;
}
- if (!found_dataack) {
- pcb->dataacks = 0;
- }
} else if (TCP_SEQ_BETWEEN(ackno, pcb->lastack + 1, pcb->snd_nxt)) {
/* We come here when the ACK acknowledges new data. */
tcpwnd_size_t acked;
@@ -1381,7 +1381,9 @@ tcp_receive(struct tcp_pcb *pcb)
/* Reset the fast retransmit variables. */
pcb->dupacks = 0;
pcb->lastack = ackno;
+#if GAZELLE_TCP_DATAACKS_REXMIT
pcb->dataacks = 0;
+#endif
/* Update the congestion control variables (cwnd and
ssthresh). */
diff --git a/src/core/tcp_out.c b/src/core/tcp_out.c
index fde3b8f..08859f5 100644
--- a/src/core/tcp_out.c
+++ b/src/core/tcp_out.c
@@ -83,7 +83,7 @@
#if GAZELLE_ENABLE
#include "lwipsock.h"
#include <rte_prefetch.h>
-#if CHECKSUM_GEN_TCP_HW
+#if OFFLOAD_CHECKSUM_GEN_TCP
#include "gazelle_dpdk_offload.h"
#endif
#endif
@@ -1825,10 +1825,10 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
#if CHECKSUM_GEN_TCP
IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) {
-#if CHECKSUM_GEN_TCP_HW
+#if OFFLOAD_CHECKSUM_GEN_TCP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) {
- tcph_cksum_set(seg->p, TCPH_HDRLEN_BYTES(seg->tcphdr));
- seg->tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP,seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip);
+ ol_chksum_gen_tcp(seg->p, TCPH_HDRLEN_BYTES(seg->tcphdr));
+ seg->tcphdr->chksum = ol_ip_chksum_pseudo(IP_PROTO_TCP,seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip);
} else {
#if TCP_CHECKSUM_ON_COPY
u32_t acc;
@@ -1901,7 +1901,7 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP,
seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip);
#endif /* TCP_CHECKSUM_ON_COPY */
-#endif /* CHECKSUM_GEN_TCP_HW */
+#endif /* OFFLOAD_CHECKSUM_GEN_TCP */
}
#endif /* CHECKSUM_GEN_TCP */
TCP_STATS_INC(tcp.xmit);
@@ -2261,10 +2261,10 @@ tcp_output_control_segment(struct tcp_pcb *pcb, struct pbuf *p,
#if CHECKSUM_GEN_TCP
IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) {
struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload;
-#if CHECKSUM_GEN_TCP_HW
+#if OFFLOAD_CHECKSUM_GEN_TCP
if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) {
- tcph_cksum_set(p, TCPH_HDRLEN_BYTES(tcphdr));
- tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP, p->tot_len, src, dst);
+ ol_chksum_gen_tcp(p, TCPH_HDRLEN_BYTES(tcphdr));
+ tcphdr->chksum = ol_ip_chksum_pseudo(IP_PROTO_TCP, p->tot_len, src, dst);
} else {
tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len,
src, dst);
diff --git a/src/core/udp.c b/src/core/udp.c
index fc86ac5..170c911 100644
--- a/src/core/udp.c
+++ b/src/core/udp.c
@@ -911,7 +911,7 @@ udp_sendto_if_src_chksum(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *d
LWIP_DEBUGF(UDP_DEBUG, ("udp_send: ip_output_if (,,,,0x%02"X16_F",)\n", (u16_t)ip_proto));
/* output to IP */
NETIF_SET_HINTS(netif, &(pcb->netif_hints));
- udph_cksum_set(q, UDP_HLEN);
+ ol_chksum_gen_udp(q, UDP_HLEN);
err = ip_output_if_src(q, src_ip, dst_ip, ttl, pcb->tos, ip_proto, netif);
NETIF_RESET_HINTS(netif);
diff --git a/src/include/gazelle_dpdk_offload.h b/src/include/gazelle_dpdk_offload.h
index 0548258..c905a31 100644
--- a/src/include/gazelle_dpdk_offload.h
+++ b/src/include/gazelle_dpdk_offload.h
@@ -34,60 +34,59 @@
#define _GAZELLE_DPDK_OFFLOAD_H_
#include "lwipopts.h"
+
#if GAZELLE_ENABLE
#include <rte_ethdev.h>
-
-#if CHECKSUM_OFFLOAD_ALL
#include <rte_mbuf_core.h>
#include "lwip/pbuf.h"
-#endif
extern uint64_t get_eth_params_rx_ol(void);
extern uint64_t get_eth_params_tx_ol(void);
-#if CHECKSUM_CHECK_IP_HW
-// for ip4_input
-static inline u64_t is_cksum_ipbad(struct pbuf *p) {
+
+#if OFFLOAD_CHECKSUM_CHECK_IP
+// replaces inet_chksum() for ip4_input
+static inline u64_t ol_chksum_check_ip(struct pbuf *p) {
return p->ol_flags & (RTE_MBUF_F_RX_IP_CKSUM_BAD);
}
-#endif /* CHECKSUM_CHECK_IP_HW */
+#endif /* OFFLOAD_CHECKSUM_CHECK_IP */
-#if CHECKSUM_CHECK_TCP_HW
-// for tcp_input
-static inline u64_t is_cksum_tcpbad(struct pbuf *p) {
+#if OFFLOAD_CHECKSUM_CHECK_TCP
+// replace ip_chksum_pseudo() for tcp_input
+static inline u64_t ol_chksum_check_tcp(struct pbuf *p) {
return p->ol_flags & (RTE_MBUF_F_RX_L4_CKSUM_BAD);
}
-#endif /* CHECKSUM_CHECK_TCP_HW */
+#endif /* OFFLOAD_CHECKSUM_CHECK_TCP */
-#if CHECKSUM_GEN_IP_HW
-static inline void ethh_cksum_set(struct pbuf *p, u16_t len) {
+#if OFFLOAD_CHECKSUM_GEN_IP
+static inline void ol_chksum_gen_eth(struct pbuf *p, u16_t len) {
p->l2_len = len;
}
-// replaces IPH_CHKSUM_SET
-static inline void iph_cksum_set(struct pbuf *p, u16_t len, bool do_ipcksum) {
+// replaces inet_chksum() for ip4_output
+static inline void ol_chksum_gen_ip(struct pbuf *p, u16_t len, bool do_ipcksum) {
p->ol_flags |= RTE_MBUF_F_TX_IPV4;
if (do_ipcksum) {
p->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
}
p->l3_len = len;
}
-#endif /* CHECKSUM_GEN_IP_HW */
+#endif /* OFFLOAD_CHECKSUM_GEN_IP */
// replace ip_chksum_pseudo
-#if CHECKSUM_GEN_TCP_HW
+#if OFFLOAD_CHECKSUM_GEN_TCP
#include <rte_ip.h>
-static inline void tcph_cksum_set(struct pbuf *p, u16_t len) {
+// replace ip_chksum_pseudo() for tcp_output
+static inline void ol_chksum_gen_tcp(struct pbuf *p, u16_t len) {
p->l4_len = len;
p->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
}
-static inline void udph_cksum_set(struct pbuf *p, u16_t len) {
+static inline void ol_chksum_gen_udp(struct pbuf *p, u16_t len) {
p->l4_len = len;
}
-static inline u16_t ip_chksum_pseudo_offload(u8_t proto, u16_t proto_len,
- const ip_addr_t *src, const ip_addr_t *dst)
+static inline u16_t ol_ip_chksum_pseudo(u8_t proto, u16_t proto_len, const ip_addr_t *src, const ip_addr_t *dst)
{
struct ipv4_psd_header {
uint32_t src_addr; /* IP address of source host. */
@@ -105,7 +104,7 @@ static inline u16_t ip_chksum_pseudo_offload(u8_t proto, u16_t proto_len,
return rte_raw_cksum(&psd_hdr, sizeof(psd_hdr));
}
-#endif /* CHECKSUM_GEN_TCP_HW */
+#endif /* OFFLOAD_CHECKSUM_GEN_TCP */
#endif /* GAZELLE_ENABLE */
#endif /* _GAZELLE_DPDK_OFFLOAD_H_ */
diff --git a/src/include/gazelle_tcp_priv.h b/src/include/gazelle_tcp_priv.h
index 01d4953..fee9a0a 100644
--- a/src/include/gazelle_tcp_priv.h
+++ b/src/include/gazelle_tcp_priv.h
@@ -35,7 +35,6 @@
#include "lwip/opt.h"
#include "lwip/sys.h"
-#include "gazelle_hlist.h"
#define __TCP_REG(pcbs, npcb) \
do { \
@@ -96,6 +95,10 @@
#endif /* LWIP_DEBUG */
#if GAZELLE_TCP_PCB_HASH
+#include "gazelle_hlist.h"
+#include <rte_jhash.h>
+#define tcp_hash_tuple4_fn(lip, lport, rip, rport) rte_jhash_3words(lip, rip, lport|(rport<<16), 0)
+
struct tcp_hashbucket {
sys_mutex_t mutex;
struct hlist_head chain;
@@ -106,17 +109,13 @@ struct tcp_hash_table {
};
extern PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/lip/fip */
-#include <rte_jhash.h>
-#define TUPLE4_HASH_FN(laddr, lport, faddr, fport) rte_jhash_3words(laddr, faddr, lport|(fport<<16), 0)
-
#define TCP_REG_HASH(pcbs, npcb) \
do { \
struct hlist_head *head; \
struct tcp_hash_table *htb = pcbs; \
- u32_t idx = TUPLE4_HASH_FN((npcb)->local_ip.addr, (npcb)->local_port, \
- (npcb)->remote_ip.addr, (npcb)->remote_port) & \
- (htb->size - 1); \
- head = &htb->array[idx].chain; \
+ u32_t idx = tcp_hash_tuple4_fn((npcb)->local_ip.addr, (npcb)->local_port, \
+ (npcb)->remote_ip.addr, (npcb)->remote_port); \
+ head = &htb->array[idx & (htb->size - 1)].chain; \
hlist_add_head(&(npcb)->tcp_node, head); \
tcp_timer_needed(); \
} while (0)
@@ -152,4 +151,42 @@ extern PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/
} while (0)
#endif /* GAZELLE_TCP_REUSE_IPPORT */
+
+#if GAZELLE_ENABLE
+#include "gazelle_tcp_reg.h"
+static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pcb *pcb)
+{
+ LWIP_ASSERT("Invalid parameter", pcb != NULL);
+
+ struct gazelle_quintuple qtuple;
+ qtuple.protocol = 0;
+ qtuple.src_ip = pcb->local_ip.addr;
+ qtuple.src_port = lwip_htons(pcb->local_port);
+ qtuple.dst_ip = pcb->remote_ip.addr;
+ qtuple.dst_port = lwip_htons(pcb->remote_port);
+
+#if GAZELLE_TCP_REUSE_IPPORT
+ if (reg_type == REG_RING_TCP_CONNECT_CLOSE) {
+ struct tcp_pcb_listen* lpcb = pcb->listener;
+ if (lpcb != NULL) {
+ lpcb->connect_num--;
+ }
+ }
+#endif
+
+ return vdev_reg_xmit(reg_type, &qtuple);
+}
+static inline void vdev_unreg_done(const struct tcp_pcb *pcb)
+{
+ if (pcb->local_port == 0) {
+ return;
+ }
+ if (pcb->state == LISTEN) {
+ vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, pcb);
+ } else {
+ vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, pcb);
+ }
+}
+#endif
+
#endif /* __GAZELLE_TCP_PRIV_H__ */
\ No newline at end of file
diff --git a/src/include/lwip/pbuf.h b/src/include/lwip/pbuf.h
index fb21134..728c5e4 100644
--- a/src/include/lwip/pbuf.h
+++ b/src/include/lwip/pbuf.h
@@ -222,7 +222,7 @@ struct pbuf {
/** For incoming packets, this contains the input netif's index */
u8_t if_idx;
-#if GAZELLE_ENABLE && CHECKSUM_OFFLOAD_ALL
+#if GAZELLE_ENABLE && OFFLOAD_CHECKSUM_ENABLE
/** checksum offload ol_flags */
u64_t ol_flags;
/* < L2 (MAC) Header Length for non-tunneling pkt. */
diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h
index 7a04c99..074528e 100644
--- a/src/include/lwip/priv/tcp_priv.h
+++ b/src/include/lwip/priv/tcp_priv.h
@@ -340,43 +340,6 @@ extern PER_THREAD struct tcp_pcb *tcp_tw_pcbs; /* List of all TCP PCBs in T
#define NUM_TCP_PCB_LISTS 4
extern PER_THREAD struct tcp_pcb ** tcp_pcb_lists[NUM_TCP_PCB_LISTS];
-#if GAZELLE_ENABLE
-#include "gazelle_tcp_reg.h"
-static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pcb *pcb)
-{
- LWIP_ASSERT("Invalid parameter", pcb != NULL);
-
- struct gazelle_quintuple qtuple;
- qtuple.protocol = 0;
- qtuple.src_ip = pcb->local_ip.addr;
- qtuple.src_port = lwip_htons(pcb->local_port);
- qtuple.dst_ip = pcb->remote_ip.addr;
- qtuple.dst_port = lwip_htons(pcb->remote_port);
-
-#if GAZELLE_TCP_REUSE_IPPORT
- if (reg_type == REG_RING_TCP_CONNECT_CLOSE) {
- struct tcp_pcb_listen* lpcb = pcb->listener;
- if (lpcb != NULL) {
- lpcb->connect_num--;
- }
- }
-#endif
-
- return vdev_reg_xmit(reg_type, &qtuple);
-}
-static inline void vdev_unreg_done(const struct tcp_pcb *pcb)
-{
- if (pcb->local_port == 0) {
- return;
- }
- if (pcb->state == LISTEN) {
- vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, pcb);
- } else {
- vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, pcb);
- }
-}
-#endif
-
/* Axioms about the above lists:
1) Every TCP PCB that is not CLOSED is in one of the lists.
2) A PCB is only in one of the lists.
diff --git a/src/include/lwip/tcp.h b/src/include/lwip/tcp.h
index 432282f..ef51895 100644
--- a/src/include/lwip/tcp.h
+++ b/src/include/lwip/tcp.h
@@ -326,8 +326,10 @@ struct tcp_pcb {
/* fast retransmit/recovery */
u8_t dupacks;
- u32_t dataacks;
u32_t lastack; /* Highest acknowledged seqno. */
+#if GAZELLE_TCP_DATAACKS_REXMIT
+ u32_t dataacks;
+#endif
/* congestion avoidance/control variables */
tcpwnd_size_t cwnd;
diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h
index f39386f..6b5a2d1 100644
--- a/src/include/lwipopts.h
+++ b/src/include/lwipopts.h
@@ -52,9 +52,12 @@
#define GAZELLE_RESERVED_CLIENTS (2000)
#define GAZELLE_TCP_PCB_HASH 1
-#define GAZELLE_TCP_ACTIVE_HTABLE_SIZE (GAZELLE_MAX_CLIENTS >> 1)
+#define GAZELLE_TCP_ACTIVE_HTABLE_SIZE 32768
-#define GAZELLE_TCP_MAX_DATA_ACK_NUM 256
+#define GAZELLE_TCP_NEW_PORT 1
+
+#define GAZELLE_TCP_DATAACKS_REXMIT 1
+#define GAZELLE_TCP_DATAACKS_NUM 256
#define GAZELLE_TCP_MAX_PBUF_CHAIN_LEN 40
@@ -67,22 +70,16 @@
*/
#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 /* checksum ability check before checksum*/
-// rx cksum
-#define CHECKSUM_CHECK_IP 1 /* master switch */
-#define CHECKSUM_CHECK_TCP 1 /* master switch */
-// tx cksum
-#define CHECKSUM_GEN_IP 1 /* master switch */
-#define CHECKSUM_GEN_TCP 1 /* master switch */
-
// rx offload cksum
-#define CHECKSUM_CHECK_IP_HW (1 && CHECKSUM_CHECK_IP) /* hardware switch */
-#define CHECKSUM_CHECK_TCP_HW (1 && CHECKSUM_CHECK_TCP) /* hardware switch */
+#define OFFLOAD_CHECKSUM_CHECK_IP (1 && CHECKSUM_CHECK_IP) /* hardware switch */
+#define OFFLOAD_CHECKSUM_CHECK_TCP (1 && CHECKSUM_CHECK_TCP) /* hardware switch */
// tx offload cksum
-#define CHECKSUM_GEN_IP_HW (1 && CHECKSUM_GEN_IP) /* hardware switch */
-#define CHECKSUM_GEN_TCP_HW (1 && CHECKSUM_GEN_TCP) /* hardware switch */
-
-#define CHECKSUM_OFFLOAD_ALL (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW || CHECKSUM_CHECK_IP_HW || CHECKSUM_CHECK_TCP_HW)
+#define OFFLOAD_CHECKSUM_GEN_IP (1 && CHECKSUM_GEN_IP) /* hardware switch */
+#define OFFLOAD_CHECKSUM_GEN_TCP (1 && CHECKSUM_GEN_TCP) /* hardware switch */
+#define OFFLOAD_CHECKSUM_ENABLE \
+ (OFFLOAD_CHECKSUM_CHECK_IP || OFFLOAD_CHECKSUM_CHECK_TCP || \
+ OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP)
/*
---------------------------------------
diff --git a/src/netif/ethernet.c b/src/netif/ethernet.c
index deddf3f..8310485 100644
--- a/src/netif/ethernet.c
+++ b/src/netif/ethernet.c
@@ -56,7 +56,7 @@
#include "netif/ppp/pppoe.h"
#endif /* PPPOE_SUPPORT */
-#if GAZELLE_ENABLE && (CHECKSUM_GEN_TCP_HW || CHECKSUM_GEN_IP_HW)
+#if GAZELLE_ENABLE && (OFFLOAD_CHECKSUM_GEN_TCP || OFFLOAD_CHECKSUM_GEN_IP)
#include "gazelle_dpdk_offload.h"
#endif
@@ -312,8 +312,8 @@ ethernet_output(struct netif * netif, struct pbuf * p,
LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE,
("ethernet_output: sending packet %p\n", (void *)p));
-#if CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW
- ethh_cksum_set(p, sizeof(*ethhdr));
+#if OFFLOAD_CHECKSUM_GEN_IP || OFFLOAD_CHECKSUM_GEN_TCP
+ ol_chksum_gen_eth(p, sizeof(*ethhdr));
#endif
/* send the packet */
--
2.22.0.windows.1
Reference in New Issue View Git Blame Copy Permalink