316 lines
12 KiB
Diff
316 lines
12 KiB
Diff
From 1929cc2a07a8e6d6a81b13a8338d816d7f09e709 Mon Sep 17 00:00:00 2001
|
|
From: yinbin6 <yinbin8@huawei.com>
|
|
Date: Wed, 6 Mar 2024 10:51:42 +0800
|
|
Subject: [PATCH] FAULT INJECT: add duplicate and reorder methods
|
|
|
|
---
|
|
src/common/gazelle_fault_inject_common.h | 4 +-
|
|
src/lstack/netif/lstack_fault_inject.c | 162 ++++++++++++++++++++++-
|
|
src/ltran/ltran_dfx.c | 20 ++-
|
|
3 files changed, 174 insertions(+), 12 deletions(-)
|
|
|
|
diff --git a/src/common/gazelle_fault_inject_common.h b/src/common/gazelle_fault_inject_common.h
|
|
index 3a77f39..72d778c 100644
|
|
--- a/src/common/gazelle_fault_inject_common.h
|
|
+++ b/src/common/gazelle_fault_inject_common.h
|
|
@@ -17,10 +17,10 @@
|
|
|
|
enum GAZELLE_FAULT_INJECT_TYPE {
|
|
GAZELLE_FAULT_INJECT_TYPE_ERR = 0,
|
|
- GAZELLE_FAULT_INJECT_PACKET_DELAY,
|
|
GAZELLE_FAULT_INJECT_PACKET_LOSS,
|
|
- GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
|
|
GAZELLE_FAULT_INJECT_PACKET_REORDER,
|
|
+ GAZELLE_FAULT_INJECT_PACKET_DELAY,
|
|
+ GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
|
|
GAZELLE_FAULT_INJECT_TYPE_MAX,
|
|
};
|
|
|
|
diff --git a/src/lstack/netif/lstack_fault_inject.c b/src/lstack/netif/lstack_fault_inject.c
|
|
index 41e7d95..4edc6cc 100644
|
|
--- a/src/lstack/netif/lstack_fault_inject.c
|
|
+++ b/src/lstack/netif/lstack_fault_inject.c
|
|
@@ -9,7 +9,7 @@
|
|
* PURPOSE.
|
|
* See the Mulan PSL v2 for more details.
|
|
*/
|
|
-
|
|
+#include <pthread.h>
|
|
#include <securec.h>
|
|
#include <rte_gro.h>
|
|
#include <rte_net.h>
|
|
@@ -31,6 +31,15 @@ struct inject_tbl {
|
|
};
|
|
static struct inject_tbl g_inject_tbl[GAZELLE_FAULT_INJECT_TYPE_MAX];
|
|
|
|
+struct reorder_stat {
|
|
+ int32_t enable;
|
|
+ int32_t arr_size;
|
|
+ int32_t cur_cnt;
|
|
+ struct rte_mbuf **array;
|
|
+};
|
|
+
|
|
+static struct reorder_stat g_reorder[PROTOCOL_STACK_MAX];
|
|
+
|
|
struct inject_func_tbl {
|
|
enum GAZELLE_FAULT_INJECT_TYPE type;
|
|
enum GAZELLE_FAULT_INJECT_RULE rule;
|
|
@@ -41,10 +50,16 @@ static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct r
|
|
uint32_t nr_pkts, struct gazelle_fault_inject_data data);
|
|
static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
|
|
uint32_t nr_pkts, struct gazelle_fault_inject_data data);
|
|
+static int32_t inject_packet_duplicate_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
|
|
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data);
|
|
+static int32_t inject_packet_reorder_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
|
|
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data);
|
|
|
|
static struct inject_func_tbl g_inject_func_tbl[] = {
|
|
{GAZELLE_FAULT_INJECT_PACKET_LOSS, INJECT_LOSS_RANDOM, inject_packet_loss_random},
|
|
{GAZELLE_FAULT_INJECT_PACKET_DELAY, INJECT_DELAY_RANDOM, inject_packet_delay_random},
|
|
+ {GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, INJECT_DUPLICATE_RANDOM, inject_packet_duplicate_random},
|
|
+ {GAZELLE_FAULT_INJECT_PACKET_REORDER, INJECT_REORDER_RANDOM, inject_packet_reorder_random},
|
|
};
|
|
|
|
static int32_t inject_func_tbl_update()
|
|
@@ -68,16 +83,16 @@ static int32_t inject_func_tbl_update()
|
|
static uint32_t inject_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
|
|
{
|
|
for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
|
|
- if (g_inject_tbl[i].inject_data.fault_inject_on) {
|
|
+ if (g_inject_tbl[i].inject_data.fault_inject_on && g_inject_tbl[i].inject_func) {
|
|
int32_t xmit_pkts = 0;
|
|
- xmit_pkts = g_inject_tbl[i].inject_func(stack, pkts, nr_pkts,
|
|
- g_inject_tbl[i].inject_data);
|
|
+ xmit_pkts = g_inject_tbl[i].inject_func(stack, pkts, nr_pkts, g_inject_tbl[i].inject_data);
|
|
if (xmit_pkts == nr_pkts) {
|
|
continue;
|
|
}
|
|
return xmit_pkts;
|
|
}
|
|
}
|
|
+
|
|
if (rte_mbuf_refcnt_read(*pkts) == 1) {
|
|
return nr_pkts;
|
|
}
|
|
@@ -105,7 +120,7 @@ static int32_t inject_strategy_update()
|
|
}
|
|
return 0;
|
|
}
|
|
-
|
|
+
|
|
for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
|
|
struct protocol_stack *stack = stack_group->stacks[i];
|
|
vdev_dev_ops_init(&stack->dev_ops);
|
|
@@ -150,6 +165,132 @@ static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rt
|
|
return nr_pkts;
|
|
}
|
|
|
|
+static int32_t inject_packet_duplicate_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
|
|
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data)
|
|
+{
|
|
+ if (rte_mbuf_refcnt_read(*pkts) == 1) {
|
|
+ return nr_pkts;
|
|
+ }
|
|
+
|
|
+ double duplicate_rate = data.inject_data.duplicate.duplicate_rate;
|
|
+ int32_t boundary = (int32_t) (duplicate_rate * INJECT_MODULO);
|
|
+ int32_t count_max = data.inject_data.duplicate.duplicate_sigle_count;
|
|
+
|
|
+ uint32_t rand_num = rte_rand() % INJECT_MODULO;
|
|
+ if (rand_num > boundary) {
|
|
+ return nr_pkts;
|
|
+ }
|
|
+
|
|
+ struct rte_mempool *mp = stack->rxtx_mbuf_pool;
|
|
+ struct rte_mbuf *mbuf_clone = NULL;
|
|
+ int32_t ret = 0;
|
|
+
|
|
+ for (int32_t i = 0; i < nr_pkts; ++i) {
|
|
+ int32_t count = count_max;
|
|
+ while (count--) {
|
|
+ mbuf_clone = rte_pktmbuf_clone(pkts[i], mp);
|
|
+ rte_pktmbuf_free(pkts[i]);
|
|
+ if (mbuf_clone == NULL) {
|
|
+ LSTACK_LOG(ERR, LSTACK, "fault inject mbuf_clone fail.\n");
|
|
+ return 0;
|
|
+ }
|
|
+ ret = vdev_tx_xmit(stack, &mbuf_clone, 1);
|
|
+ if (ret < 1) {
|
|
+ rte_pktmbuf_free(mbuf_clone);
|
|
+ return ret;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ return nr_pkts;
|
|
+}
|
|
+
|
|
+static int32_t send_reorder_array(struct protocol_stack *stack)
|
|
+{
|
|
+ int32_t idx = stack->stack_idx;
|
|
+ int32_t ret = 0;
|
|
+
|
|
+ for (int32_t i = 0; i < g_reorder[idx].cur_cnt; ++i) {
|
|
+ ret = vdev_tx_xmit(stack, g_reorder[idx].array + i, 1);
|
|
+ if (ret < 1) {
|
|
+ rte_pktmbuf_free(*(g_reorder[idx].array + i));
|
|
+ }
|
|
+ }
|
|
+ g_reorder[idx].cur_cnt = 0;
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+static int32_t inject_packet_reorder_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
|
|
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data)
|
|
+{
|
|
+ if (rte_mbuf_refcnt_read(*pkts) == 1) {
|
|
+ return nr_pkts;
|
|
+ }
|
|
+
|
|
+ double reorder_rate = data.inject_data.reorder.reorder_rate;
|
|
+ int32_t boundary = (int32_t) (reorder_rate * INJECT_MODULO);
|
|
+ int32_t count_max = data.inject_data.reorder.reorder_sigle_count;
|
|
+
|
|
+ uint32_t rand_num = rte_rand() % INJECT_MODULO;
|
|
+ if (rand_num > boundary) {
|
|
+ return nr_pkts;
|
|
+ }
|
|
+
|
|
+ struct rte_mempool *mp = stack->rxtx_mbuf_pool;
|
|
+ struct rte_mbuf *mbuf_clone = NULL;
|
|
+ int32_t idx = stack->stack_idx;
|
|
+ for (int32_t i = 0; i < nr_pkts; ++i) {
|
|
+ if (g_reorder[idx].cur_cnt < count_max) {
|
|
+ mbuf_clone = rte_pktmbuf_clone(pkts[i], mp);
|
|
+ if (mbuf_clone == NULL) {
|
|
+ LSTACK_LOG(ERR, LSTACK, "fault inject mbuf_clone fail.\n");
|
|
+ return 0;
|
|
+ }
|
|
+ *(g_reorder[idx].array + g_reorder[idx].cur_cnt++) = mbuf_clone;
|
|
+ /* func rte_pktmbuf_clone will add refcnt of mbuf, so following operation will free mbuf double */
|
|
+ rte_pktmbuf_free(pkts[i]);
|
|
+ rte_pktmbuf_free(pkts[i]);
|
|
+ } else {
|
|
+ send_reorder_array(stack);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return nr_pkts;
|
|
+}
|
|
+
|
|
+static int32_t inject_reorder_mem_release()
|
|
+{
|
|
+ struct protocol_stack_group *stack_group = get_protocol_stack_group();
|
|
+
|
|
+ for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
|
|
+ struct protocol_stack *stack = stack_group->stacks[i];
|
|
+ if (!g_reorder[i].enable) {
|
|
+ return 0;
|
|
+ }
|
|
+ send_reorder_array(stack);
|
|
+ free(g_reorder[i].array);
|
|
+ g_reorder[i].enable = 0;
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+static int32_t inject_reorder_mem_alloc()
|
|
+{
|
|
+ struct protocol_stack_group *stack_group = get_protocol_stack_group();
|
|
+
|
|
+ for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
|
|
+ g_reorder[i].enable = 1;
|
|
+
|
|
+ g_reorder[i].arr_size =
|
|
+ g_inject_tbl[GAZELLE_FAULT_INJECT_PACKET_REORDER].inject_data.inject_data.reorder.reorder_sigle_count;
|
|
+ g_reorder[i].cur_cnt = 0;
|
|
+ g_reorder[i].array =
|
|
+ (struct rte_mbuf**) malloc(sizeof(struct rte_mbuf**) * (g_reorder[i].arr_size));
|
|
+ }
|
|
+ return 0;
|
|
+}
|
|
+
|
|
static int32_t inject_respond_msg(int32_t sockfd)
|
|
{
|
|
struct gazelle_stack_dfx_data rsp = {0};
|
|
@@ -192,6 +333,10 @@ static int32_t inject_unset_cmd(int32_t sockfd, struct gazelle_fault_inject_data
|
|
}
|
|
}
|
|
|
|
+ if (!g_inject_tbl[GAZELLE_FAULT_INJECT_PACKET_REORDER].inject_data.fault_inject_on) {
|
|
+ inject_reorder_mem_release();
|
|
+ }
|
|
+
|
|
inject_strategy_update();
|
|
|
|
return inject_respond_msg(sockfd);
|
|
@@ -208,8 +353,13 @@ static int32_t inject_set_cmd(int32_t sockfd, struct gazelle_fault_inject_data i
|
|
return -1;
|
|
}
|
|
|
|
+ if (inject.inject_type == GAZELLE_FAULT_INJECT_PACKET_REORDER) {
|
|
+ inject_reorder_mem_release();
|
|
+ inject_reorder_mem_alloc();
|
|
+ }
|
|
+
|
|
inject_strategy_update();
|
|
-
|
|
+
|
|
return inject_respond_msg(sockfd);
|
|
}
|
|
|
|
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
|
|
index 96509e5..2cc3504 100644
|
|
--- a/src/ltran/ltran_dfx.c
|
|
+++ b/src/ltran/ltran_dfx.c
|
|
@@ -77,10 +77,10 @@ struct gazelle_fault_inject_type_list {
|
|
};
|
|
|
|
static struct gazelle_fault_inject_type_list inject_type_list[] = {
|
|
- {"delay", GAZELLE_FAULT_INJECT_PACKET_DELAY, parse_inject_packet_delay_digit},
|
|
{"loss", GAZELLE_FAULT_INJECT_PACKET_LOSS, parse_inject_packet_loss_digit},
|
|
- {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
|
|
{"reorder", GAZELLE_FAULT_INJECT_PACKET_REORDER, parse_inject_packet_reorder_digit},
|
|
+ {"delay", GAZELLE_FAULT_INJECT_PACKET_DELAY, parse_inject_packet_delay_digit},
|
|
+ {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
|
|
};
|
|
|
|
struct gazelle_fault_inject_rule_list {
|
|
@@ -1514,7 +1514,7 @@ static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_dat
|
|
}
|
|
|
|
if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_DELAY) {
|
|
- printf("\t| inject_type: delay | delay_time: %-7d | delay_range: %-3d | "
|
|
+ printf("\t| inject_type: delay | delay_time: %-7d | delay_range: %-3d | "
|
|
"inject_rule: random |\n", inject->inject_data.delay.delay_time,
|
|
inject->inject_data.delay.delay_range);
|
|
}
|
|
@@ -1522,10 +1522,22 @@ static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_dat
|
|
#define INJECT_PERCENT 100
|
|
|
|
if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_LOSS) {
|
|
- printf("\t| inject_type: loss | loss_rate: %-3.1f%% | loss_single_count: %-3d | "
|
|
+ printf("\t| inject_type: loss | loss_rate: %-4.1f%% | loss_single_count: %-3d | "
|
|
"inject_rule: random |\n", inject->inject_data.loss.loss_rate * INJECT_PERCENT,
|
|
inject->inject_data.loss.loss_sigle_count);
|
|
}
|
|
+
|
|
+ if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE) {
|
|
+ printf("\t| inject_type: duplicate | duplicate_rate: %-4.1f%% | duplicate_single_count: %-3d | "
|
|
+ "inject_rule: random |\n", inject->inject_data.duplicate.duplicate_rate * INJECT_PERCENT,
|
|
+ inject->inject_data.duplicate.duplicate_sigle_count);
|
|
+ }
|
|
+
|
|
+ if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_REORDER) {
|
|
+ printf("\t| inject_type: reorder | reorder_rate: %-4.1f%% | reorder_sigle_count: %-3d | "
|
|
+ "inject_rule: random |\n", inject->inject_data.reorder.reorder_rate * INJECT_PERCENT,
|
|
+ inject->inject_data.reorder.reorder_sigle_count);
|
|
+ }
|
|
printf("\n");
|
|
}
|
|
|
|
--
|
|
2.33.0
|
|
|