packages/gazelle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
gazelle/0140-FAULT-INJECT-gazelle-add-packet-delay-and-packet-dro.patch
jiangheng 4c132e3e07 sync FAULT INJECT: gazelle add packet delay and packet drop 
(cherry picked from commit b8f329ae854d5dc96cf94d0b1634eb95881917f4)
2024-03-14 17:06:31 +08:00

799 lines
31 KiB
Diff
Raw Blame History

From f28b880f3be68377003c60005011a835eb18e105 Mon Sep 17 00:00:00 2001
From: yinbin6 <yinbin8@huawei.com>
Date: Sun, 18 Feb 2024 17:11:05 +0800
Subject: [PATCH] FAULT INJECT: gazelle add packet delay and packet drop
---
src/common/gazelle_dfx_msg.h | 15 ++
src/common/gazelle_fault_inject_common.h | 75 +++++++
src/lstack/Makefile | 5 +
src/lstack/core/lstack_control_plane.c | 13 +-
src/lstack/include/lstack_fault_inject.h | 20 ++
src/lstack/include/lstack_vdev.h | 1 +
src/lstack/netif/dir.mk | 3 +
src/lstack/netif/lstack_fault_inject.c | 187 ++++++++++++++++
src/lstack/netif/lstack_vdev.c | 2 +-
src/ltran/CMakeLists.txt | 5 +
src/ltran/ltran_dfx.c | 274 ++++++++++++++++++++++-
11 files changed, 595 insertions(+), 5 deletions(-)
create mode 100644 src/common/gazelle_fault_inject_common.h
create mode 100644 src/lstack/include/lstack_fault_inject.h
create mode 100644 src/lstack/netif/lstack_fault_inject.c
diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 7f8422a..8d528d6 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -18,6 +18,10 @@
#include <lwip/reg_sock.h>
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#include "gazelle_fault_inject_common.h"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
#define GAZELLE_CLIENT_NUM_MIN 1
#define GAZELLE_LOG_LEVEL_MAX 10
#define GAZELLECTL_TIMEOUT 5000 // millisecond
@@ -54,6 +58,11 @@ enum GAZELLE_STAT_MODE {
GAZELLE_STAT_LSTACK_SHOW_AGGREGATE,
GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES,
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ GAZELLE_STAT_FAULT_INJECT_SET,
+ GAZELLE_STAT_FAULT_INJECT_UNSET,
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
GAZELLE_STAT_MODE_MAX,
};
@@ -277,6 +286,9 @@ struct gazelle_stack_dfx_data {
struct gazelle_stat_lstack_snmp snmp;
struct nic_eth_xstats nic_xstats;
struct nic_eth_features nic_features;
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ struct gazelle_fault_inject_data inject;
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
} data;
};
@@ -307,6 +319,9 @@ struct gazelle_stat_msg_request {
union stat_param {
char log_level[GAZELLE_LOG_LEVEL_MAX];
uint16_t low_power_mod;
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ struct gazelle_fault_inject_data inject;
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
} data;
};
diff --git a/src/common/gazelle_fault_inject_common.h b/src/common/gazelle_fault_inject_common.h
new file mode 100644
index 0000000..3a77f39
--- /dev/null
+++ b/src/common/gazelle_fault_inject_common.h
@@ -0,0 +1,75 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef __GAZELLE_INJECT_COMMON_H__
+#define __GAZELLE_INJECT_COMMON_H__
+
+#include <stdint.h>
+
+enum GAZELLE_FAULT_INJECT_TYPE {
+ GAZELLE_FAULT_INJECT_TYPE_ERR = 0,
+ GAZELLE_FAULT_INJECT_PACKET_DELAY,
+ GAZELLE_FAULT_INJECT_PACKET_LOSS,
+ GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
+ GAZELLE_FAULT_INJECT_PACKET_REORDER,
+ GAZELLE_FAULT_INJECT_TYPE_MAX,
+};
+
+enum GAZELLE_FAULT_INJECT_RULE {
+ INJECT_RULE_ERR = 0,
+ /* packet delay rule */
+ INJECT_DELAY_RANDOM,
+ /* packet loss rule */
+ INJECT_LOSS_RANDOM,
+ /* packet duplicate */
+ INJECT_DUPLICATE_RANDOM,
+ /* packet reorder */
+ INJECT_REORDER_RANDOM,
+};
+
+/* fault inject delay: packet delay's time and range, time unit is "ms" */
+struct delay_data {
+ uint32_t delay_time;
+ uint32_t delay_range;
+};
+
+/* fault inject loss: packet loss rate */
+struct loss_data {
+ double loss_rate;
+ uint32_t loss_sigle_count;
+};
+
+/* fault inject duplicate: packet duplicate rate and duplicate count */
+struct duplicate_data {
+ double duplicate_rate;
+ uint32_t duplicate_sigle_count;
+};
+
+/* fault inject reorder: packet reorder rate and reorder count */
+struct reorder_data {
+ double reorder_rate;
+ uint32_t reorder_sigle_count;
+};
+
+struct gazelle_fault_inject_data {
+ int32_t fault_inject_on;
+ enum GAZELLE_FAULT_INJECT_TYPE inject_type;
+ enum GAZELLE_FAULT_INJECT_RULE inject_rule;
+ union {
+ struct delay_data delay;
+ struct loss_data loss;
+ struct duplicate_data duplicate;
+ struct reorder_data reorder;
+ } inject_data;
+};
+
+#endif /* __GAZELLE_INJECT_COMMON_H__ */
diff --git a/src/lstack/Makefile b/src/lstack/Makefile
index df1ddaa..d2c039d 100644
--- a/src/lstack/Makefile
+++ b/src/lstack/Makefile
@@ -47,6 +47,11 @@ ifeq ($(GAZELLE_COVERAGE_ENABLE), 1)
CFLAGS += -fprofile-arcs -ftest-coverage
endif
+ifeq ($(GAZELLE_FAULT_INJECT_ENABLE), 1)
+ LDFLAGS += -DGAZELLE_FAULT_INJECT_ENABLE
+ CFLAGS += -DGAZELLE_FAULT_INJECT_ENABLE
+endif
+
ifeq ($(shell $(CC) -dumpmachine | cut -d"-" -f1), x86_64)
CFLAGS += -mssse3
endif
diff --git a/src/lstack/core/lstack_control_plane.c b/src/lstack/core/lstack_control_plane.c
index 025291d..783f21c 100644
--- a/src/lstack/core/lstack_control_plane.c
+++ b/src/lstack/core/lstack_control_plane.c
@@ -34,6 +34,10 @@
#include "lstack_protocol_stack.h"
#include "lstack_control_plane.h"
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#include "lstack_fault_inject.h"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
/* intervals between two connection attempts and two registration attempts, in second */
#define CONNECT_TO_LTRAN_INFINITE (-1)
#define CONNECT_TO_LTRAN_RETRY_INTERVAL 1
@@ -582,7 +586,14 @@ static int32_t handle_stat_request(int32_t sockfd)
LSTACK_LOG(ERR, LSTACK, "recv wrong stat mode %d\n", (int32_t)msg.stat_mode);
return 0;
}
-
+
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ if (msg.stat_mode == GAZELLE_STAT_FAULT_INJECT_SET ||
+ msg.stat_mode == GAZELLE_STAT_FAULT_INJECT_UNSET) {
+ return handle_fault_inject_cmd(sockfd, &msg);
+ }
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
if (msg.stat_mode == GAZELLE_STAT_LSTACK_LOG_LEVEL_SET ||
msg.stat_mode == GAZELLE_STAT_LSTACK_LOW_POWER_MDF) {
return handle_proc_cmd(sockfd, &msg);
diff --git a/src/lstack/include/lstack_fault_inject.h b/src/lstack/include/lstack_fault_inject.h
new file mode 100644
index 0000000..d54f1f2
--- /dev/null
+++ b/src/lstack/include/lstack_fault_inject.h
@@ -0,0 +1,20 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef __GAZELLE_INJECT_H__
+#define __GAZELLE_INJECT_H__
+
+#include <stdbool.h>
+
+int32_t handle_fault_inject_cmd(int32_t sockfd, struct gazelle_stat_msg_request *msg);
+
+#endif /* __GAZELLE_INJECT_H__ */
diff --git a/src/lstack/include/lstack_vdev.h b/src/lstack/include/lstack_vdev.h
index 4e5d191..1a8342a 100644
--- a/src/lstack/include/lstack_vdev.h
+++ b/src/lstack/include/lstack_vdev.h
@@ -18,5 +18,6 @@ struct gazelle_quintuple;
enum reg_ring_type;
void vdev_dev_ops_init(struct lstack_dev_ops *dev_ops);
int vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple);
+uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts);
#endif /* _GAZELLE_VDEV_H_ */
diff --git a/src/lstack/netif/dir.mk b/src/lstack/netif/dir.mk
index 20fb5d6..1e67734 100644
--- a/src/lstack/netif/dir.mk
+++ b/src/lstack/netif/dir.mk
@@ -9,4 +9,7 @@
# See the Mulan PSL v2 for more details.
SRC = lstack_ethdev.c lstack_vdev.c lstack_flow.c
+ifeq ($(GAZELLE_FAULT_INJECT_ENABLE), 1)
+ SRC += lstack_fault_inject.c
+endif
$(eval $(call register_dir, netif, $(SRC)))
diff --git a/src/lstack/netif/lstack_fault_inject.c b/src/lstack/netif/lstack_fault_inject.c
new file mode 100644
index 0000000..4daad10
--- /dev/null
+++ b/src/lstack/netif/lstack_fault_inject.c
@@ -0,0 +1,187 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#include <securec.h>
+#include <rte_gro.h>
+#include <rte_net.h>
+
+#include <lwip/posix_api.h>
+
+#include "lstack_cfg.h"
+#include "lstack_log.h"
+#include "lstack_vdev.h"
+
+#define INJECT_MODULO 1000 /* used in modulus operator */
+#define INJECT_US_TO_MS 1000 /* transefer time unit us to ms */
+
+typedef int32_t (*inject_xmit_func)(struct protocol_stack *stack, struct rte_mbuf **pkts,
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+struct inject_tbl {
+ struct gazelle_fault_inject_data inject_data;
+ inject_xmit_func inject_func;
+};
+static struct inject_tbl g_inject_tbl[GAZELLE_FAULT_INJECT_TYPE_MAX];
+
+struct inject_func_tbl {
+ enum GAZELLE_FAULT_INJECT_TYPE type;
+ enum GAZELLE_FAULT_INJECT_RULE rule;
+ inject_xmit_func inject_func;
+};
+
+static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+
+static struct inject_func_tbl g_inject_func_tbl[] = {
+ {GAZELLE_FAULT_INJECT_PACKET_LOSS, INJECT_LOSS_RANDOM, inject_packet_loss_random},
+ {GAZELLE_FAULT_INJECT_PACKET_DELAY, INJECT_DELAY_RANDOM, inject_packet_delay_random},
+};
+
+static int32_t inject_func_tbl_update()
+{
+ int32_t func_count = sizeof(g_inject_func_tbl) / sizeof(g_inject_func_tbl[0]);
+
+ for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+ if (!g_inject_tbl[i].inject_data.fault_inject_on) {
+ continue;
+ }
+ for (int32_t j = 0; j < func_count; ++j) {
+ if (g_inject_func_tbl[j].type == g_inject_tbl[i].inject_data.inject_type &&
+ g_inject_func_tbl[j].rule == g_inject_tbl[i].inject_data.inject_rule) {
+ g_inject_tbl[i].inject_func = g_inject_func_tbl[j].inject_func;
+ }
+ }
+ }
+ return 0;
+}
+
+static uint32_t inject_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
+{
+ for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+ if (g_inject_tbl[i].inject_data.fault_inject_on) {
+ int32_t xmit_pkts = 0;
+ xmit_pkts = g_inject_tbl[i].inject_func(stack, pkts, nr_pkts,
+ g_inject_tbl[i].inject_data);
+ if (xmit_pkts == nr_pkts) {
+ continue;
+ }
+ return xmit_pkts;
+ }
+ }
+ if (rte_mbuf_refcnt_read(*pkts) == 1) {
+ return nr_pkts;
+ }
+ return vdev_tx_xmit(stack, pkts, nr_pkts);
+}
+
+static int32_t inject_strategy_update()
+{
+ inject_func_tbl_update();
+
+ int32_t inject_on = 0;
+ for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+ if (g_inject_tbl[i].inject_data.fault_inject_on) {
+ inject_on = 1;
+ break;
+ }
+ }
+
+ struct protocol_stack_group *stack_group = get_protocol_stack_group();
+
+ if (inject_on) {
+ for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+ struct protocol_stack *stack = stack_group->stacks[i];
+ stack->dev_ops.tx_xmit = inject_tx_xmit;
+ }
+ return 0;
+ }
+
+ for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+ struct protocol_stack *stack = stack_group->stacks[i];
+ vdev_dev_ops_init(&stack->dev_ops);
+ }
+
+ return 0;
+}
+
+static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+ /* while *pkts->refcnt == 1, other inject type is on, and the packets have been loss. */
+ if (rte_mbuf_refcnt_read(*pkts) == 1) {
+ return nr_pkts;
+ }
+ int32_t delay_time = data.inject_data.delay.delay_time;
+ int32_t delay_range = data.inject_data.delay.delay_range;
+ int32_t rand_num = rte_rand();
+ rand_num %= INJECT_MODULO;
+ delay_time += delay_range * rand_num / INJECT_MODULO;
+ rte_delay_us(delay_time * INJECT_US_TO_MS);
+
+ return nr_pkts;
+}
+
+static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+ uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+ double loss_rate = data.inject_data.loss.loss_rate;
+ int32_t boundary = (int32_t) (loss_rate * INJECT_MODULO);
+
+ uint32_t rand_num = rte_rand();
+ rand_num %= INJECT_MODULO;
+
+ if (rand_num > boundary) {
+ return nr_pkts;
+ }
+
+ for (int32_t i = 0; i < nr_pkts; ++i) {
+ rte_pktmbuf_free(pkts[i]);
+ }
+ return nr_pkts;
+}
+
+int32_t handle_fault_inject_cmd(int32_t sockfd, struct gazelle_stat_msg_request *msg)
+{
+ struct gazelle_stack_dfx_data rsp = {0};
+ int32_t ret = 0;
+
+ ret = memcpy_s(&g_inject_tbl[msg->data.inject.inject_type].inject_data,
+ sizeof(struct gazelle_fault_inject_data),
+ &msg->data.inject, sizeof(struct gazelle_fault_inject_data));
+ if (ret != EOK) {
+ LSTACK_LOG(ERR, LSTACK, "fault inject memcpy_s error, ret = %d", ret);
+ return -1;
+ }
+
+ inject_strategy_update();
+
+ for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+ ret = memcpy_s(&rsp.data.inject, sizeof(struct gazelle_fault_inject_data),
+ &g_inject_tbl[i].inject_data, sizeof(struct gazelle_fault_inject_data));
+ if (ret != EOK) {
+ LSTACK_LOG(ERR, LSTACK, "fault inject memcpy_s error, ret = %d", ret);
+ return -1;
+ }
+ if (i == GAZELLE_FAULT_INJECT_TYPE_MAX -1) {
+ rsp.eof = 1;
+ }
+ ret = (int32_t) posix_api->write_fn(sockfd, (void *)&rsp, sizeof(rsp));
+ if (ret <= 0) {
+ LSTACK_LOG(ERR, LSTACK, "write msg from peer failed, errno %d. ret=%d\n", errno, ret);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index 3703092..107ee8c 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -150,7 +150,7 @@ static uint32_t ltran_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pk
return sent_pkts;
}
-static uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
+uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
{
uint32_t sent_pkts = 0;
diff --git a/src/ltran/CMakeLists.txt b/src/ltran/CMakeLists.txt
index f739ceb..a484ae3 100644
--- a/src/ltran/CMakeLists.txt
+++ b/src/ltran/CMakeLists.txt
@@ -27,6 +27,11 @@ if($ENV{GAZELLE_COVERAGE_ENABLE})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftest-coverage -fprofile-arcs")
endif($ENV{GAZELLE_COVERAGE_ENABLE})
+if($ENV{GAZELLE_FAULT_INJECT_ENABLE})
+ message("Enable Fault inject option")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DGAZELLE_FAULT_INJECT_ENABLE")
+endif($ENV{GAZELLE_FAULT_INJECT_ENABLE})
+
execute_process(
COMMAND rpm -qa dpdk
OUTPUT_VARIABLE DPDK_VERSION_FULL
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 344afb2..413bf89 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -60,6 +60,46 @@ static struct gazelle_stat_ltran_total g_last_ltran_total;
static struct gazelle_stat_lstack_total g_last_lstack_total[GAZELLE_MAX_STACK_ARRAY_SIZE];
static struct gazelle_stack_dfx_data g_last_lstack_data[GAZELLE_MAX_STACK_ARRAY_SIZE];
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#define INJECT_NAME_SIZE 32
+#define INJECT_RULE_SIZE 32
+
+typedef int32_t (*inject_parse_digit_fun)(char*, char*, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_delay_digit(char *time, char *range, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_loss_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_duplicate_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_reorder_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+
+struct gazelle_fault_inject_type_list {
+ char inject_type_item[INJECT_NAME_SIZE];
+ enum GAZELLE_FAULT_INJECT_TYPE inject_type_parsed;
+ inject_parse_digit_fun parse_digit_func;
+};
+
+static struct gazelle_fault_inject_type_list inject_type_list[] = {
+ {"delay", GAZELLE_FAULT_INJECT_PACKET_DELAY, parse_inject_packet_delay_digit},
+ {"loss", GAZELLE_FAULT_INJECT_PACKET_LOSS, parse_inject_packet_loss_digit},
+ {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
+ {"reorder", GAZELLE_FAULT_INJECT_PACKET_REORDER, parse_inject_packet_reorder_digit},
+};
+
+struct gazelle_fault_inject_rule_list {
+ char inject_rule_item[INJECT_RULE_SIZE];
+ enum GAZELLE_FAULT_INJECT_RULE inject_rule_parsed;
+ enum GAZELLE_FAULT_INJECT_TYPE rule_parse_assit;
+};
+
+static struct gazelle_fault_inject_rule_list g_gazelle_fault_inject_rule_list[] = {
+ {"random", INJECT_DELAY_RANDOM, GAZELLE_FAULT_INJECT_PACKET_DELAY},
+ {"random", INJECT_LOSS_RANDOM, GAZELLE_FAULT_INJECT_PACKET_LOSS},
+ {"random", INJECT_DUPLICATE_RANDOM, GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE},
+ {"random", INJECT_REORDER_RANDOM, GAZELLE_FAULT_INJECT_PACKET_REORDER},
+};
+
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
static bool g_use_ltran = false;
static char* g_unix_prefix;
@@ -95,6 +135,11 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg
static void gazelle_print_lstack_aggregate(void *buf, const struct gazelle_stat_msg_request *req_msg);
static void gazelle_print_lstack_nic_features(void *buf, const struct gazelle_stat_msg_request *req_msg);
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+static void gazelle_print_fault_inject_unset_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = {
{GAZELLE_STAT_LTRAN_SHOW, sizeof(struct gazelle_stat_ltran_total), gazelle_print_ltran_stat_total},
{GAZELLE_STAT_LTRAN_SHOW_RATE, sizeof(struct gazelle_stat_ltran_total), gazelle_print_ltran_stat_rate},
@@ -121,6 +166,11 @@ static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = {
{GAZELLE_STAT_LSTACK_SHOW_XSTATS, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_xstats},
{GAZELLE_STAT_LSTACK_SHOW_AGGREGATE, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_aggregate},
{GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_nic_features},
+
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ {GAZELLE_STAT_FAULT_INJECT_SET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_set_status},
+ {GAZELLE_STAT_FAULT_INJECT_UNSET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_unset_status},
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
};
static int32_t g_wait_reply = 1;
@@ -1121,7 +1171,10 @@ static void show_usage(void)
{
printf("Usage: gazellectl [-h | help] \n"
" or: gazellectl ltran {quit | show | set} [LTRAN_OPTIONS] [-u UNIX_PREFIX]\n"
- " or: gazellectl lstack {show | set} ip [LSTACK_OPTIONS] [-u UNIX_PREFIX]\n \n"
+ " or: gazellectl lstack {show | set} ip [LSTACK_OPTIONS] [-u UNIX_PREFIX]\n\n"
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ " or: gazellectl inject [inject_type] [digit_param_1] [digit_param_2] [inject_rule]\n\n"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
" quit ltran process exit \n \n"
" where LTRAN_OPTIONS := \n"
" show: \n"
@@ -1146,7 +1199,13 @@ static void show_usage(void)
" set: \n"
" loglevel {error | info | debug} set lstack loglevel \n"
" lowpower {0 | 1} set lowpower enable \n"
- " [time] measure latency time default 1S, maximum 30mins \n");
+ " [time] measure latency time default 1S, maximum 30mins \n\n"
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ " *inject params*\n"
+ " |inject_type | digit_param_1 | digit_param_2 | inject_rule |\n"
+ " | delay | delay_time(unit: ms) | delay_range(unit: ms) | random |\n"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+ );
}
static int32_t parse_dfx_ltran_set_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
@@ -1436,6 +1495,211 @@ static int32_t parse_dfx_lstack_args(int32_t argc, char *argv[], struct gazelle_
return num_cmd;
}
+
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+
+#define GAZELLE_SET_FAULT_INJECT_PARAM_COUNT 6
+#define GAZELLE_UNSET_FAULT_INJECT_PARAM_COUNT 4
+#define INJECT_TYPE_INDEX 2
+#define INJECT_DIGITAL_FIRST_INDEX 3
+#define INJECT_DIGITAL_SECOND_INDEX 4
+#define INJECT_RULE_INDEX 5
+
+
+static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_data *inject)
+{
+ if (!inject->fault_inject_on) {
+ return;
+ }
+
+ if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_DELAY) {
+ printf("\t| inject_type: delay | delay_time: %-7d | delay_range: %-3d | "
+ "inject_rule: random |\n", inject->inject_data.delay.delay_time,
+ inject->inject_data.delay.delay_range);
+ }
+
+#define INJECT_PERCENT 100
+
+ if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_LOSS) {
+ printf("\t| inject_type: loss | loss_rate: %-3.1f%% | loss_single_count: %-3d | "
+ "inject_rule: random |\n", inject->inject_data.loss.loss_rate * INJECT_PERCENT,
+ inject->inject_data.loss.loss_sigle_count);
+ }
+ printf("\n");
+}
+
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t ret;
+ struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf;
+ struct gazelle_fault_inject_data *inject = &stat->data.inject;
+
+ printf("\n\n\t\t\t\t\t **** FAULT INJECT INFO **** \n\n");
+ do {
+ gazelle_print_fault_inject_type_info(inject);
+ if (stat->eof != 0) {
+ break;
+ }
+ ret = dfx_stat_read_from_ltran(buf, sizeof(struct gazelle_stack_dfx_data), req_msg->stat_mode);
+ if (ret != GAZELLE_OK) {
+ return;
+ }
+ } while (true);
+}
+
+static void gazelle_print_fault_inject_unset_status(void *buf, const struct gazelle_stat_msg_request *req_msg)
+{
+ return;
+}
+
+static int32_t parse_inject_packet_delay_digit(char* time, char* range, struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t parse_success = 0;
+ int32_t delay_time = atoi(time);
+ if (delay_time <= 0) {
+ printf("FAULT INJECT error: delay time error -- %d, need positive integer.\n", delay_time);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.delay.delay_time = (uint32_t) delay_time;
+
+ int32_t delay_range = atoi(range);
+ if (delay_range < 0) {
+ printf("FAULT INJECT error: delay range error -- %d, need positive integer.\n", delay_range);
+ return parse_success;
+ }
+ if (delay_time - delay_range <= 0) {
+ printf("FAULT INJECT error: delay range should lower than delay time.\n");
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.delay.delay_range = delay_range;
+
+ return ++parse_success;
+}
+
+#define INJECT_RATE_LOWER 0.001
+
+static int32_t parse_inject_packet_loss_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t parse_success = 0;
+ double loss_rate = atof(rate);
+ if (loss_rate < INJECT_RATE_LOWER || loss_rate >= 1) {
+ printf("FAULT INJECT error: loss rate error, range in [0.001, 1), now is %f\n", loss_rate);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.loss.loss_rate = loss_rate;
+
+ int32_t loss_counts = atoi(count);
+ if (loss_counts <= 0) {
+ printf("FAULT INJECT error: single loss counts wrong --%d, need positive integer.", loss_counts);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.loss.loss_sigle_count = loss_counts;
+
+ return ++parse_success;
+}
+
+static int32_t parse_inject_packet_duplicate_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t parse_success = 0;
+ double duplicate_rate = atof(rate);
+ if (duplicate_rate < INJECT_RATE_LOWER || duplicate_rate >= 1) {
+ printf("FAULT INJECT error: duplicate rate error, range in [0.001, 1), now is %f\n", duplicate_rate);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.duplicate.duplicate_rate = duplicate_rate;
+
+ int32_t duplicate_counts = atoi(count);
+ if (duplicate_counts <= 0) {
+ printf("FAULT INJECT error: single duplicate counts wrong --%d, need positive integer.", duplicate_counts);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.duplicate.duplicate_sigle_count = duplicate_counts;
+
+ return ++parse_success;
+}
+
+static int32_t parse_inject_packet_reorder_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t parse_success = 0;
+ double reorder_rate = atof(rate);
+ if (reorder_rate < INJECT_RATE_LOWER || reorder_rate >= 1) {
+ printf("FAULT INJECT error: reorder rate error, range in [0.001, 1), now is %f\n", reorder_rate);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.reorder.reorder_rate = reorder_rate;
+
+ int32_t reorder_counts = atoi(count);
+ if (reorder_counts <= 0) {
+ printf("FAULT INJECT error: single duplicate counts wrong --%d, need positive integer.", reorder_counts);
+ return parse_success;
+ }
+ req_msg->data.inject.inject_data.reorder.reorder_sigle_count = reorder_counts;
+
+ return ++parse_success;
+}
+
+static int32_t parse_fault_inject_digital_data(char *argv[], struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t parse_success = 0;
+ int32_t func_count = sizeof(inject_type_list) / sizeof(inject_type_list[0]);
+ for (int32_t i = 0; i < func_count; ++i) {
+ if (inject_type_list[i].inject_type_parsed == req_msg->data.inject.inject_type) {
+ parse_success = inject_type_list[i].parse_digit_func(argv[INJECT_DIGITAL_FIRST_INDEX],
+ argv[INJECT_DIGITAL_SECOND_INDEX], req_msg);
+ break;
+ }
+ }
+
+ return parse_success;
+}
+
+static int32_t parse_dfx_fault_inject_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t num_cmd = 0; /* while parse error, num_cmd will return as 0, or num_cmd should be returned as 1. */
+
+ req_msg->data.inject.fault_inject_on = 1; /* set fault inject on */
+
+ if (argc == GAZELLE_SET_FAULT_INJECT_PARAM_COUNT) {
+ req_msg->stat_mode = GAZELLE_STAT_FAULT_INJECT_SET;
+ } else if (argc == GAZELLE_UNSET_FAULT_INJECT_PARAM_COUNT) {
+ req_msg->stat_mode = GAZELLE_STAT_FAULT_INJECT_UNSET;
+ } else {
+ printf("FAULT_INJECT error: Count of params wrong , correct count is 6 or 4, now is %d\n", argc);
+ return num_cmd;
+ }
+
+ int32_t inject_type_count = sizeof(inject_type_list) / sizeof(inject_type_list[0]);
+ for (int32_t i = 0; i < inject_type_count; ++i) {
+ if (strcmp(inject_type_list[i].inject_type_item, argv[INJECT_TYPE_INDEX]) == 0) {
+ req_msg->data.inject.inject_type = inject_type_list[i].inject_type_parsed;
+ break;
+ }
+ }
+ if (req_msg->data.inject.inject_type == GAZELLE_FAULT_INJECT_TYPE_ERR) {
+ printf("FAULT_INJECT error: input inject type is wrong -- %s\n", argv[INJECT_TYPE_INDEX]);
+ return num_cmd;
+ }
+
+ int32_t inject_rule_count = sizeof(g_gazelle_fault_inject_rule_list) / sizeof(g_gazelle_fault_inject_rule_list[0]);
+ for (int32_t i = 0; i < inject_rule_count; ++i) {
+ if (strcmp(g_gazelle_fault_inject_rule_list[i].inject_rule_item, argv[INJECT_RULE_INDEX]) == 0 &&
+ g_gazelle_fault_inject_rule_list[i].rule_parse_assit == req_msg->data.inject.inject_type) {
+ req_msg->data.inject.inject_rule = g_gazelle_fault_inject_rule_list[i].inject_rule_parsed;
+ break;
+ }
+ }
+ if (req_msg->data.inject.inject_rule == INJECT_RULE_ERR) {
+ printf("FAULT_INJECT error: input inject rule is wrong -- %s\n", argv[INJECT_RULE_INDEX]);
+ return num_cmd;
+ }
+
+ num_cmd = parse_fault_inject_digital_data(argv, req_msg);
+
+ return num_cmd;
+}
+
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
static int32_t parse_dfx_cmd_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
{
int32_t num_cmd = 0;
@@ -1451,7 +1715,11 @@ static int32_t parse_dfx_cmd_args(int32_t argc, char *argv[], struct gazelle_sta
if (strcmp(param, "lstack") == 0) {
num_cmd = parse_dfx_lstack_args(argc, argv, req_msg);
}
-
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+ if (strcmp(param, "inject") == 0) {
+ num_cmd = parse_dfx_fault_inject_args(argc, argv, req_msg);
+ }
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
return num_cmd;
}
--
2.27.0
Reference in New Issue View Git Blame Copy Permalink