From c95f6cd656e9ccfd2e2b7169626cbc2f50b5e24b Mon Sep 17 00:00:00 2001 From: ctyunsystem Date: Tue, 18 Jan 2022 20:41:46 -0500 Subject: [PATCH] fix cblock parse for LCOLD/LHOT/.cold.NUM, .init_array and support gnu_unique_object --- src/arch/aarch64/arch_parse.c | 24 +++- src/arch/x86/arch_parse.c | 49 +++++++- src/include/kpatch_parse.h | 2 + src/kpatch_parse.c | 106 +++++++++++++++++- tests/Makefile | 10 +- tests/gcc_ge8_gensrc/Makefile | 26 +++++ .../cold_func_suffix/cold_func_suffix.cpp | 48 ++++++++ .../gcc_ge8_gensrc/cold_func_suffix/sub_desc | 13 +++ .../gnu_unique_object/gnu_unique_object.cpp | 35 ++++++ .../gcc_ge8_gensrc/gnu_unique_object/sub_desc | 23 ++++ .../gcc_ge8_gensrc/init_array/init_array.cpp | 28 +++++ tests/gcc_ge8_gensrc/init_array/sub_desc | 13 +++ .../gcc_ge8_gensrc/run_gcc_ge8_gensrc_test.sh | 51 +++++++++ 13 files changed, 419 insertions(+), 9 deletions(-) create mode 100644 tests/gcc_ge8_gensrc/Makefile create mode 100644 tests/gcc_ge8_gensrc/cold_func_suffix/cold_func_suffix.cpp create mode 100644 tests/gcc_ge8_gensrc/cold_func_suffix/sub_desc create mode 100644 tests/gcc_ge8_gensrc/gnu_unique_object/gnu_unique_object.cpp create mode 100644 tests/gcc_ge8_gensrc/gnu_unique_object/sub_desc create mode 100644 tests/gcc_ge8_gensrc/init_array/init_array.cpp create mode 100644 tests/gcc_ge8_gensrc/init_array/sub_desc create mode 100755 tests/gcc_ge8_gensrc/run_gcc_ge8_gensrc_test.sh diff --git a/src/arch/aarch64/arch_parse.c b/src/arch/aarch64/arch_parse.c index f91ef0c..66ccc7e 100644 --- a/src/arch/aarch64/arch_parse.c +++ b/src/arch/aarch64/arch_parse.c @@ -1,4 +1,10 @@ /****************************************************************************** + * 2022.01.18 - add recog_func_attr() for count "%function" + * China Telecom, + * + * 2021.12.13 - support the type of C++ "gnu_unique_object" variable in is_variable_start() + * China Telecom, + * * 2021.10.08 - enhance kpatch_gensrc and kpatch_elf and kpatch_cc code * Huawei Technologies Co., Ltd. * @@ -43,11 +49,26 @@ int is_function_start(struct kp_file *f, int l, kpstr_t *nm) func = func ? 1 : ctype(f, l) == DIRECTIVE_TYPE; continue; } + break; } return func; } +void recog_func_attr(struct kp_file *f, int i, kpstr_t *nm, int *cnt) +{ + kpstr_t func_nm, func_attr; + + if(ctype(f, i) == DIRECTIVE_TYPE) { + kpstrset(&func_nm, "", 0); + kpstrset(&func_attr, "", 0); + + get_type_args(cline(f, i), &func_nm, &func_attr); + if(!kpstrcmpz(&func_attr, "%function") && !kpstrcmp(&func_nm, nm)) /* verify name matches */ + ++(*cnt); + } +} + int is_data_def(char *s, int type) { kpstr_t t; @@ -127,6 +148,7 @@ int is_variable_start(struct kp_file *f, int l, int *e, int *pglobl, kpstr_t *nm s = cline(f, l); if (*s == '\0' && l != l0) continue; + switch (ctype(f, l)) { case DIRECTIVE_TYPE: case DIRECTIVE_GLOBL: @@ -159,7 +181,7 @@ int is_variable_start(struct kp_file *f, int l, int *e, int *pglobl, kpstr_t *nm return 1; case DIRECTIVE_TYPE: get_type_args(cline(f, l), &nm2, &attr); - if (kpstrcmpz(&attr, "%object") && kpstrcmpz(&attr, "%tls_object")) + if (kpstrcmpz(&attr, "%object") && kpstrcmpz(&attr, "%tls_object") && kpstrcmpz(&attr, "%gnu_unique_object")) return 0; break; case DIRECTIVE_GLOBL: diff --git a/src/arch/x86/arch_parse.c b/src/arch/x86/arch_parse.c index 15cf9fe..31caa46 100644 --- a/src/arch/x86/arch_parse.c +++ b/src/arch/x86/arch_parse.c @@ -1,4 +1,13 @@ /****************************************************************************** + * 2022.01.18 - add recog_func_attr() for count "@function" + * China Telecom, + * + * 2021.12.13 - support the type of C++ "gnu_unique_object" variable in is_variable_start() + * China Telecom, + * + * 2021.12.13 - support the appearance of ".LCOLD*" or ".LHOT*" label in is_function_start() and is_variable_start() + * China Telecom, + * * 2021.10.08 - enhance kpatch_gensrc and kpatch_elf and kpatch_cc code * Huawei Technologies Co., Ltd. ******************************************************************************/ @@ -37,11 +46,40 @@ int is_function_start(struct kp_file *f, int l, kpstr_t *nm) func = func ? 1 : ctype(f, l) == DIRECTIVE_TYPE; continue; } + + /* particularly: for "-freorder-functions" optimization under -O2/-O3/-Os, + ".LCOLD*" or ".LHOT*" label may appear at the head of function or variable cblock, + it should not be divided into an independent cblock belonging to ATTR or OTHER */ + if(ctype(f, l) == DIRECTIVE_LABEL) { + s = cline(f, l); + if(strstr(s, ".LCOLD") || strstr(s, ".LHOT")) + continue; + } + break; } return func; } +void recog_func_attr(struct kp_file *f, int i, kpstr_t *nm, int *cnt) +{ + kpstr_t func_nm, func_attr; + + if(ctype(f, i) == DIRECTIVE_TYPE) { + kpstrset(&func_nm, "", 0); + kpstrset(&func_attr, "", 0); + + get_type_args(cline(f, i), &func_nm, &func_attr); + if(!kpstrcmpz(&func_attr, "@function")) { + if(func_nm.l > nm->l) + remove_cold_hot_suffix(&func_nm); /* remove .cold. / .hot. */ + + if(!kpstrcmp(&func_nm, nm)) /* verify name matches */ + ++(*cnt); + } + } +} + int is_data_def(char *s, int type) { kpstr_t t; @@ -90,6 +128,15 @@ int is_variable_start(struct kp_file *f, int l, int *e, int *pglobl, kpstr_t *nm s = cline(f, l); if (*s == '\0' && l != l0) continue; + + /* particularly: for "-freorder-functions" optimization under -O2/-O3/-Os, + ".LCOLD*" or ".LHOT*" label may appear at the head of function or variable cblock, + it should not be divided into an independent cblock belonging to ATTR or OTHER */ + if(ctype(f, l) == DIRECTIVE_LABEL) { + if(strstr(s, ".LCOLD") || strstr(s, ".LHOT")) + continue; + } + switch (ctype(f, l)) { case DIRECTIVE_TYPE: case DIRECTIVE_GLOBL: @@ -115,7 +162,7 @@ int is_variable_start(struct kp_file *f, int l, int *e, int *pglobl, kpstr_t *nm break; case DIRECTIVE_TYPE: get_type_args(cline(f, l), &nm2, &attr); - if (kpstrcmpz(&attr, "@object")) + if (kpstrcmpz(&attr, "@object") && kpstrcmpz(&attr, "@gnu_unique_object")) return 0; break; case DIRECTIVE_GLOBL: diff --git a/src/include/kpatch_parse.h b/src/include/kpatch_parse.h index a36a015..c52a1e3 100644 --- a/src/include/kpatch_parse.h +++ b/src/include/kpatch_parse.h @@ -106,9 +106,11 @@ struct cblock { void get_token(char **str, kpstr_t *x); void __get_token(char **str, kpstr_t *x, const char *delim); +void remove_cold_hot_suffix(kpstr_t *nm); int is_function_start(struct kp_file *f, int l, kpstr_t *nm); int is_function_end(struct kp_file *f, int l, kpstr_t *nm); +void recog_func_attr(struct kp_file *f, int i, kpstr_t *nm, int *cnt); void get_type_args(char *s, kpstr_t *nm, kpstr_t *attr); int is_variable_start(struct kp_file *f, int l, int *e, int *globl, kpstr_t *nm); diff --git a/src/kpatch_parse.c b/src/kpatch_parse.c index ddf58f8..0885cbe 100644 --- a/src/kpatch_parse.c +++ b/src/kpatch_parse.c @@ -1,4 +1,10 @@ /****************************************************************************** + * 2021.12.16 - kpatch_parse: enhance init_other_block() to extend function cblock to cover .init_array + * China Telecom, + * + * 2021.12.13 - kpatch_parse: adjust the judgment for the end of function cblock in init_func_block() + * China Telecom, + * * 2021.10.11 - kpatch: fix code checker warning * Huawei Technologies Co., Ltd. * @@ -72,6 +78,19 @@ void get_token(char **str, kpstr_t *x) __get_token(str, x, delim); } +/* remove .cold. / .hot. in function name */ +void remove_cold_hot_suffix(kpstr_t *nm) +{ + if(!nm->s) + return; + + char *suffix_loc = strstr(nm->s, ".cold."); + if(!suffix_loc) + suffix_loc = strstr(nm->s, ".hot."); + if(suffix_loc) + nm->l = suffix_loc - nm->s; /* remove .cold. / .hot. */ +} + /* ------------------------------ as directives parsing ---------------------------------- */ static struct { @@ -303,13 +322,26 @@ static void init_func_block(struct kp_file *f, int *i, kpstr_t *nm) int flags = 0; struct cblock *blk; - while (e < f->nr_lines - 1 && !is_function_end(f, e, nm)) { + int func_cnt = 0; + + while (e < f->nr_lines - 1) { if (ctype(f, e) == DIRECTIVE_GLOBL) globl = 1; if (ctype(f, e) == DIRECTIVE_KPFLAGS) { flags |= get_kpatch_flags(cline(f, e)); cline(f, e)[0] = 0; } + + /* if compiling is optimized by -freorder-functions, e.g funcA, it will contains like ".type funcA.cold.xxx,@function" inside funcA, + and the end of funcA is not the first size directive matched with funcA. At present, use count for "@function" to judge*/ + recog_func_attr(f, e, nm, &func_cnt); + + if(is_function_end(f, e, nm)) { + --func_cnt; + if(!func_cnt) + break; + } + e++; } @@ -357,16 +389,76 @@ static void init_set_block(struct kp_file *f, int *i, kpstr_t *nm) (*i)++; } +/*if funcA is needed in initialization, e.g constructor in C++, the function pointer will be put into .init_array section. +the directives will appear right after the function size directive like this, + + .size funcA, .-funcA + .section .init_array,"aw" + .align 8 + .quad funcA + +since LCOLD* or LHOT* label may appear inside, and the label may change after patched, if classified as OTHER or VAR cblock, +label change will conflict with the corresponding matching rules. also, we cannot set a proper VAR cblock name with no violation. +it can only be treated as an extension of FUNC cblock. */ + +#define EXT_INIARR_FLAG 1 +#define EXT_UPDATE_FLAG 2 + static void init_other_block(struct kp_file *f, int *i) { int s = *i, e = *i; + int flag = 0; + kpstr_t nm; + kpstrset(&nm, "", 0); - while (e < f->nr_lines && !(is_function_start(f, e, &nm) || is_variable_start(f, e, NULL, NULL, &nm))) - e++; + char *line = NULL; + kpstr_t nm2; + kpstrset(&nm2, "", 0); + + struct rb_node *node = NULL; + struct cblock *blk = NULL; + + while (e < f->nr_lines && !(is_function_start(f, e, &nm) || is_variable_start(f, e, NULL, NULL, &nm))) { + if(ctype(f, e) == DIRECTIVE_SECTION && !strcmp(csect(f, e)->name, ".init_array")) + flag = EXT_INIARR_FLAG; + + if(flag && ctype(f, e) == DIRECTIVE_OTHER) { + line = cline(f, e); + + if (is_data_def(line, DIRECTIVE_OTHER)) { + get_token(&line, &nm2); + get_token(&line, &nm2); + + node = rb_last(&f->cblocks_by_start); + if(!node) { + ++e; + break; + } + + blk = rb_entry(node, struct cblock, rbs); + if(blk->type == CBLOCK_FUNC && !kpstrcmp(&blk->name, &nm2)) { + kplog(LOG_DEBUG, "Extend cblock %.*s (%d: %d-%d) to (%d: %d-%d)\n", + blk->name.l, blk->name.s, f->id, blk->start, blk->end-1, f->id, blk->start, e); + blk->end = ++e; + flag = EXT_UPDATE_FLAG; + break; + } + } + } + ++e; + } + + if(flag == EXT_INIARR_FLAG) { + while (e < f->nr_lines && !(is_function_start(f, e, &nm) || is_variable_start(f, e, NULL, NULL, &nm))) + ++e; + } + + if(flag != EXT_UPDATE_FLAG) { + kpstrset(&nm, "", 0); + cblock_add(f, s, e, &nm, CBLOCK_OTHER, 0); + } - kpstrset(&nm, "", 0); - cblock_add(f, s, e, &nm, CBLOCK_OTHER, 0); *i = e; } @@ -696,6 +788,10 @@ int is_function_end(struct kp_file *f, int l, kpstr_t *nm) char *s = cline(f, l); get_token(&s, &nm2); /* skip command */ get_token(&s, &nm2); + + if(nm2.l > nm->l) + remove_cold_hot_suffix(&nm2); /* remove .cold. / .hot. */ + if (kpstrcmp(nm, &nm2)) /* verify name matches */ return 0; diff --git a/tests/Makefile b/tests/Makefile index c9edaf3..ce97dbd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -8,13 +8,15 @@ ifeq ($(ARCH), aarch64) SUBDIRS := $(filter-out $(AARCH64_NO_SUPPORT_TESTS), $(SUBDIRS)) endif +GCC_GE8_GENSRC_TESTS := $(patsubst gcc_ge8_gensrc/%/sub_desc,%,$(wildcard gcc_ge8_gensrc/*/sub_desc)) + KPATCH_PATH:=$(CURDIR)/../src export KPATCH_PATH all: run list: - @echo TESTS: $(SUBDIRS) + @echo TESTS: $(SUBDIRS) $(GCC_GE8_GENSRC_TESTS) fastsleep.so: CFLAGS += -fPIC fastsleep.so: fastsleep.c @@ -26,6 +28,7 @@ clean: $(addprefix clean-,$(SUBDIRS)) $(CURDIR)/lpmakelevel-patchroot rm -f fastsleep.so make -C execve clean + make -C gcc_ge8_gensrc clean clean-%: FORCE make -C $* clean @@ -98,6 +101,9 @@ run-lpmakelevel: RUNTESTSFLAGS := -d lpmake -p $(CURDIR)/lpmakelevel-patchroot run-lpmakelevel: fastsleep.so run-lpmakelevel: run-startup-lpmakelevel -run: run-build run-patchlevel # run-lpmake run-lpmakelevel +run-gcc_ge8_gensrc: + make -C gcc_ge8_gensrc + +run: run-build run-patchlevel run-gcc_ge8_gensrc # run-lpmake run-lpmakelevel FORCE: diff --git a/tests/gcc_ge8_gensrc/Makefile b/tests/gcc_ge8_gensrc/Makefile new file mode 100644 index 0000000..9b0a3d9 --- /dev/null +++ b/tests/gcc_ge8_gensrc/Makefile @@ -0,0 +1,26 @@ +.PHONY: all clean + +SOURCE = $(wildcard */*.cpp) +ASM_SOURCE = $(patsubst %.cpp, %.orig.s, $(SOURCE)) + +GCC_REQUIRED=8 +GCC_MAJOR = $(shell echo __GNUC__ | $(CXX) -E -x c - | tail -n 1) +GCC_MAJOR_GTE8 = $(shell expr $(GCC_MAJOR) \>= $(GCC_REQUIRED)) + +COMPILE_COMMAND = +TEST_COMMAND = + +ifeq ($(GCC_MAJOR_GTE8), 1) + COMPILE_COMMAND = $(CXX) $^ -S -O2 -std=c++17 -o $@ +endif + +all: test + +test: $(ASM_SOURCE) + ./run_gcc_ge8_gensrc_test.sh $(GCC_MAJOR) $(GCC_REQUIRED) + +clean: + rm -f $(shell find ./ -name *.s) + +%.orig.s: %.cpp + $(COMPILE_COMMAND) \ No newline at end of file diff --git a/tests/gcc_ge8_gensrc/cold_func_suffix/cold_func_suffix.cpp b/tests/gcc_ge8_gensrc/cold_func_suffix/cold_func_suffix.cpp new file mode 100644 index 0000000..da1ed43 --- /dev/null +++ b/tests/gcc_ge8_gensrc/cold_func_suffix/cold_func_suffix.cpp @@ -0,0 +1,48 @@ +#include + +extern int ext_func(int a, int b) __attribute__((cold)); + +void swap(int &a, int &b) +{ + int temp = a; + a = b; + b = temp; +} + +int cold_func(int a, int b) +{ + int c = 0; + if(__builtin_expect(a > 0, false)) + c = a*2 + b; + else + c = ext_func(a, b) + 7; + + return c; +} + +void reverse(int &a) +{ + int org = a; + int res = 0; + while(org > 0) + { + res *= 10; + res += org % 10; + org /= 10; + } + a = res; +} + +int main() +{ + int i = 9527; + int m = i/9; + int n = i%9; + + int k = cold_func(m, n); + swap(m, n); + reverse(i); + + std::cout << "k=" << k << " i=" << i << std::endl; + return 0; +} diff --git a/tests/gcc_ge8_gensrc/cold_func_suffix/sub_desc b/tests/gcc_ge8_gensrc/cold_func_suffix/sub_desc new file mode 100644 index 0000000..0e25f29 --- /dev/null +++ b/tests/gcc_ge8_gensrc/cold_func_suffix/sub_desc @@ -0,0 +1,13 @@ +test LCOLD/LHOT func.cold.NUM for recent gcc(>=gcc 8) with __attribute__((cold)) and __builtin_expect + +steps: +1) compile the source code: + g++ cold_func_suffix.cpp -S -O2 -o cold_func_suffix.s + +2) generate diff-asm file with no difference to check the result of cblock + kpatch_gensrc --os=rhel6 -i cold_func_suffix.s -i cold_func_suffix.s -o tmp.s + +3) tmp.s should be the same as cold_func_suffix.s, except the "#---var----" and "#----func---" + sed '/^#/d' tmp.s > same.s + diff cold_func_suffix.s same.s | wc -l +the result should be 0 \ No newline at end of file diff --git a/tests/gcc_ge8_gensrc/gnu_unique_object/gnu_unique_object.cpp b/tests/gcc_ge8_gensrc/gnu_unique_object/gnu_unique_object.cpp new file mode 100644 index 0000000..f1b432d --- /dev/null +++ b/tests/gcc_ge8_gensrc/gnu_unique_object/gnu_unique_object.cpp @@ -0,0 +1,35 @@ +#include + +class StudentManage +{ +public: + void setStudent(int id, int age) + { + student.stu_id = id; + student.stu_age = age; + } + void displayStudent() + { + std::cout << "student " << student.stu_id << " age : " << student.stu_age << std::endl; + } + +private: + struct Student + { + int stu_id; + int stu_age; + }; + + inline static thread_local Student student; +}; + + +int main() +{ + StudentManage ms; + ms.setStudent(9581, 40); + ms.displayStudent(); + ms.setStudent(9587, 36); + ms.displayStudent(); + return 0; +} diff --git a/tests/gcc_ge8_gensrc/gnu_unique_object/sub_desc b/tests/gcc_ge8_gensrc/gnu_unique_object/sub_desc new file mode 100644 index 0000000..60194f5 --- /dev/null +++ b/tests/gcc_ge8_gensrc/gnu_unique_object/sub_desc @@ -0,0 +1,23 @@ +test var with @gnu_unique_object assigned in .tbss by using a "inline static thread_local" data member + +note: +1) the source code must compile with -std=c++17 +2) the test situation also can be constructed with a thread_local var in c++ template, eg: +template +T func(T num1, T num2) +{ + thread_local T s0; + ... ... +} + +steps: +1) compile the source code: + g++ gnu_unique_object.cpp -S -O2 -std=c++17 -o gnu_unique_object.s + +2) generate diff-asm file with no difference to check the result of cblock + kpatch_gensrc --os=rhel6 -i gnu_unique_object.s -i gnu_unique_object.s -o tmp.s + +3) tmp.s should be the same as gnu_unique_object.s, except the "#---var----" and "#----func---" + sed '/^#/d' tmp.s > same.s + diff gnu_unique_object.s same.s | wc -l +the result should be 0 \ No newline at end of file diff --git a/tests/gcc_ge8_gensrc/init_array/init_array.cpp b/tests/gcc_ge8_gensrc/init_array/init_array.cpp new file mode 100644 index 0000000..bfbe74c --- /dev/null +++ b/tests/gcc_ge8_gensrc/init_array/init_array.cpp @@ -0,0 +1,28 @@ +#include + +class CTest +{ +public: + CTest():m_i2(1) {} + void print() + { + int sum = m_i1+m_i2; + std::cout << "sum is " << sum << std::endl; + + int sub = m_i1-m_i2; + std::cout << "sub is " << sub << std::endl; + } +private: + static int m_i1; + int m_i2; +}; + +int CTest::m_i1 = 10; + +int main() +{ + CTest ct1; + ct1.print(); + + return 0; +} diff --git a/tests/gcc_ge8_gensrc/init_array/sub_desc b/tests/gcc_ge8_gensrc/init_array/sub_desc new file mode 100644 index 0000000..1ac5e0d --- /dev/null +++ b/tests/gcc_ge8_gensrc/init_array/sub_desc @@ -0,0 +1,13 @@ +test .init_array cblock partition + +steps: +1) compile the source code: + g++ init_array.cpp -S -O2 -o init_array.s + +2) generate diff-asm file with no difference to check the result of cblock + kpatch_gensrc --os=rhel6 -i init_array.s -i init_array.s -o tmp.s + +3) tmp.s should be the same as init_array.s, except the "#---var----" and "#----func---" + sed '/^#/d' tmp.s > same.s + diff init_array.s same.s | wc -l +the result should be 0 \ No newline at end of file diff --git a/tests/gcc_ge8_gensrc/run_gcc_ge8_gensrc_test.sh b/tests/gcc_ge8_gensrc/run_gcc_ge8_gensrc_test.sh new file mode 100755 index 0000000..4534038 --- /dev/null +++ b/tests/gcc_ge8_gensrc/run_gcc_ge8_gensrc_test.sh @@ -0,0 +1,51 @@ +#/bin/sh + +echo "the following case only for gcc $2 and later:" + +CURDIR=$(cd $(dirname $0); pwd) +SOURCE_SET=$(find $CURDIR -name *.orig.s) +TOTAL_CASE=$(find $CURDIR -name sub_desc | wc -l) +KPATCH_GENSRC=$CURDIR/../../src/kpatch_gensrc + +OK_CNT=0 +FAIL_CNT=0 +SKIP_CNT=0 + +if [ $1 -lt $2 ]; then + SKIP_CNT=$TOTAL_CASE + echo "gcc is too old to test, test: gcc $1 < required: gcc $2)" + echo "OK $OK_CNT FAIL $FAIL_CNT SKIP $SKIP_CNT TOTAL $TOTAL_CASE" + exit 0 +fi + +for SOURCE in $SOURCE_SET; do + FILENAME=${SOURCE##*/} + CASENAME=${FILENAME%.orig.s} + if [ $CASENAME == "cold_func_suffix" ]; then + KEY_WORD="\.cold." + else + KEY_WORD=$CASENAME + fi + + KEY_WORD_LINE=$(grep -c $KEY_WORD $SOURCE) + if [ $KEY_WORD_LINE -lt "2" ]; then + echo "SKIP: $CASENAME, $KEY_WORD not found" + SKIP_CNT=$(($SKIP_CNT+1)) + continue + fi + + $KPATCH_GENSRC --os=rhel6 -i $SOURCE -i $SOURCE -o ${SOURCE/.orig/.o} + sed -i '/^#/d' ${SOURCE/.orig/.o} + + DIFF_LINE=$(diff $SOURCE ${SOURCE/.orig/.o} | grep -c $KEY_WORD) + if [ $DIFF_LINE -gt "0" ]; then + echo "TEST $CASENAME IS FAIL" + FAIL_CNT=$(($FAIL_CNT+1)) + else + echo "TEST $CASENAME IS OK" + OK_CNT=$(($OK_CNT+1)) + fi +done + +echo "OK $OK_CNT FAIL $FAIL_CNT SKIP $SKIP_CNT TOTAL $TOTAL_CASE" +exit 0 \ No newline at end of file -- 2.27.0