From 69730a8687a8e56aad46b9b4f309af1c3df6ed06 Mon Sep 17 00:00:00 2001
From: yangjiaqi <yangjiaqi16@huawei.com>
Date: Thu, 25 Jan 2024 15:28:20 +0800
Subject: [PATCH] test: add tests for docker and lxcfs

Signed-off-by: yangjiaqi <yangjiaqi16@huawei.com>
---
 Makefile                       |   5 +-
 remountcmd_test.go             |   7 +-
 tests/lib/commonlib.sh         | 224 +++++++++++++++++++++++++++++++++
 tests/run_test.sh              |  43 +++++++
 tests/src/test_docker_lxcfs.sh |  65 ++++++++++
 umountcmd_test.go              |  61 +++++++--
 6 files changed, 394 insertions(+), 11 deletions(-)
 create mode 100644 tests/lib/commonlib.sh
 create mode 100755 tests/run_test.sh
 create mode 100755 tests/src/test_docker_lxcfs.sh

diff --git a/Makefile b/Makefile
index cceab41..a55688b 100644
--- a/Makefile
+++ b/Makefile
@@ -42,11 +42,14 @@ lxcfs-hook: $(SOURCES) | $(DEPS_LINK)
 	${ENV} go build -buildmode=pie -mod=vendor -tags ${TAGS} -ldflags ${GO_LDFLAGS} -o build/lxcfs-hook ./hooks/lxcfs-hook
 	@echo "Done!"
 
-tests:
+tests: funtests
 	@echo "Checking lxcfs-tools"
 	${ENV} go test -mod=vendor -tags ${TAGS} -p 1 -v ./... -coverprofile=cover.out
 	@echo "Done!"
 
+funtests:
+	bash tests/run_test.sh
+
 clean:
 	rm -rf build
 
diff --git a/remountcmd_test.go b/remountcmd_test.go
index 106ac5e..aeb83ea 100644
--- a/remountcmd_test.go
+++ b/remountcmd_test.go
@@ -66,12 +66,17 @@ func TestRemountAll(t *testing.T) {
 	}
 	defer recoveriSulad()
 
+	if !tryMoveRunc() {
+		return
+	}
+	defer recoverRunc()
+
 	tests := []struct {
 		name, initMountns, initUserns string
 		wantErr                       bool
 	}{
 		{
-			name:        "TC1-absent of isulad",
+			name:        "TC1-absent of container tools",
 			initMountns: "1",
 			initUserns:  "5",
 			wantErr:     true,
diff --git a/tests/lib/commonlib.sh b/tests/lib/commonlib.sh
new file mode 100644
index 0000000..2ff4776
--- /dev/null
+++ b/tests/lib/commonlib.sh
@@ -0,0 +1,224 @@
+#!/bin/bash
+###
+ # Copyright (c) Huawei Technologies Co., Ltd. 2021-2024. All rights reserved. 
+ # rubik licensed under the Mulan PSL v2.
+ # You can use this software according to the terms and conditions of the Mulan PSL v2.
+ # You may obtain a copy of Mulan PSL v2 at: 
+ #     http://license.coscl.org.cn/MulanPSL2 
+ # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ # PURPOSE.
+ # See the Mulan PSL v2 for more details.
+ # Date: 2024-01-24
+ # Description: This file is common functions file
+### 
+
+current_dir="$(dirname "$(realpath "$0")")"
+project_dir="$current_dir/../.."
+build_dir="$project_dir/build"
+exit_flag=0
+tools_bak_name="/usr/local/bin/lxcfs-tools.test.bak"
+
+function echoTxt() {
+    TXT="[$(date "+%Y-%m-%d-%H-%M-%S")]$1"
+    COLOR=$2
+    if [ "${COLOR}" = "red" ]; then
+        echo -e "${color_red}${TXT}${color_reset}"
+    elif [ "${COLOR}" = "green" ]; then
+        echo -e "${color_green}${TXT}${color_reset}"
+    elif [ "${COLOR}" = "yellow" ]; then
+        echo -e "${color_yellow}${TXT}${color_reset}"
+    else
+        echo "${TXT}"
+    fi
+}
+
+function tsm_error() {
+    txt_str=$1
+    echoTxt "$txt_str" red
+}
+
+function tsm_info() {
+    txt_str=$1
+    echoTxt "$txt_str" green
+}
+
+# 检查是否安装了docker和lxcfs rpm包
+function check_packages() {
+    if ! rpm -q docker-engine >/dev/null 2>&1; then
+        echo "docker-engine is not installed, trying to install..."
+        if ! yum install -y docker-engine >/dev/null 2>&1; then
+            echo "Failed to install Docker, skip test cases"
+            exit 0
+        fi
+    fi
+    
+    if ! rpm -q lxcfs >/dev/null 2>&1; then
+        echo "LXCFS is not installed, trying to install..."
+        if ! yum install -y lxcfs >/dev/null 2>&1; then
+            echo "Failed to install LXCFS, skip test cases"
+            exit 0
+        fi
+    fi
+}
+
+# 检查服务是否可用
+function check_service() {
+    if ! systemctl is-active $1 >/dev/null 2>&1; then
+        echo "$1 is not running, trying to start..."
+        if ! systemctl start $1 >/dev/null 2>&1; then
+            echo "Failed to start $1, cannot run test cases"
+            exit 0
+        fi
+    fi
+    
+    if ! systemctl is-active $1 >/dev/null 2>&1; then
+        echo "$1 is not running, cannot run test cases"
+        exit 0
+    fi
+}
+
+# 检查lxcfs-tools是否可用
+function check_lxcfs_tools() {
+    if [ ! -f "$build_dir"/lxcfs-tools ]; then
+        echo "lxcfs-tools binary not found, compiling..."
+        cd "$project_dir"
+        make || { echo "$project_dir Compilation failed"; exit 0; }
+    fi
+
+    if [ ! "$build_dir"/lxcfs-tools ]; then
+        echo "lxcfs-tools binary still not found, something went wrong"
+        exit 0
+    fi
+
+    # 检查 /usr/local/bin/ 目录下是否存在 lxcfs-tools 二进制
+    if [ -f /usr/local/bin/lxcfs-tools ]; then
+        # 存在，则将其重命名为 lxcfs-tools.bak
+        mv /usr/local/bin/lxcfs-tools "${tools_bak_name}"
+    fi
+
+    # 将 build 目录下的 lxcfs-tools 二进制拷贝到 /usr/local/bin/ 目录
+    cp "$build_dir"/lxcfs-tools /usr/local/bin/
+
+}
+
+function ensure_docker_image_exists() {
+  local image_name="$1"
+
+  # 检查镜像是否存在
+  if ! docker inspect "$image_name" &> /dev/null; then
+    echo "镜像 $image_name 不存在，开始拉取镜像..."
+    docker pull "$image_name"
+
+    # 再次检查镜像是否存在
+    if ! docker inspect "$image_name" &> /dev/null; then
+      echo "拉取镜像失败，退出进程"
+      exit 0
+    fi
+  fi
+}
+
+#####################################################################
+
+# $1:result   $2 expect result
+# $1 equal $2
+#############################################################
+
+function fn_check_result() {
+    if [ "$1" = "$2" ]; then
+        tsm_info "PASS:return $1 as expected!($3)"
+    else
+        tsm_error "FAILED:return $1 not as expected $2!($3)"
+        ((exit_flag++))
+    fi
+}
+
+# wait untill the state of container turns to the specified one
+# $1:container ID/container name
+# $2:the expected state
+
+#############################################################
+
+function WaitInspect() {
+    container=$1
+    expect_state=$2
+    result="FAIL"
+    if [ $# -lt 2 ]; then
+        tsm_error "FAILED:  take at lease 2 input parameters"
+        ((exit_flag++))
+        return 1
+    fi
+    for ((i = 0; i < 30; i++)); do
+        current_state=$(docker inspect -f '{{.State.Status}}' "$container")
+        if [ "$current_state" == "$expect_state" ]; then
+            result="PASS"
+            break
+        else
+            sleep 1
+        fi
+    done
+    if [ "$result" == "PASS" ]; then
+        tsm_info "PASS:return $current_state as expected!($3)"
+        return 0
+    else
+        tsm_error "FAILED:return $current_state not as expected!($3)"
+        ((exit_flag++))
+        return 1
+    fi
+}
+
+function wait_service_lxcfs_running() {
+    local reTry=${1:-30}
+    local lxcfs_status
+    local i
+    for i in $(seq "${reTry}"); do
+        lxcfs_status=$(systemctl status lxcfs | awk -F "[()]" '/Active/{print $2}')
+        if [[ "${lxcfs_status}" == "running" ]]; then
+            return 0
+        fi
+        sleep 1
+    done
+    return 1
+}
+
+function restart_lxcfs() {
+    local ret i
+    for i in $(seq 10); do
+        systemctl restart lxcfs
+        wait_service_lxcfs_running
+        ret=$?
+        [[ "$ret" -eq 0 ]] && break
+    done
+    return "$ret"
+}
+
+function stop_services() {
+    systemctl stop docker
+    systemctl stop lxcfs
+
+    # 等待服务停止
+    for i in {1..10}; do
+        if [[ $(systemctl is-active docker) == "inactive" && $(systemctl is-active lxcfs) == "inactive" ]]; then
+            echo "Services stopped successfully"
+            return 0
+        fi
+        sleep 1
+    done
+
+    echo "Timeout waiting for services to stop"
+    return 0
+}
+
+function clean_env() {
+    rm -rf /usr/local/bin/lxcfs-tools
+    if [ -f "${tools_bak_name}" ]; then
+        # 存在，则将其重命名为 lxcfs-tools
+        mv "${tools_bak_name}" /usr/local/bin/lxcfs-tools
+    fi
+    docker rm -f `docker ps -aq` > /dev/null
+    stop_services
+}
+
+
+
+
diff --git a/tests/run_test.sh b/tests/run_test.sh
new file mode 100755
index 0000000..6339a70
--- /dev/null
+++ b/tests/run_test.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+###
+ # Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. 
+ # rubik licensed under the Mulan PSL v2.
+ # You can use this software according to the terms and conditions of the Mulan PSL v2.
+ # You may obtain a copy of Mulan PSL v2 at: 
+ #     http://license.coscl.org.cn/MulanPSL2 
+ # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ # PURPOSE.
+ # See the Mulan PSL v2 for more details.
+ # Date: 2024-01-25
+ # Description: This file is used for running testcases
+###
+
+current_dir="$(cd "$(dirname "$0")" && pwd)"
+src_dir="$current_dir/src"
+log_dir="$current_dir/logs"
+fail_cnt=0
+mkdir -p "$log_dir"
+
+for script in "$src_dir"/*.sh; do
+    script_name=$(basename "$script")
+    log_file="${log_dir}/${script_name%.sh}-$(date +%Y-%m-%d-%H-%M-%S).log"
+    echo "Running $script_name..."
+    bash -x "$script" > "$log_file" 2>&1
+    exit_code=$?
+
+    if [ $exit_code -eq 0 ]; then
+        echo "$(date '+%Y-%m-%d %H:%M:%S') $script_name succeeded" | tee -a "$log_file"
+    else
+        cat ${log_file}
+        echo "$(date '+%Y-%m-%d %H:%M:%S') $script_name failed with exit code $exit_code" | tee -a "$log_file"
+        fail_cnt=$((fail_cnt+1))
+    fi
+done
+
+rm -rf "$log_dir"
+
+if [ $fail_cnt -ne 0 ]; then
+    echo "There were $fail_cnt failed test cases."
+    exit 1
+fi
diff --git a/tests/src/test_docker_lxcfs.sh b/tests/src/test_docker_lxcfs.sh
new file mode 100755
index 0000000..bf260d8
--- /dev/null
+++ b/tests/src/test_docker_lxcfs.sh
@@ -0,0 +1,65 @@
+###
+ # Copyright (c) Huawei Technologies Co., Ltd. 2021-2024. All rights reserved. 
+ # rubik licensed under the Mulan PSL v2.
+ # You can use this software according to the terms and conditions of the Mulan PSL v2.
+ # You may obtain a copy of Mulan PSL v2 at: 
+ #     http://license.coscl.org.cn/MulanPSL2 
+ # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ # PURPOSE.
+ # See the Mulan PSL v2 for more details.
+ # Author: Jiaqi Yang
+ # Date: 2024-01-24
+ # Description: This file is used for testing docker with lxcfs
+### 
+
+current_dir="$(cd "$(dirname "$0")" && pwd)"
+lib_dir="${current_dir}"/../lib
+image_name="rnd-dockerhub.huawei.com/official/ubuntu:latest"
+con_name="test"
+source "${lib_dir}"/commonlib.sh
+
+function pre() {
+    check_packages
+    check_service docker
+    check_service lxcfs
+    check_lxcfs_tools
+    ensure_docker_image_exists "$image_name"
+    docker rm -f "${con_name}" > /dev/null
+}
+
+function test_fun() {
+    cid=`docker run -itd --cpuset-cpus="0-1" -m 256m -v /var/lib/lxc:/var/lib/lxc:slave \
+    -v /var/lib/lxc/lxcfs/proc/cpuinfo:/proc/cpuinfo:rw \
+    -v /var/lib/lxc/lxcfs/proc/meminfo:/proc/meminfo:rw \
+    -v /var/lib/lxc/lxcfs/proc/stat:/proc/stat:rw \
+    -v /var/lib/lxc/lxcfs/sys/devices/system/cpu:/sys/devices/system/cpu:rw \
+    --name ${con_name} ${image_name} /bin/bash`
+
+    WaitInspect $cid running
+
+    docker exec -it ${con_name} bash -c "free -h" | grep Mem | grep 256
+    fn_check_result $? 0 "check meminfo"
+
+    restart_lxcfs
+    fn_check_result $? 0 "restart lxcfs"
+
+    docker exec -it ${con_name} bash -c "free -h" | grep Mem | grep 256
+    fn_check_result $? 0 "check meminfo"
+
+    docker exec -it test bash -c "cat /proc/cpuinfo | grep processor | wc -l | grep 2"
+    fn_check_result $? 0 "check cpuinfo"
+
+    docker exec -it test bash -c "cat /proc/stat | grep cpu | wc -l | grep 3"
+    fn_check_result $? 0 "check stat"
+}
+
+function main() {
+    pre
+    echo "All services are ready, running test cases..."
+    test_fun
+    clean_env
+    exit $exit_flag
+}
+
+main
diff --git a/umountcmd_test.go b/umountcmd_test.go
index bcae021..dd15a3a 100644
--- a/umountcmd_test.go
+++ b/umountcmd_test.go
@@ -23,7 +23,8 @@ import (
 )
 
 var (
-	oldPath, newPath string
+	oldPath, newPath         string
+	oldRuncPath, newRuncPath string
 )
 
 func isuladExisted() bool {
@@ -34,8 +35,19 @@ func isuladExisted() bool {
 	return true
 }
 
-func whereIsIsulad() string {
-	cmd := exec.Command("whereis", "isula")
+func runcExisted() bool {
+	cmd := exec.Command("runc", "-v")
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+	return true
+}
+
+func whereIsCmd(cmdStr string) string {
+	if cmdStr != "runc" && cmdStr != "isula" {
+		return ""
+	}
+	cmd := exec.Command("whereis", cmdStr)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return ""
@@ -45,12 +57,12 @@ func whereIsIsulad() string {
 	return path
 }
 
-func rename(oldPath, newPath string) error {
-	if err := os.Rename(oldPath, newPath); err != nil {
+func rename(old, new string) error {
+	if err := os.Rename(old, new); err != nil {
 		fmt.Println(err)
 		return err
 	}
-	fmt.Printf("rename isula path from %s to %s\n", oldPath, newPath)
+	fmt.Printf("rename path from %s to %s\n", old, new)
 	return nil
 }
 
@@ -58,8 +70,8 @@ func tryMoveiSulad() bool {
 	oldPath = ""
 	newPath = ""
 	if isuladExisted() {
-		fmt.Println("isuila existed ")
-		oldPath = whereIsIsulad()
+		fmt.Println("isuila existed")
+		oldPath = whereIsCmd("isula")
 		if oldPath == "" {
 			return false
 		}
@@ -71,6 +83,23 @@ func tryMoveiSulad() bool {
 	return true
 }
 
+func tryMoveRunc() bool {
+	oldRuncPath = ""
+	newRuncPath = ""
+	if runcExisted() {
+		fmt.Println("runc existed")
+		oldRuncPath = whereIsCmd("runc")
+		if oldRuncPath == "" {
+			return false
+		}
+		newRuncPath = oldRuncPath + ".bak"
+		if err := rename(oldRuncPath, newRuncPath); err != nil {
+			return false
+		}
+	}
+	return true
+}
+
 func recoveriSulad() {
 	if oldPath == "" || newPath == "" {
 		return
@@ -80,6 +109,15 @@ func recoveriSulad() {
 	}
 }
 
+func recoverRunc() {
+	if oldRuncPath == "" || newRuncPath == "" {
+		return
+	}
+	if err := rename(newRuncPath, oldRuncPath); err != nil {
+		return
+	}
+}
+
 // TestUmountAll tests umountAll
 func TestUmountAll(t *testing.T) {
 	if !tryMoveiSulad() {
@@ -87,12 +125,17 @@ func TestUmountAll(t *testing.T) {
 	}
 	defer recoveriSulad()
 
+	if !tryMoveRunc() {
+		return
+	}
+	defer recoverRunc()
+
 	tests := []struct {
 		name, initMountns, initUserns string
 		wantErr                       bool
 	}{
 		{
-			name:        "TC1-absent of isulad",
+			name:        "TC1-absent of container tools",
 			initMountns: "1",
 			initUserns:  "5",
 			wantErr:     true,
-- 
2.33.0

