diff --git a/0001-add-omitted-musl-adaption-code.patch b/0001-add-omitted-musl-adaption-code.patch deleted file mode 100644 index ece3f8c..0000000 --- a/0001-add-omitted-musl-adaption-code.patch +++ /dev/null @@ -1,44 +0,0 @@ -From 79a0eeae7ebfc2379917144befe5b0b5e39656e4 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Wed, 18 Jan 2023 10:08:22 +0800 -Subject: [PATCH 01/19] add omitted musl adaption code - -Signed-off-by: zhongtao ---- - cmake/set_build_flags.cmake | 4 ++-- - src/daemon/executor/container_cb/execution_stream.c | 2 ++ - 2 files changed, 4 insertions(+), 2 deletions(-) - -diff --git a/cmake/set_build_flags.cmake b/cmake/set_build_flags.cmake -index 89c9468c..aa1b1193 100644 ---- a/cmake/set_build_flags.cmake -+++ b/cmake/set_build_flags.cmake -@@ -1,9 +1,9 @@ - # set common FLAGS --set(CMAKE_C_FLAGS "-fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -Wall -Werror -fPIE") -+set(CMAKE_C_FLAGS "-fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -Wall -fPIE") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'") - - if (GRPC_CONNECTOR) -- set(CMAKE_CXX_FLAGS "-fPIC -std=c++11 -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -Wall -Werror") -+ set(CMAKE_CXX_FLAGS "-fPIC -std=c++11 -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -Wall") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'") - endif() - set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-E -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines -shared -pthread") -diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c -index ebb9ee2b..fde0335e 100644 ---- a/src/daemon/executor/container_cb/execution_stream.c -+++ b/src/daemon/executor/container_cb/execution_stream.c -@@ -68,7 +68,9 @@ - #define PTHREAD_CANCEL_ENABLE 1 - #define PTHREAD_CANCEL_DISABLE 0 - -+#if defined __ANDROID__ - typedef long pthread_t; -+#endif - - static int pthread_setcancelstate(int state, int *oldstate) - { --- -2.25.1 - diff --git a/0002-add-cpu-rt-CI.patch b/0002-add-cpu-rt-CI.patch deleted file mode 100644 index 2bf2b4e..0000000 --- a/0002-add-cpu-rt-CI.patch +++ /dev/null @@ -1,162 +0,0 @@ -From dc36962aa20a2a0d29b070bad2d419ae77d56dfd Mon Sep 17 00:00:00 2001 -From: songbuhuang <544824346@qq.com> -Date: Mon, 30 Jan 2023 15:33:55 +0800 -Subject: [PATCH 02/19] add cpu-rt CI - -Signed-off-by: songbuhuang <544824346@qq.com> ---- - CI/test_cases/container_cases/cpu_rt.sh | 142 ++++++++++++++++++++++++ - 1 file changed, 142 insertions(+) - create mode 100644 CI/test_cases/container_cases/cpu_rt.sh - -diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh -new file mode 100644 -index 00000000..3d70c840 ---- /dev/null -+++ b/CI/test_cases/container_cases/cpu_rt.sh -@@ -0,0 +1,142 @@ -+#!/bin/bash -+# -+# attributes: isulad basic cpu realtime -+# concurrent: NA -+ -+####################################################################### -+##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+# - iSulad licensed under the Mulan PSL v2. -+# - You can use this software according to the terms and conditions of the Mulan PSL v2. -+# - You may obtain a copy of Mulan PSL v2 at: -+# - http://license.coscl.org.cn/MulanPSL2 -+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# - PURPOSE. -+# - See the Mulan PSL v2 for more details. -+##- @Description:CI -+##- @Author: huangsong -+##- @Create: 2023-01-29 -+####################################################################### -+ -+declare -r curr_path=$(dirname $(readlink -f "$0")) -+source ../helpers.sh -+ -+function test_cpu_rt_isulad_spec() -+{ -+ local ret=0 -+ local test="isulad cpu realtime test => (${FUNCNAME[@]})" -+ -+ msg_info "${test} starting..." -+ -+ isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) -+ -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+function test_cpu_rt_isula_spec() -+{ -+ local ret=0 -+ local image="busybox" -+ local test="container cpu realtime test => (${FUNCNAME[@]})" -+ -+ msg_info "${test} starting..." -+ -+ #start isulad without cpu_rt -+ start_isulad_without_valgrind -+ -+ isula pull ${image} -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} -+ -+ isula images | grep busybox -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) -+ -+ test_isula_run_spec -+ -+ #start isulad without cpu_rt:isulad cpu.rt_period_us default value is the cpu.rt_period_us of the upper-layer directory,cpu.rt_runtime_us is 0. -+ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "failed to write 10000" | grep "cpu.rt_runtime_us: Invalid argument" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ -+ stop_isulad_without_valgrind -+ -+ #start isulad with cpu_rt -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ -+ test_isula_run_spec -+ -+ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) -+ -+ isula update --cpu-rt-runtime 90000 $c_id -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) -+ -+ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "90000" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 90000" && ((ret++)) -+ -+ isula rm -f $c_id -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+function test_kernel_without_cpu_rt_spec() -+{ -+ local ret=0 -+ local image="busybox" -+ local test="kernel does not support cpu-rt test => (${FUNCNAME[@]})" -+ -+ msg_info "${test} starting..." -+ -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ -+ isula pull ${image} -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} -+ -+ isula images | grep busybox -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1 | grep "Your kernel does not support cgroup rt" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt" && ((ret++)) -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+function test_isula_run_spec() -+{ -+ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period xx --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period xx --cpu-rt-runtime xx $image /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period -1 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) -+} -+ -+declare -i ans=0 -+ -+if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then -+ test_cpu_rt_isulad_spec || ((ans++)) -+ test_cpu_rt_isula_spec || ((ans++)) -+else -+ test_kernel_without_cpu_rt_spec || ((ans++)) -+fi -+ -+show_result ${ans} "${curr_path}/${0}" --- -2.25.1 - diff --git a/0003-add-cpu-rt-ut.patch b/0003-add-cpu-rt-ut.patch deleted file mode 100644 index b17d07c..0000000 --- a/0003-add-cpu-rt-ut.patch +++ /dev/null @@ -1,141 +0,0 @@ -From c798e4f1abb27d950af6698903891bd2f4dbc63e Mon Sep 17 00:00:00 2001 -From: songbuhuang <544824346@qq.com> -Date: Mon, 30 Jan 2023 15:34:34 +0800 -Subject: [PATCH 03/19] add cpu-rt ut - -Signed-off-by: songbuhuang <544824346@qq.com> ---- - test/CMakeLists.txt | 1 + - test/cgroup/CMakeLists.txt | 3 ++ - test/cgroup/cpu/CMakeLists.txt | 28 +++++++++++++++ - test/cgroup/cpu/cgroup_cpu_ut.cc | 61 ++++++++++++++++++++++++++++++++ - 4 files changed, 93 insertions(+) - create mode 100644 test/cgroup/CMakeLists.txt - create mode 100644 test/cgroup/cpu/CMakeLists.txt - create mode 100644 test/cgroup/cpu/cgroup_cpu_ut.cc - -diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt -index 06adb602..ad348131 100644 ---- a/test/CMakeLists.txt -+++ b/test/CMakeLists.txt -@@ -47,6 +47,7 @@ IF(ENABLE_UT) - add_subdirectory(sha256) - add_subdirectory(buffer) - add_subdirectory(console) -+ add_subdirectory(cgroup) - ENDIF(ENABLE_UT) - - IF(ENABLE_FUZZ) -diff --git a/test/cgroup/CMakeLists.txt b/test/cgroup/CMakeLists.txt -new file mode 100644 -index 00000000..d904a6a0 ---- /dev/null -+++ b/test/cgroup/CMakeLists.txt -@@ -0,0 +1,3 @@ -+project(iSulad_UT) -+ -+add_subdirectory(cpu) -diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt -new file mode 100644 -index 00000000..159b0d85 ---- /dev/null -+++ b/test/cgroup/cpu/CMakeLists.txt -@@ -0,0 +1,28 @@ -+project(iSulad_UT) -+ -+SET(EXE cgroup_cpu_ut) -+ -+add_executable(${EXE} -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c -+ cgroup_cpu_ut.cc) -+ -+target_include_directories(${EXE} PUBLIC -+ ${GTEST_INCLUDE_DIR} -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../include -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common -+ ${CMAKE_BINARY_DIR}/conf -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad -+ ) -+ -+target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) -+add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc -new file mode 100644 -index 00000000..1cfdf86f ---- /dev/null -+++ b/test/cgroup/cpu/cgroup_cpu_ut.cc -@@ -0,0 +1,61 @@ -+/* -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Description: specs unit test -+ * Author: huangsong -+ * Create: 2023-01-29 -+ */ -+ -+#include -+#include -+#include -+#include "daemon_arguments.h" -+#include "isulad_config.h" -+#include "utils.h" -+ -+struct service_arguments *new_args(int64_t cpu_rt_period, int64_t cpu_rt_runtime) -+{ -+ struct service_arguments *args = (struct service_arguments *)util_common_calloc_s(sizeof(struct service_arguments)); -+ if (args == nullptr) { -+ std::cerr << "Out of memory" << std::endl; -+ return nullptr; -+ } -+ -+ args->json_confs = (isulad_daemon_configs *)util_common_calloc_s(sizeof(isulad_daemon_configs)); -+ if (args->json_confs == nullptr) { -+ std::cerr << "Out of memory" << std::endl; -+ free(args); -+ return nullptr; -+ } -+ -+ args->json_confs->cpu_rt_period = cpu_rt_period; -+ args->json_confs->cpu_rt_runtime = cpu_rt_runtime; -+ -+ return args; -+} -+ -+TEST(CgroupCpuUnitTest, test_conf_get_cgroup_cpu_rt) -+{ -+ int64_t cpu_rt_period = 0; -+ int64_t cpu_rt_runtime = 0; -+ -+ ASSERT_EQ(conf_get_cgroup_cpu_rt(nullptr, nullptr), -1); -+ ASSERT_EQ(conf_get_cgroup_cpu_rt(&cpu_rt_period, nullptr), -1); -+ ASSERT_EQ(conf_get_cgroup_cpu_rt(nullptr, &cpu_rt_runtime), -1); -+ -+ struct service_arguments *args = new_args(cpu_rt_period, cpu_rt_runtime); -+ ASSERT_EQ(save_args_to_conf(args), 0); -+ ASSERT_EQ(conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime), 0); -+ ASSERT_EQ(cpu_rt_period, 0); -+ ASSERT_EQ(cpu_rt_runtime, 0); -+ -+} -+ -+ --- -2.25.1 - diff --git a/0004-add-info-log-when-isulad-shutdown.patch b/0004-add-info-log-when-isulad-shutdown.patch deleted file mode 100644 index d3e69d3..0000000 --- a/0004-add-info-log-when-isulad-shutdown.patch +++ /dev/null @@ -1,148 +0,0 @@ -From d6632459d8fe0e3ddbce6b360ecaf765cf646fa3 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Mon, 30 Jan 2023 17:41:19 +0800 -Subject: [PATCH 04/19] add info log when isulad shutdown - -Signed-off-by: zhangxiaoyu ---- - src/cmd/isulad/main.c | 5 +++++ - src/daemon/entry/connect/service_common.c | 4 +++- - src/daemon/modules/image/image.c | 1 + - .../graphdriver/devmapper/deviceset.c | 17 +++++++++++++---- - .../storage/layer_store/graphdriver/driver.c | 2 ++ - 5 files changed, 24 insertions(+), 5 deletions(-) - -diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c -index 9b664bee..d22ab407 100644 ---- a/src/cmd/isulad/main.c -+++ b/src/cmd/isulad/main.c -@@ -267,15 +267,20 @@ static void clean_residual_files() - - static void daemon_shutdown() - { -+ EVENT("Begin shutdown daemon"); -+ - /* shutdown server */ - server_common_shutdown(); - - /* clean resource first, left time to wait finish */ - image_module_exit(); -+ EVENT("Image module exit completed"); - - umount_daemon_mntpoint(); -+ EVENT("Umount daemon mntpoint completed"); - - clean_residual_files(); -+ EVENT("Clean residual files completed"); - - sem_post(&g_daemon_wait_shutdown_sem); - } -diff --git a/src/daemon/entry/connect/service_common.c b/src/daemon/entry/connect/service_common.c -index 162cec2b..093f05c0 100644 ---- a/src/daemon/entry/connect/service_common.c -+++ b/src/daemon/entry/connect/service_common.c -@@ -16,13 +16,13 @@ - #include "service_common.h" - - #include -+#include - - #include "daemon_arguments.h" - #ifdef GRPC_CONNECTOR - #include "grpc_service.h" - #else - #include "rest_service.h" --#include "isula_libutils/log.h" - #endif - - /* server common init */ -@@ -58,8 +58,10 @@ void server_common_shutdown(void) - { - #ifdef GRPC_CONNECTOR - grpc_server_shutdown(); -+ EVENT("Grpc Server shutdown completed"); - #else - rest_server_shutdown(); -+ EVENT("Rest Server shutdown completed"); - #endif - } - -diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c -index 6fac7cc4..158b3aa8 100644 ---- a/src/daemon/modules/image/image.c -+++ b/src/daemon/modules/image/image.c -@@ -1909,6 +1909,7 @@ void image_module_exit() - continue; - } - g_bims[i].ops->clean_resource(); -+ EVENT("Image %s clean resource completed", g_bims[i].image_type); - } - } - -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c -index b157510a..76059b81 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c -@@ -3419,9 +3419,13 @@ static int umount_deactivate_dev_all(const struct device_set *devset) - - device_info = lookup_device(devset, entry->d_name); - if (device_info == NULL) { -- DEBUG("devmapper: shutdown lookup device %s err", entry->d_name); -- } else if (deactivate_device(devset, device_info->info) != 0) { -- DEBUG("devmapper: shutdown deactivate device %s err", entry->d_name); -+ WARN("devmapper: shutdown lookup device %s err", entry->d_name); -+ continue; -+ } -+ if (deactivate_device(devset, device_info->info) != 0) { -+ WARN("devmapper: shutdown deactivate device %s err", entry->d_name); -+ } else { -+ INFO("devmapper: shutdown deactivate device %s complete", entry->d_name); - } - devmapper_device_info_ref_dec(device_info); - } -@@ -3429,7 +3433,9 @@ static int umount_deactivate_dev_all(const struct device_set *devset) - device_info = lookup_device(devset, "base"); - if (device_info != NULL) { - if (deactivate_device(devset, device_info->info) != 0) { -- DEBUG("devmapper: shutdown deactivate base device err"); -+ WARN("devmapper: shutdown deactivate base device err"); -+ } else { -+ INFO("devmapper: shutdown deactivate base device complete"); - } - devmapper_device_info_ref_dec(device_info); - } -@@ -3453,15 +3459,18 @@ int device_set_shutdown(struct device_set *devset, const char *home) - return -1; - } - -+ EVENT("Devmapper: begin shutdown device set"); - if (save_deviceset_matadata(devset)) { - DEBUG("devmapper: save deviceset metadata failed"); - } -+ EVENT("Devmapper: save deviceset metadata completed"); - - if (umount_deactivate_dev_all(devset) != 0) { - ERROR("devmapper: Shutdown umount device failed"); - ret = -1; - goto free_out; - } -+ EVENT("Devmapper: Shutdown umount device completed"); - - free_out: - if (pthread_rwlock_unlock(&(devset->devmapper_driver_rwlock)) != 0) { -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -index 4d685bb6..d3b5209a 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -@@ -514,6 +514,8 @@ int graphdriver_cleanup(void) - // notes, do not call driver_unlock and destroy the lock, becase the other threads may wait for it - // if we unlock and destroy the lock, may cause the lock failure, and result to coredump - -+ EVENT("Graph driver %s cleanup completed", g_graphdriver->name); -+ - out: - return ret; - } --- -2.25.1 - diff --git a/0005-create-a-log-file-for-shim-v2-and-remove-10-229-devi.patch b/0005-create-a-log-file-for-shim-v2-and-remove-10-229-devi.patch deleted file mode 100644 index 8706767..0000000 --- a/0005-create-a-log-file-for-shim-v2-and-remove-10-229-devi.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 973c12fcddfb64bd14e0076bc5c21170f1feca6a Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Thu, 2 Feb 2023 10:51:42 +0800 -Subject: [PATCH 05/19] create a log file for shim v2 and remove 10,229 device - from the default config - -Signed-off-by: zhongtao ---- - src/common/constants.h | 4 ++++ - src/contrib/config/config.json | 7 ------- - src/daemon/modules/runtime/shim/shim_rt_ops.c | 18 ++++++++++++++++++ - 3 files changed, 22 insertions(+), 7 deletions(-) - -diff --git a/src/common/constants.h b/src/common/constants.h -index e6809998..93a069de 100644 ---- a/src/common/constants.h -+++ b/src/common/constants.h -@@ -126,6 +126,10 @@ extern "C" { - #define ISULAD_ENABLE_PLUGINS_SEPERATOR "," - #define ISULAD_ENABLE_PLUGINS_SEPERATOR_CHAR ',' - -+#ifdef ENABLE_SHIM_V2 -+#define SHIM_V2_LOG "/log" -+#endif -+ - #define MAX_HOSTS 10 - - #define OPT_MAX_LEN 255 -diff --git a/src/contrib/config/config.json b/src/contrib/config/config.json -index cfcdc375..f84f3394 100644 ---- a/src/contrib/config/config.json -+++ b/src/contrib/config/config.json -@@ -210,13 +210,6 @@ - "major": 10, - "minor": 200, - "access": "rwm" -- }, -- { -- "allow": false, -- "type": "c", -- "major": 10, -- "minor": 229, -- "access": "rwm" - } - ] - }, -diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c -index 9d23a0e7..1b497358 100644 ---- a/src/daemon/modules/runtime/shim/shim_rt_ops.c -+++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c -@@ -250,9 +250,11 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t - { - int ret = 0; - int pid = 0; -+ int fd = -1; - char addr[PATH_MAX] = {0}; - char *exit_fifo_path = NULL; - char *state_path = NULL; -+ char *log_path = NULL; - - if (id == NULL || runtime == NULL || params == NULL) { - ERROR("Invalid input params"); -@@ -273,6 +275,21 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t - goto out; - } - -+ log_path = util_string_append(SHIM_V2_LOG, params->bundle); -+ if (log_path == NULL) { -+ ERROR("Fail to append log path"); -+ ret = -1; -+ goto out; -+ } -+ -+ fd = util_open(log_path, O_RDWR | O_CREAT | O_TRUNC, DEFAULT_SECURE_FILE_MODE); -+ if (fd < 0) { -+ ERROR("Failed to create log file for shim v2: %s", log_path); -+ ret = -1; -+ goto out; -+ } -+ close(fd); -+ - if (shim_bin_v2_create(runtime, id, params->bundle, NULL, addr, state_path) != 0) { - ERROR("%s: failed to create v2 shim", id); - ret = -1; -@@ -294,6 +311,7 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t - } - - out: -+ free(log_path); - free(exit_fifo_path); - free(state_path); - return ret; --- -2.25.1 - diff --git a/0006-ensure-isula-exec-inherits-the-config-of-create.patch b/0006-ensure-isula-exec-inherits-the-config-of-create.patch deleted file mode 100644 index 90d9f12..0000000 --- a/0006-ensure-isula-exec-inherits-the-config-of-create.patch +++ /dev/null @@ -1,137 +0,0 @@ -From d1960a64b04888f21173f5c4bb4993c86553f417 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Thu, 2 Feb 2023 06:49:03 +0800 -Subject: [PATCH 06/19] ensure isula exec inherits the config of create. - -Signed-off-by: zhongtao ---- - .../modules/service/service_container.c | 73 ++++++++++++++++--- - 1 file changed, 63 insertions(+), 10 deletions(-) - -diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c -index 58c43476..340dea07 100644 ---- a/src/daemon/modules/service/service_container.c -+++ b/src/daemon/modules/service/service_container.c -@@ -1755,8 +1755,34 @@ out: - return ret; - } - -+static int append_rlimit_from_oci_spec(defs_process *spec, const oci_runtime_spec *oci_spec) -+{ -+ size_t j; -+ -+ spec->rlimits = (defs_process_rlimits_element **)util_smart_calloc_s(sizeof(defs_process_rlimits_element *), -+ (size_t)oci_spec->process->rlimits_len); -+ if (spec->rlimits == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ -+ for (j = 0; j < oci_spec->process->rlimits_len; j++) { -+ spec->rlimits[j] = util_common_calloc_s(sizeof(defs_process_rlimits_element)); -+ if (spec->rlimits[j] == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ spec->rlimits[j]->type = util_strdup_s(oci_spec->process->rlimits[j]->type); -+ spec->rlimits[j]->hard = oci_spec->process->rlimits[j]->hard; -+ spec->rlimits[j]->soft = oci_spec->process->rlimits[j]->soft; -+ spec->rlimits_len++; -+ } -+ -+ return 0; -+} -+ - static defs_process *make_exec_process_spec(const container_config *container_spec, defs_process_user *puser, -- const char *runtime, const container_exec_request *request) -+ const char *runtime, const container_exec_request *request, const oci_runtime_spec *oci_spec) - { - int ret = 0; - defs_process *spec = NULL; -@@ -1767,25 +1793,38 @@ static defs_process *make_exec_process_spec(const container_config *container_sp - } - - if (strcasecmp(runtime, "lcr") != 0) { -+ // for oci runtime: -+ // step 1: merge env from container; - ret = merge_exec_from_container_env(spec, container_spec); - if (ret != 0) { - ERROR("Failed to dup args for exec process spec"); - goto err_out; - } -- } -- -- ret = merge_envs_from_request_env(spec, (const char **)request->env, request->env_len); -- if (ret != 0) { -- ERROR("Failed to dup args for exec process spec"); -- goto err_out; -- } - -- if (strcasecmp(runtime, "lcr") != 0) { -+ // step 2: merge process env including PATH, HOATNAME and TERM(if tty is true); - ret = append_necessary_process_env(request->tty, container_spec, spec); - if (ret != 0) { - ERROR("Failed to append necessary for exec process spec"); - goto err_out; - } -+ -+ ret = append_rlimit_from_oci_spec(spec, oci_spec); -+ if (ret != 0) { -+ ERROR("Failed to append rlimit for exec process spec"); -+ goto err_out; -+ } -+ -+ spec->no_new_privileges = oci_spec->process->no_new_privileges; -+ } -+ -+ // for oci runtime: -+ // step 3 : Finally, merge env from request to ensure that the env in the request is not overwritten; -+ // for lcr: -+ // since the container env and the process env have been stored in the config file, lcr only needs to merge the env in the request. -+ ret = merge_envs_from_request_env(spec, (const char **)request->env, request->env_len); -+ if (ret != 0) { -+ ERROR("Failed to dup args for exec process spec"); -+ goto err_out; - } - - ret = util_dup_array_of_strings((const char **)request->argv, request->argv_len, &(spec->args), &(spec->args_len)); -@@ -1822,6 +1861,8 @@ static int do_exec_container(const container_t *cont, const char *runtime, char - char *engine_log_path = NULL; - char *loglevel = NULL; - char *logdriver = NULL; -+ const char *id = cont->common_config->id; -+ oci_runtime_spec *oci_spec = NULL; - defs_process *process_spec = NULL; - rt_exec_params_t params = { 0 }; - -@@ -1844,7 +1885,18 @@ static int do_exec_container(const container_t *cont, const char *runtime, char - goto out; - } - -- process_spec = make_exec_process_spec(cont->common_config->config, puser, runtime, request); -+ // lcr reads the config from the file and will not lose it. -+ // so there is no need to get the config from oci_spec. -+ if (strcasecmp(runtime, "lcr") != 0) { -+ oci_spec = load_oci_config(cont->root_path, id); -+ if (oci_spec == NULL) { -+ ERROR("Failed to load oci config"); -+ ret = -1; -+ goto out; -+ } -+ } -+ -+ process_spec = make_exec_process_spec(cont->common_config->config, puser, runtime, request, oci_spec); - if (process_spec == NULL) { - ERROR("Exec: Failed to make process spec"); - ret = -1; -@@ -1873,6 +1925,7 @@ out: - free(engine_log_path); - free(logdriver); - free_defs_process(process_spec); -+ free_oci_runtime_spec(oci_spec); - - return ret; - } --- -2.25.1 - diff --git a/0007-Delete-meaningless-thread-creation-and-ensure-the-ta.patch b/0007-Delete-meaningless-thread-creation-and-ensure-the-ta.patch deleted file mode 100644 index 107a7be..0000000 --- a/0007-Delete-meaningless-thread-creation-and-ensure-the-ta.patch +++ /dev/null @@ -1,199 +0,0 @@ -From 8ac6ba83be4514b94132748428cef0bbef7726d7 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sun, 29 Jan 2023 15:48:07 +0800 -Subject: [PATCH 07/19] Delete meaningless thread creation and ensure the - task_console_accept thread ends before destroying the io_thread - -Signed-off-by: zhongtao ---- - src/cmd/isulad-shim/main.c | 5 ++-- - src/cmd/isulad-shim/process.c | 52 ++++++++++++++++++++++------------- - src/cmd/isulad-shim/process.h | 4 +-- - 3 files changed, 38 insertions(+), 23 deletions(-) - -diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c -index 3ab22d86..eedd8fda 100644 ---- a/src/cmd/isulad-shim/main.c -+++ b/src/cmd/isulad-shim/main.c -@@ -98,6 +98,7 @@ int main(int argc, char **argv) - int ret = SHIM_ERR; - int efd = -1; - process_t *p = NULL; -+ pthread_t tid_accept; - - g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); - if (g_log_fd < 0) { -@@ -151,7 +152,7 @@ int main(int argc, char **argv) - exit(EXIT_FAILURE); - } - -- ret = open_io(p); -+ ret = open_io(p, &tid_accept); - if (ret != SHIM_OK) { - exit(EXIT_FAILURE); - } -@@ -166,5 +167,5 @@ int main(int argc, char **argv) - - released_timeout_exit(); - -- return process_signal_handle_routine(p); -+ return process_signal_handle_routine(p, tid_accept); - } -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index 92dd2701..66c604f3 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -423,6 +423,14 @@ static int start_io_copy_threads(process_t *p) - - /* 4 threads for stdin, stdout, stderr and exec resize */ - for (i = 0; i < 4; i++) { -+ /* -+ * if the terminal is used, we do not need to active the io copy of stderr pipe, -+ * for stderr and stdout are mixed together -+ */ -+ if (i == STDID_ERR && p->state->terminal) { -+ continue; -+ } -+ - ret = create_io_copy_thread(p, i); - if (ret != SHIM_OK) { - return SHIM_ERR; -@@ -512,11 +520,6 @@ static void *task_console_accept(void *data) - int ret = SHIM_ERR; - console_accept_t *ac = (console_accept_t *)data; - -- if ((pthread_detach(pthread_self())) != 0) { -- write_message(g_log_fd, ERR_MSG, "detach thread failed"); -- return NULL; -- } -- - conn_fd = accept(ac->listen_fd, NULL, NULL); - if (conn_fd < 0) { - write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno)); -@@ -549,12 +552,6 @@ static void *task_console_accept(void *data) - goto out; - } - -- /* -- * if the terminal is used, we do not need to active the io copy of stderr pipe, -- * for stderr and stdout are mixed together -- */ -- destroy_io_thread(ac->p, STDID_ERR); -- - out: - /* release listen socket at the first time */ - close_fd(&ac->listen_fd); -@@ -634,7 +631,7 @@ static int new_temp_console_path(process_t *p) - return SHIM_OK; - } - --static int console_init(process_t *p) -+static int console_init(process_t *p, pthread_t *tid_accept) - { - int ret = SHIM_ERR; - int fd = -1; -@@ -670,8 +667,7 @@ static int console_init(process_t *p) - ac->p = p; - ac->listen_fd = fd; - -- pthread_t tid_accept; -- ret = pthread_create(&tid_accept, NULL, task_console_accept, ac); -+ ret = pthread_create(tid_accept, NULL, task_console_accept, ac); - if (ret != SHIM_OK) { - goto failure; - } -@@ -762,7 +758,7 @@ failure: - return NULL; - } - --static int open_terminal_io(process_t *p) -+static int open_terminal_io(process_t *p, pthread_t *tid_accept) - { - int ret = SHIM_ERR; - -@@ -773,7 +769,7 @@ static int open_terminal_io(process_t *p) - } - - /* begin listen and accept fd from p->console_sock_path */ -- return console_init(p); -+ return console_init(p, tid_accept); - } - - static int open_generic_io(process_t *p) -@@ -916,7 +912,7 @@ failure: - return NULL; - } - --int open_io(process_t *p) -+int open_io(process_t *p, pthread_t *tid_accept) - { - int ret = SHIM_ERR; - -@@ -926,7 +922,7 @@ int open_io(process_t *p) - } - - if (p->state->terminal) { -- return open_terminal_io(p); -+ return open_terminal_io(p, tid_accept); - } - - return open_generic_io(p); -@@ -1216,12 +1212,13 @@ static int try_wait_all_child(void) { - return 1; - } - --int process_signal_handle_routine(process_t *p) -+int process_signal_handle_routine(process_t *p, const pthread_t tid_accept) - { - int ret = SHIM_ERR; - bool exit_shim = false; - int nret = 0; - int i; -+ struct timespec ts; - - for (;;) { - int status; -@@ -1257,6 +1254,23 @@ int process_signal_handle_routine(process_t *p) - if (p->exit_fd > 0) { - (void)write_nointr(p->exit_fd, &status, sizeof(int)); - } -+ // wait for task_console_accept thread termination. In order to make sure that -+ // the io_copy connection is established and io_thread is not used by multiple threads. -+ if (p->state->terminal) { -+ if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { -+ write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); -+ nret = pthread_join(tid_accept, NULL); -+ } else { -+ // Set the maximum waiting time to 60s to prevent stuck. -+ ts.tv_sec += 60; -+ nret = pthread_timedjoin_np(tid_accept, NULL, &ts); -+ } -+ -+ if (nret != 0) { -+ write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); -+ } -+ } -+ - for (i = 0; i < 3; i++) { - destroy_io_thread(p, i); - } -diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h -index 11d1bf64..66820f68 100644 ---- a/src/cmd/isulad-shim/process.h -+++ b/src/cmd/isulad-shim/process.h -@@ -94,10 +94,10 @@ typedef struct { - - process_t* new_process(char *id, char *bundle, char *runtime); - --int open_io(process_t *p); -+int open_io(process_t *p, pthread_t *tid_accept); - int process_io_init(process_t *p); - int create_process(process_t *p); --int process_signal_handle_routine(process_t *p); -+int process_signal_handle_routine(process_t *p, const pthread_t tid_accept); - - #ifdef __cplusplus - } --- -2.25.1 - diff --git a/0008-fix-cpu-rt-review-comments.patch b/0008-fix-cpu-rt-review-comments.patch deleted file mode 100644 index 54514f4..0000000 --- a/0008-fix-cpu-rt-review-comments.patch +++ /dev/null @@ -1,44 +0,0 @@ -From c05c92988577067ea376a2e7263a2c9080fcb478 Mon Sep 17 00:00:00 2001 -From: songbuhuang <544824346@qq.com> -Date: Mon, 13 Feb 2023 10:43:52 +0800 -Subject: [PATCH 08/19] fix cpu rt review comments - -Signed-off-by: songbuhuang <544824346@qq.com> ---- - src/daemon/executor/container_cb/execution_create.c | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c -index da01a57f..04154765 100644 ---- a/src/daemon/executor/container_cb/execution_create.c -+++ b/src/daemon/executor/container_cb/execution_create.c -@@ -1295,7 +1295,7 @@ static int save_container_config_before_create(const char *id, const char *runti - static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) - { - int ret; -- int fd = 0; -+ int fd = -1; - ssize_t nwrite; - char fpath[PATH_MAX] = { 0 }; - char buf[ISULAD_NUMSTRLEN64] = { 0 }; -@@ -1310,13 +1310,13 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const - return -1; - } - -- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); -- if (nret < 0 || nret >= sizeof(fpath)) { -+ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); -+ if (ret < 0 || ret >= sizeof(fpath)) { - ERROR("Failed to print string"); - return -1; - } -- nret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); -- if (nret < 0 || (size_t)nret >= sizeof(buf)) { -+ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); -+ if (ret < 0 || (size_t)ret >= sizeof(buf)) { - ERROR("Failed to print string"); - return -1; - } --- -2.25.1 - diff --git a/0009-fix-inspect.sh-failed.patch b/0009-fix-inspect.sh-failed.patch deleted file mode 100644 index 6f03751..0000000 --- a/0009-fix-inspect.sh-failed.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 921de80e43a15392d14d53ef6cf7e61ff453685b Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Mon, 13 Feb 2023 17:42:30 +0800 -Subject: [PATCH 09/19] fix inspect.sh failed - -Signed-off-by: zhangxiaoyu ---- - CI/test_cases/container_cases/inspect.sh | 7 +------ - 1 file changed, 1 insertion(+), 6 deletions(-) - -diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh -index 0d4ccb02..cde9ea1f 100755 ---- a/CI/test_cases/container_cases/inspect.sh -+++ b/CI/test_cases/container_cases/inspect.sh -@@ -103,12 +103,7 @@ function test_inspect_spec() - isula inspect --format='{{.Image}}' $containername 2>&1 | grep "sha256:${image_id}" - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) - -- if [ -d /sys/fs/cgroup/files ];then -- grepval="100" -- else -- grepval="0" -- fi -- isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep "$grepval" -+ isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep 0 - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++)) - - isula inspect --format='{{json .Config.Env}}' $containername 2>&1 | grep "a=1" --- -2.25.1 - diff --git a/0010-ensure-list-name-is-not-null.patch b/0010-ensure-list-name-is-not-null.patch deleted file mode 100644 index 85dd397..0000000 --- a/0010-ensure-list-name-is-not-null.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 4d5fff88868354403fffa8ae0f6aa8e051842a70 Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Wed, 15 Feb 2023 11:27:04 +0800 -Subject: [PATCH 10/19] ensure list name is not null - -Signed-off-by: haozi007 ---- - src/daemon/entry/cri/cni_network_plugin.cc | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc -index 976a21a4..9b03bea1 100644 ---- a/src/daemon/entry/cri/cni_network_plugin.cc -+++ b/src/daemon/entry/cri/cni_network_plugin.cc -@@ -296,7 +296,7 @@ void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vec - continue; - } - -- if (n_list == nullptr || n_list->plugin_len == 0) { -+ if (n_list == nullptr || n_list->name == nullptr || n_list->plugin_len == 0) { - WARN("CNI config list %s has no networks, skipping", elem.c_str()); - free_cni_network_list_conf(n_list); - n_list = nullptr; --- -2.25.1 - diff --git a/0011-Bugfix-in-config-and-executor.patch b/0011-Bugfix-in-config-and-executor.patch deleted file mode 100644 index edb1e72..0000000 --- a/0011-Bugfix-in-config-and-executor.patch +++ /dev/null @@ -1,52 +0,0 @@ -From 14d06efa2e5fad9986a12a067e1c6009bf58ad47 Mon Sep 17 00:00:00 2001 -From: Xuepeng Xu -Date: Wed, 15 Feb 2023 12:19:40 +0800 -Subject: [PATCH 11/19] Bugfix in config and executor - -Signed-off-by: Xuepeng Xu ---- - src/daemon/config/isulad_config.c | 2 +- - src/daemon/executor/container_cb/execution_create.c | 6 +++--- - 2 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c -index 86a9609a..ad82aeff 100644 ---- a/src/daemon/config/isulad_config.c -+++ b/src/daemon/config/isulad_config.c -@@ -314,7 +314,7 @@ char *conf_get_routine_rootdir(const char *runtime) - } - - /* path = conf->rootpath + / + engines + / + runtime + /0 */ -- if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 3) { -+ if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME) - strlen(runtime)) - 3) { - ERROR("Graph path is too long"); - goto out; - } -diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c -index 04154765..0d03afcb 100644 ---- a/src/daemon/executor/container_cb/execution_create.c -+++ b/src/daemon/executor/container_cb/execution_create.c -@@ -587,7 +587,7 @@ static char *try_generate_id() - int i = 0; - int max_time = 10; - char *id = NULL; -- char *value = NULL; -+ container_t *value = NULL; - - id = util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1)); - if (id == NULL) { -@@ -601,9 +601,9 @@ static char *try_generate_id() - goto err_out; - } - -- value = container_name_index_get(id); -+ value = containers_store_get(id); - if (value != NULL) { -- free(value); -+ container_unref(value); - value = NULL; - continue; - } else { --- -2.25.1 - diff --git a/0012-fix-isula-cpu-rt-CI.patch b/0012-fix-isula-cpu-rt-CI.patch deleted file mode 100644 index 6c0c893..0000000 --- a/0012-fix-isula-cpu-rt-CI.patch +++ /dev/null @@ -1,204 +0,0 @@ -From 3a85cbbb425fa6bd31efd4a296111b9ec3a5e99f Mon Sep 17 00:00:00 2001 -From: songbuhuang <544824346@qq.com> -Date: Tue, 14 Feb 2023 14:37:09 +0800 -Subject: [PATCH 12/19] fix isula cpu-rt CI - -Signed-off-by: songbuhuang <544824346@qq.com> ---- - CI/test_cases/container_cases/cpu_rt.sh | 118 ++++++++++++++++++------ - 1 file changed, 90 insertions(+), 28 deletions(-) - -diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh -index 3d70c840..3dcf4791 100644 ---- a/CI/test_cases/container_cases/cpu_rt.sh -+++ b/CI/test_cases/container_cases/cpu_rt.sh -@@ -21,71 +21,91 @@ - declare -r curr_path=$(dirname $(readlink -f "$0")) - source ../helpers.sh - --function test_cpu_rt_isulad_spec() -+function test_cpurt_isulad_abnormal() - { - local ret=0 -- local test="isulad cpu realtime test => (${FUNCNAME[@]})" -+ local test="isulad cpu realtime abnormal test => (${FUNCNAME[@]})" - -- msg_info "${test} starting..." -+ msg_info "${test} starting..." - -- isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' -+ isulad --cpu-rt-period xx --cpu-rt-runtime 950000 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument' - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++)) - -- isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument' - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) - - msg_info "${test} finished with return ${ret}..." - return ${ret} - } - --function test_cpu_rt_isula_spec() -+function test_isula_update_normal() - { - local ret=0 - local image="busybox" -- local test="container cpu realtime test => (${FUNCNAME[@]})" -+ local test="isulad update cpu realtime normal test => (${FUNCNAME[@]})" - - msg_info "${test} starting..." - -- #start isulad without cpu_rt -- start_isulad_without_valgrind -+ #start isulad with cpu_rt -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ -+ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) - -- isula pull ${image} -- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} -+ isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) - -- isula images | grep busybox -- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) -+ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "2000" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 2000" && ((ret++)) - -- test_isula_run_spec -+ isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++)) - -- #start isulad without cpu_rt:isulad cpu.rt_period_us default value is the cpu.rt_period_us of the upper-layer directory,cpu.rt_runtime_us is 0. -- isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "failed to write 10000" | grep "cpu.rt_runtime_us: Invalid argument" -- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ isula rm -f $c_id -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) - - stop_isulad_without_valgrind -+ #set cpu-rt to the initial state -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+function test_isula_update_abnormal() -+{ -+ local ret=0 -+ local image="busybox" -+ local test="isulad update cpu realtime abnormal test => (${FUNCNAME[@]})" - - #start isulad with cpu_rt - isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & - wait_isulad_running -- -- test_isula_run_spec - -- c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000 ${image} sh` -+ c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000 ${image} sh` - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) - -- isula update --cpu-rt-runtime 90000 $c_id -+ isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) - -- isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "90000" -- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 90000" && ((ret++)) -+ isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++)) - - isula rm -f $c_id - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) - -+ stop_isulad_without_valgrind -+ #set cpu-rt to the initial state -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ - msg_info "${test} finished with return ${ret}..." - return ${ret} - } - --function test_kernel_without_cpu_rt_spec() -+function test_kernel_without_cpurt() - { - local ret=0 - local image="busybox" -@@ -109,8 +129,24 @@ function test_kernel_without_cpu_rt_spec() - return ${ret} - } - --function test_isula_run_spec() -+function test_isula_run_abnormal() - { -+ local ret=0 -+ local image="busybox" -+ local test="container cpu realtime test => (${FUNCNAME[@]})" -+ -+ msg_info "${test} starting..." -+ -+ #start isulad without cpu_rt -+ isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 & -+ wait_isulad_running -+ -+ isula pull ${image} -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} -+ -+ isula images | grep busybox -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) -+ - isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument" - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) - -@@ -128,15 +164,41 @@ function test_isula_run_spec() - - isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period" - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++)) -+ -+ isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument" -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++)) -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+function test_isula_run_normal() -+{ -+ local ret=0 -+ local image="busybox" -+ -+ isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1 -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++)) -+ -+ isula rm -f box -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++)) -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} - } - - declare -i ans=0 - - if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then -- test_cpu_rt_isulad_spec || ((ans++)) -- test_cpu_rt_isula_spec || ((ans++)) -+ test_isula_run_abnormal || ((ans++)) -+ test_isula_run_normal || ((ans++)) -+ test_cpurt_isulad_abnormal || ((ans++)) -+ test_isula_update_normal || ((ans++)) -+ test_isula_update_abnormal || ((ans++)) - else -- test_kernel_without_cpu_rt_spec || ((ans++)) -+ test_kernel_without_cpurt || ((ans++)) - fi - -+isula rm -f $(isula ps -aq) -+ - show_result ${ans} "${curr_path}/${0}" --- -2.25.1 - diff --git a/0013-add-CRI-ContainerStats-Service.patch b/0013-add-CRI-ContainerStats-Service.patch deleted file mode 100644 index dc1ea20..0000000 --- a/0013-add-CRI-ContainerStats-Service.patch +++ /dev/null @@ -1,505 +0,0 @@ -From 18c2515d9a630242387a90c70f421622531a8dec Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Thu, 16 Feb 2023 14:36:23 +0800 -Subject: [PATCH 13/19] add CRI ContainerStats Service - -Signed-off-by: zhangxiaoyu ---- - .../connect/grpc/runtime_image_service.cc | 12 ++--- - .../connect/grpc/runtime_runtime_service.cc | 45 +++++++++++----- - .../connect/grpc/runtime_runtime_service.h | 3 ++ - .../entry/cri/cri_container_manager_service.h | 3 ++ - .../cri/cri_container_manager_service_impl.cc | 54 +++++++++++++++++++ - .../cri/cri_container_manager_service_impl.h | 3 ++ - src/daemon/entry/cri/cri_runtime_service.h | 3 ++ - .../entry/cri/cri_runtime_service_impl.cc | 6 +++ - .../entry/cri/cri_runtime_service_impl.h | 3 ++ - src/daemon/modules/events/collector.c | 4 +- - src/daemon/modules/image/image.c | 16 +++--- - .../modules/image/oci/oci_common_operators.c | 8 +-- - 12 files changed, 128 insertions(+), 32 deletions(-) - -diff --git a/src/daemon/entry/connect/grpc/runtime_image_service.cc b/src/daemon/entry/connect/grpc/runtime_image_service.cc -index 23447baf..e593a9c6 100644 ---- a/src/daemon/entry/connect/grpc/runtime_image_service.cc -+++ b/src/daemon/entry/connect/grpc/runtime_image_service.cc -@@ -54,7 +54,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, - std::vector> images; - Errors error; - -- WARN("Event: {Object: CRI, Type: Listing all images}"); -+ INFO("Event: {Object: CRI, Type: Listing all images}"); - - rService->ListImages(request->filter(), &images, error); - if (!error.Empty()) { -@@ -70,7 +70,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context, - *image = *(iter->get()); - } - -- WARN("Event: {Object: CRI, Type: Listed all images}"); -+ INFO("Event: {Object: CRI, Type: Listed all images}"); - - return grpc::Status::OK; - } -@@ -82,7 +82,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, - std::unique_ptr image_info = nullptr; - Errors error; - -- WARN("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); -+ INFO("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str()); - - image_info = rService->ImageStatus(request->image(), error); - if (!error.Empty() && !CRIHelpers::IsImageNotFoundError(error.GetMessage())) { -@@ -96,7 +96,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context, - *image = *image_info; - } - -- WARN("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); -+ INFO("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str()); - - return grpc::Status::OK; - } -@@ -108,7 +108,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, - std::vector> usages; - Errors error; - -- WARN("Event: {Object: CRI, Type: Statusing image fs info}"); -+ INFO("Event: {Object: CRI, Type: Statusing image fs info}"); - - rService->ImageFsInfo(&usages, error); - if (!error.Empty()) { -@@ -125,7 +125,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context, - *fs_info = *(iter->get()); - } - -- WARN("Event: {Object: CRI, Type: Statused image fs info}"); -+ INFO("Event: {Object: CRI, Type: Statused image fs info}"); - return grpc::Status::OK; - } - -diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -index 7cceefc9..451eeeef 100644 ---- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -+++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -@@ -181,7 +181,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Listing all Container}"); -+ INFO("Event: {Object: CRI, Type: Listing all Container}"); - - std::vector> containers; - rService->ListContainers(request->has_filter() ? &request->filter() : nullptr, &containers, error); -@@ -199,7 +199,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont - *container = *(iter->get()); - } - -- WARN("Event: {Object: CRI, Type: Listed all Container}"); -+ INFO("Event: {Object: CRI, Type: Listed all Container}"); - - return grpc::Status::OK; - } -@@ -210,7 +210,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Listing all Container stats}"); -+ INFO("Event: {Object: CRI, Type: Listing all Container stats}"); - - std::vector> containers; - rService->ListContainerStats(request->has_filter() ? &request->filter() : nullptr, &containers, error); -@@ -228,7 +228,28 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext * - *container = *(iter->get()); - } - -- WARN("Event: {Object: CRI, Type: Listed all Container stats}"); -+ INFO("Event: {Object: CRI, Type: Listed all Container stats}"); -+ -+ return grpc::Status::OK; -+} -+ -+grpc::Status RuntimeRuntimeServiceImpl::ContainerStats(grpc::ServerContext *context, -+ const runtime::v1alpha2::ContainerStatsRequest *request, -+ runtime::v1alpha2::ContainerStatsResponse *reply) -+{ -+ Errors error; -+ -+ INFO("Event: {Object: CRI, Type: Getting Container Stats: %s}", request->container_id().c_str()); -+ -+ std::unique_ptr contStats = -+ rService->ContainerStats(request->container_id(), error); -+ if (!error.Empty() || !contStats) { -+ ERROR("Object: CRI, Type: Failed to get container stats %s", request->container_id().c_str()); -+ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); -+ } -+ *(reply->mutable_stats()) = *contStats; -+ -+ INFO("Event: {Object: CRI, Type: Got Container stats: %s}", request->container_id().c_str()); - - return grpc::Status::OK; - } -@@ -239,7 +260,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); -+ INFO("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str()); - - std::unique_ptr contStatus = - rService->ContainerStatus(request->container_id(), error); -@@ -249,7 +270,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con - } - *(reply->mutable_status()) = *contStatus; - -- WARN("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); -+ INFO("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str()); - - return grpc::Status::OK; - } -@@ -339,7 +360,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); -+ INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); - - std::unique_ptr podStatus; - podStatus = rService->PodSandboxStatus(request->pod_sandbox_id(), error); -@@ -350,7 +371,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co - } - *(reply->mutable_status()) = *podStatus; - -- WARN("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); -+ INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); - - return grpc::Status::OK; - } -@@ -361,7 +382,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Listing all Pods}"); -+ INFO("Event: {Object: CRI, Type: Listing all Pods}"); - - std::vector> pods; - rService->ListPodSandbox(request->has_filter() ? &request->filter() : nullptr, &pods, error); -@@ -378,7 +399,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont - *pod = *(iter->get()); - } - -- WARN("Event: {Object: CRI, Type: Listed all Pods}"); -+ INFO("Event: {Object: CRI, Type: Listed all Pods}"); - - return grpc::Status::OK; - } -@@ -470,7 +491,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, - { - Errors error; - -- WARN("Event: {Object: CRI, Type: Statusing daemon}"); -+ INFO("Event: {Object: CRI, Type: Statusing daemon}"); - - std::unique_ptr status = rService->Status(error); - if (status == nullptr || error.NotEmpty()) { -@@ -479,7 +500,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context, - } - *(reply->mutable_status()) = *status; - -- WARN("Event: {Object: CRI, Type: Statused daemon}"); -+ INFO("Event: {Object: CRI, Type: Statused daemon}"); - - return grpc::Status::OK; - } -diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.h b/src/daemon/entry/connect/grpc/runtime_runtime_service.h -index f6e1634e..94543793 100644 ---- a/src/daemon/entry/connect/grpc/runtime_runtime_service.h -+++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.h -@@ -58,6 +58,9 @@ public: - const runtime::v1alpha2::ListContainerStatsRequest *request, - runtime::v1alpha2::ListContainerStatsResponse *reply) override; - -+ grpc::Status ContainerStats(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatsRequest *request, -+ runtime::v1alpha2::ContainerStatsResponse *reply) override; -+ - grpc::Status ContainerStatus(grpc::ServerContext *context, - const runtime::v1alpha2::ContainerStatusRequest *request, - runtime::v1alpha2::ContainerStatusResponse *reply) override; -diff --git a/src/daemon/entry/cri/cri_container_manager_service.h b/src/daemon/entry/cri/cri_container_manager_service.h -index f9090971..f362f798 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service.h -+++ b/src/daemon/entry/cri/cri_container_manager_service.h -@@ -46,6 +46,9 @@ public: - std::vector> *containerstats, - Errors &error) = 0; - -+ virtual auto ContainerStats(const std::string &containerID, -+ Errors &error) -> std::unique_ptr = 0; -+ - virtual auto ContainerStatus(const std::string &containerID, - Errors &error) -> std::unique_ptr = 0; - -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index b160ce31..93b939c9 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -891,6 +891,60 @@ cleanup: - free_container_stats_response(response); - } - -+auto ContainerManagerServiceImpl::ContainerStats(const std::string &containerID, Errors &error) -+-> std::unique_ptr -+{ -+ container_stats_request *request { nullptr }; -+ container_stats_response *response { nullptr }; -+ std::unique_ptr contStats { nullptr }; -+ std::vector> contStatsVec; -+ -+ if (containerID.empty()) { -+ error.SetError("Empty container id"); -+ return nullptr; -+ } -+ -+ if (m_cb == nullptr || m_cb->container.stats == nullptr) { -+ error.SetError("Unimplemented callback"); -+ return nullptr; -+ } -+ -+ request = (container_stats_request *)util_common_calloc_s(sizeof(container_stats_request)); -+ if (request == nullptr) { -+ error.SetError("Out of memory"); -+ return nullptr; -+ } -+ -+ request->containers = (char **)util_smart_calloc_s(sizeof(char *), 1); -+ if (request->containers == nullptr) { -+ error.SetError("Out of memory"); -+ goto cleanup; -+ } -+ -+ request->containers[0] = util_strdup_s(containerID.c_str()); -+ request->containers_len = 1; -+ -+ if (m_cb->container.stats(request, &response) != 0) { -+ if (response != nullptr && response->errmsg != nullptr) { -+ error.SetError(response->errmsg); -+ } else { -+ error.SetError("Failed to call stats container callback"); -+ } -+ goto cleanup; -+ } -+ -+ ContainerStatsToGRPC(response, &contStatsVec, error); -+ if (error.NotEmpty()) { -+ goto cleanup; -+ } -+ contStats = std::move(contStatsVec[0]); -+ -+cleanup: -+ free_container_stats_request(request); -+ free_container_stats_response(response); -+ return contStats; -+} -+ - void ContainerManagerServiceImpl::PackContainerImageToStatus( - container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) - { -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.h b/src/daemon/entry/cri/cri_container_manager_service_impl.h -index 805ef100..6ac1cbc4 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.h -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.h -@@ -56,6 +56,9 @@ public: - std::vector> *containerstats, - Errors &error) override; - -+ auto ContainerStats(const std::string &containerID, -+ Errors &error) -> std::unique_ptr override; -+ - auto ContainerStatus(const std::string &containerID, - Errors &error) -> std::unique_ptr override; - -diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h -index 4727230f..5e4740cb 100644 ---- a/src/daemon/entry/cri/cri_runtime_service.h -+++ b/src/daemon/entry/cri/cri_runtime_service.h -@@ -49,6 +49,9 @@ public: - std::vector> *containerstats, - Errors &error) = 0; - -+ virtual auto ContainerStats(const std::string &containerID, -+ Errors &error) -> std::unique_ptr = 0; -+ - virtual auto ContainerStatus(const std::string &containerID, - Errors &error) -> std::unique_ptr = 0; - -diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.cc b/src/daemon/entry/cri/cri_runtime_service_impl.cc -index 46643a05..e00bc6e2 100644 ---- a/src/daemon/entry/cri/cri_runtime_service_impl.cc -+++ b/src/daemon/entry/cri/cri_runtime_service_impl.cc -@@ -60,6 +60,12 @@ void CRIRuntimeServiceImpl::ListContainerStats(const runtime::v1alpha2::Containe - m_containerManager->ListContainerStats(filter, containerstats, error); - } - -+auto CRIRuntimeServiceImpl::ContainerStats(const std::string &containerID, -+ Errors &error) -> std::unique_ptr -+{ -+ return m_containerManager->ContainerStats(containerID, error); -+} -+ - auto CRIRuntimeServiceImpl::ContainerStatus(const std::string &containerID, - Errors &error) -> std::unique_ptr - { -diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.h b/src/daemon/entry/cri/cri_runtime_service_impl.h -index 13d92aba..95f1424f 100644 ---- a/src/daemon/entry/cri/cri_runtime_service_impl.h -+++ b/src/daemon/entry/cri/cri_runtime_service_impl.h -@@ -58,6 +58,9 @@ public: - std::vector> *containerstats, - Errors &error) override; - -+ auto ContainerStats(const std::string &containerID, -+ Errors &error) -> std::unique_ptr override; -+ - auto ContainerStatus(const std::string &containerID, - Errors &error) -> std::unique_ptr override; - -diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c -index 433cb88b..b3c7622a 100644 ---- a/src/daemon/modules/events/collector.c -+++ b/src/daemon/modules/events/collector.c -@@ -484,7 +484,7 @@ static int write_events_log(const struct isulad_events_format *events) - - len = calculate_annaotation_info_len(events); - if (len == 1) { -- WARN("Event: {Object: %s, Type: %s}", events->id, events->opt); -+ INFO("Event: {Object: %s, Type: %s}", events->id, events->opt); - } else { - annotation = (char *)util_common_calloc_s(len); - if (annotation == NULL) { -@@ -502,7 +502,7 @@ static int write_events_log(const struct isulad_events_format *events) - } - (void)strcat(annotation, ")"); - -- WARN("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); -+ INFO("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation); - } - - out: -diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c -index 158b3aa8..923e72b2 100644 ---- a/src/daemon/modules/image/image.c -+++ b/src/daemon/modules/image/image.c -@@ -400,7 +400,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons - goto out; - } - -- WARN("Event: {Object: get image filesystem info, Type: inspecting}"); -+ INFO("Event: {Object: get image filesystem info, Type: inspecting}"); - ret = q->ops->get_filesystem_info(response); - if (ret != 0) { - if (response != NULL && *response != NULL) { -@@ -410,7 +410,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons - } - goto out; - } -- WARN("Event: {Object: get image filesystem info, Type: inspected}"); -+ INFO("Event: {Object: get image filesystem info, Type: inspected}"); - - out: - return ret; -@@ -450,7 +450,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im - request->name_id = util_strdup_s(id); - } - -- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); -+ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : ""); - ret = q->ops->container_fs_usage(request, &filesystemusage); - if (ret != 0) { - ERROR("Failed to get filesystem usage for container %s", id); -@@ -460,7 +460,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im - - *fs_usage = filesystemusage; - filesystemusage = NULL; -- WARN("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); -+ INFO("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : ""); - - out: - free_im_container_fs_usage_request(request); -@@ -900,7 +900,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) - return -1; - } - -- WARN("Event: {Object: list images, Type: listing}"); -+ INFO("Event: {Object: list images, Type: listing}"); - - for (i = 0; i < g_numbims; i++) { - if (g_bims[i].ops->list_ims == NULL) { -@@ -920,7 +920,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response) - images_tmp = NULL; - } - -- WARN("Event: {Object: list images, Type: listed}"); -+ INFO("Event: {Object: list images, Type: listed}"); - - if (g_isulad_errmsg != NULL) { - (*response)->errmsg = util_strdup_s(g_isulad_errmsg); -@@ -1655,7 +1655,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re - - image_ref = util_strdup_s(request->image.image); - -- WARN("Event: {Object: %s, Type: image inspecting}", image_ref); -+ INFO("Event: {Object: %s, Type: image inspecting}", image_ref); - - bim_type = bim_query(image_ref); - if (bim_type == NULL) { -@@ -1677,7 +1677,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re - goto pack_response; - } - -- WARN("Event: {Object: %s, Type: image inspected}", image_ref); -+ INFO("Event: {Object: %s, Type: image inspected}", image_ref); - - pack_response: - if (g_isulad_errmsg != NULL) { -diff --git a/src/daemon/modules/image/oci/oci_common_operators.c b/src/daemon/modules/image/oci/oci_common_operators.c -index 7be1ecaa..d33c3585 100644 ---- a/src/daemon/modules/image/oci/oci_common_operators.c -+++ b/src/daemon/modules/image/oci/oci_common_operators.c -@@ -431,7 +431,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response - goto pack_response; - } - -- WARN("Event: {Object: %s, Type: statusing image summary}", resolved_name); -+ INFO("Event: {Object: %s, Type: statusing image summary}", resolved_name); - - image_summary = storage_img_get_summary(resolved_name); - if (image_summary == NULL) { -@@ -444,7 +444,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response - response->image_summary = image_summary; - image_summary = NULL; - -- WARN("Event: {Object: %s, Type: statused image summary}", resolved_name); -+ INFO("Event: {Object: %s, Type: statused image summary}", resolved_name); - - pack_response: - free(resolved_name); -@@ -495,7 +495,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) - goto pack_response; - } - -- WARN("Event: {Object: %s, Type: statusing image}", resolved_name); -+ INFO("Event: {Object: %s, Type: statusing image}", resolved_name); - - image_info = storage_img_get(resolved_name); - if (image_info == NULL) { -@@ -508,7 +508,7 @@ int oci_status_image(im_status_request *request, im_status_response *response) - response->image_info->image = image_info; - image_info = NULL; - -- WARN("Event: {Object: %s, Type: statused image}", resolved_name); -+ INFO("Event: {Object: %s, Type: statused image}", resolved_name); - - pack_response: - free(resolved_name); --- -2.25.1 - diff --git a/0014-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch b/0014-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch deleted file mode 100644 index 112463d..0000000 --- a/0014-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch +++ /dev/null @@ -1,444 +0,0 @@ -From 7890d218a8f19355630f859936c22ef43f91b178 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Thu, 16 Feb 2023 15:26:10 +0800 -Subject: [PATCH 14/19] fix selinux_label_ut timeout and add timeout for all ut - -Signed-off-by: zhangxiaoyu ---- - test/buffer/CMakeLists.txt | 1 + - test/cgroup/cpu/CMakeLists.txt | 1 + - test/cmd/isula/extend/pause/CMakeLists.txt | 1 + - test/cmd/isula/extend/resume/CMakeLists.txt | 2 +- - test/cmd/isula/information/info/CMakeLists.txt | 1 + - test/cmd/isula/information/ps/CMakeLists.txt | 1 + - test/cmd/isulad-shim/CMakeLists.txt | 1 + - test/console/CMakeLists.txt | 3 +-- - test/cutils/mainloop/CMakeLists.txt | 1 + - test/cutils/map/CMakeLists.txt | 1 + - test/cutils/path/CMakeLists.txt | 1 + - test/cutils/util_atomic/CMakeLists.txt | 1 + - test/cutils/utils_aes/CMakeLists.txt | 1 + - test/cutils/utils_array/CMakeLists.txt | 1 + - test/cutils/utils_base64/CMakeLists.txt | 1 + - test/cutils/utils_convert/CMakeLists.txt | 1 + - test/cutils/utils_error/CMakeLists.txt | 1 + - test/cutils/utils_file/CMakeLists.txt | 1 + - test/cutils/utils_filters/CMakeLists.txt | 1 + - test/cutils/utils_fs/CMakeLists.txt | 1 + - test/cutils/utils_mount_spec/CMakeLists.txt | 1 + - test/cutils/utils_namespace/CMakeLists.txt | 1 + - test/cutils/utils_network/CMakeLists.txt | 1 + - test/cutils/utils_pwgr/CMakeLists.txt | 1 + - test/cutils/utils_regex/CMakeLists.txt | 1 + - test/cutils/utils_string/CMakeLists.txt | 1 + - test/cutils/utils_timestamp/CMakeLists.txt | 1 + - test/cutils/utils_utils/CMakeLists.txt | 1 + - test/cutils/utils_verify/CMakeLists.txt | 1 + - test/image/oci/oci_config_merge/CMakeLists.txt | 1 + - test/image/oci/registry/CMakeLists.txt | 1 + - test/image/oci/storage/images/CMakeLists.txt | 1 + - test/image/oci/storage/layers/CMakeLists.txt | 2 ++ - test/image/oci/storage/rootfs/CMakeLists.txt | 1 + - test/runtime/isula/CMakeLists.txt | 1 + - test/runtime/lcr/CMakeLists.txt | 1 + - .../services/execution/execute/execution_extend/CMakeLists.txt | 1 + - test/services/execution/spec/CMakeLists.txt | 2 ++ - test/services/execution/spec/selinux_label_ut.cc | 2 ++ - test/specs/specs/CMakeLists.txt | 1 + - test/specs/specs_extend/CMakeLists.txt | 1 + - 41 files changed, 44 insertions(+), 3 deletions(-) - -diff --git a/test/buffer/CMakeLists.txt b/test/buffer/CMakeLists.txt -index f900b592..1e94cc02 100644 ---- a/test/buffer/CMakeLists.txt -+++ b/test/buffer/CMakeLists.txt -@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt -index 159b0d85..6a8af719 100644 ---- a/test/cgroup/cpu/CMakeLists.txt -+++ b/test/cgroup/cpu/CMakeLists.txt -@@ -26,3 +26,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cmd/isula/extend/pause/CMakeLists.txt b/test/cmd/isula/extend/pause/CMakeLists.txt -index af48c23f..6eab5172 100644 ---- a/test/cmd/isula/extend/pause/CMakeLists.txt -+++ b/test/cmd/isula/extend/pause/CMakeLists.txt -@@ -47,3 +47,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cmd/isula/extend/resume/CMakeLists.txt b/test/cmd/isula/extend/resume/CMakeLists.txt -index 729c4d3b..0812a66e 100644 ---- a/test/cmd/isula/extend/resume/CMakeLists.txt -+++ b/test/cmd/isula/extend/resume/CMakeLists.txt -@@ -48,4 +48,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -- -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cmd/isula/information/info/CMakeLists.txt b/test/cmd/isula/information/info/CMakeLists.txt -index 2f134986..d8d8ddb7 100644 ---- a/test/cmd/isula/information/info/CMakeLists.txt -+++ b/test/cmd/isula/information/info/CMakeLists.txt -@@ -49,3 +49,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cmd/isula/information/ps/CMakeLists.txt b/test/cmd/isula/information/ps/CMakeLists.txt -index 9659808a..ca04c25c 100644 ---- a/test/cmd/isula/information/ps/CMakeLists.txt -+++ b/test/cmd/isula/information/ps/CMakeLists.txt -@@ -49,3 +49,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt -index 1c2de232..dc293f6d 100644 ---- a/test/cmd/isulad-shim/CMakeLists.txt -+++ b/test/cmd/isulad-shim/CMakeLists.txt -@@ -30,3 +30,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/console/CMakeLists.txt b/test/console/CMakeLists.txt -index acadc620..c6f2419c 100644 ---- a/test/console/CMakeLists.txt -+++ b/test/console/CMakeLists.txt -@@ -16,5 +16,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -- -- -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/mainloop/CMakeLists.txt b/test/cutils/mainloop/CMakeLists.txt -index 78e3f18d..61ea7ffc 100644 ---- a/test/cutils/mainloop/CMakeLists.txt -+++ b/test/cutils/mainloop/CMakeLists.txt -@@ -25,3 +25,4 @@ target_link_libraries(${EXE} - libutils_ut -lcrypto -lyajl -lz) - - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/map/CMakeLists.txt b/test/cutils/map/CMakeLists.txt -index 4059559f..bd21ee3f 100644 ---- a/test/cutils/map/CMakeLists.txt -+++ b/test/cutils/map/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/path/CMakeLists.txt b/test/cutils/path/CMakeLists.txt -index 745258a1..aa0c6c6e 100644 ---- a/test/cutils/path/CMakeLists.txt -+++ b/test/cutils/path/CMakeLists.txt -@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC - set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,getcwd -Wl,--wrap,readlink") - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/util_atomic/CMakeLists.txt b/test/cutils/util_atomic/CMakeLists.txt -index 071b2a04..21b05c46 100644 ---- a/test/cutils/util_atomic/CMakeLists.txt -+++ b/test/cutils/util_atomic/CMakeLists.txt -@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_aes/CMakeLists.txt b/test/cutils/utils_aes/CMakeLists.txt -index f7535bb3..c4d175c8 100644 ---- a/test/cutils/utils_aes/CMakeLists.txt -+++ b/test/cutils/utils_aes/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_array/CMakeLists.txt b/test/cutils/utils_array/CMakeLists.txt -index 71733e31..afe564bf 100644 ---- a/test/cutils/utils_array/CMakeLists.txt -+++ b/test/cutils/utils_array/CMakeLists.txt -@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC - set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,calloc") - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_base64/CMakeLists.txt b/test/cutils/utils_base64/CMakeLists.txt -index d5b99361..35e6ba04 100644 ---- a/test/cutils/utils_base64/CMakeLists.txt -+++ b/test/cutils/utils_base64/CMakeLists.txt -@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_convert/CMakeLists.txt b/test/cutils/utils_convert/CMakeLists.txt -index 30068208..fcf70a7c 100644 ---- a/test/cutils/utils_convert/CMakeLists.txt -+++ b/test/cutils/utils_convert/CMakeLists.txt -@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_error/CMakeLists.txt b/test/cutils/utils_error/CMakeLists.txt -index 28016605..9607aeeb 100644 ---- a/test/cutils/utils_error/CMakeLists.txt -+++ b/test/cutils/utils_error/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_file/CMakeLists.txt b/test/cutils/utils_file/CMakeLists.txt -index 20317e15..01b2ff47 100644 ---- a/test/cutils/utils_file/CMakeLists.txt -+++ b/test/cutils/utils_file/CMakeLists.txt -@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_filters/CMakeLists.txt b/test/cutils/utils_filters/CMakeLists.txt -index 31d3ac25..561ff4ef 100644 ---- a/test/cutils/utils_filters/CMakeLists.txt -+++ b/test/cutils/utils_filters/CMakeLists.txt -@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_fs/CMakeLists.txt b/test/cutils/utils_fs/CMakeLists.txt -index 7ff3176a..e909aee6 100644 ---- a/test/cutils/utils_fs/CMakeLists.txt -+++ b/test/cutils/utils_fs/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_mount_spec/CMakeLists.txt b/test/cutils/utils_mount_spec/CMakeLists.txt -index 24fb5add..099805d2 100644 ---- a/test/cutils/utils_mount_spec/CMakeLists.txt -+++ b/test/cutils/utils_mount_spec/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_namespace/CMakeLists.txt b/test/cutils/utils_namespace/CMakeLists.txt -index 8add4a71..fb535e71 100644 ---- a/test/cutils/utils_namespace/CMakeLists.txt -+++ b/test/cutils/utils_namespace/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_network/CMakeLists.txt b/test/cutils/utils_network/CMakeLists.txt -index 7e2c84e7..d08a6fbd 100644 ---- a/test/cutils/utils_network/CMakeLists.txt -+++ b/test/cutils/utils_network/CMakeLists.txt -@@ -25,3 +25,4 @@ target_link_libraries(${EXE} - libutils_ut -lcrypto -lyajl -lz) - - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_pwgr/CMakeLists.txt b/test/cutils/utils_pwgr/CMakeLists.txt -index 5938991e..34acb92a 100644 ---- a/test/cutils/utils_pwgr/CMakeLists.txt -+++ b/test/cutils/utils_pwgr/CMakeLists.txt -@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_regex/CMakeLists.txt b/test/cutils/utils_regex/CMakeLists.txt -index 3f6410b2..45ba2604 100644 ---- a/test/cutils/utils_regex/CMakeLists.txt -+++ b/test/cutils/utils_regex/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_string/CMakeLists.txt b/test/cutils/utils_string/CMakeLists.txt -index 1343f4e6..b9968c43 100644 ---- a/test/cutils/utils_string/CMakeLists.txt -+++ b/test/cutils/utils_string/CMakeLists.txt -@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC - set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_strdup_s -Wl,--wrap,calloc -Wl,--wrap,strcat_s") - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_timestamp/CMakeLists.txt b/test/cutils/utils_timestamp/CMakeLists.txt -index 38aec640..6da3fcc9 100644 ---- a/test/cutils/utils_timestamp/CMakeLists.txt -+++ b/test/cutils/utils_timestamp/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt -index 7b3bd546..99a83e7a 100644 ---- a/test/cutils/utils_utils/CMakeLists.txt -+++ b/test/cutils/utils_utils/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/cutils/utils_verify/CMakeLists.txt b/test/cutils/utils_verify/CMakeLists.txt -index abf9596f..dd9ef78b 100644 ---- a/test/cutils/utils_verify/CMakeLists.txt -+++ b/test/cutils/utils_verify/CMakeLists.txt -@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt -index 88047fde..42cd2e78 100644 ---- a/test/image/oci/oci_config_merge/CMakeLists.txt -+++ b/test/image/oci/oci_config_merge/CMakeLists.txt -@@ -73,3 +73,4 @@ target_include_directories(${EXE} PUBLIC - set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,merge_env") - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt -index 9ff075db..31240ae0 100644 ---- a/test/image/oci/registry/CMakeLists.txt -+++ b/test/image/oci/registry/CMakeLists.txt -@@ -62,3 +62,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz libhttpclient) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt -index b00c5a0e..3e6b69a4 100644 ---- a/test/image/oci/storage/images/CMakeLists.txt -+++ b/test/image/oci/storage/images/CMakeLists.txt -@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt -index 3fe8ab7c..952e9483 100644 ---- a/test/image/oci/storage/layers/CMakeLists.txt -+++ b/test/image/oci/storage/layers/CMakeLists.txt -@@ -66,6 +66,7 @@ target_link_libraries(${DRIVER_EXE} - -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) - - add_test(NAME ${DRIVER_EXE} COMMAND ${DRIVER_EXE} --gtest_output=xml:${DRIVER_EXE}-Results.xml) -+set_tests_properties(${DRIVER_EXE} PROPERTIES TIMEOUT 120) - - # storage_layers_ut - SET(LAYER_EXE storage_layers_ut) -@@ -141,3 +142,4 @@ target_link_libraries(${LAYER_EXE} - -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz) - - add_test(NAME ${LAYER_EXE} COMMAND ${LAYER_EXE} --gtest_output=xml:${LAYER_EXE}-Results.xml) -+set_tests_properties(${LAYER_EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt -index 5bf568f9..4d7d3533 100644 ---- a/test/image/oci/storage/rootfs/CMakeLists.txt -+++ b/test/image/oci/storage/rootfs/CMakeLists.txt -@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt -index 38a454b0..f5821953 100644 ---- a/test/runtime/isula/CMakeLists.txt -+++ b/test/runtime/isula/CMakeLists.txt -@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lpthread -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt -index 6f8f784c..979cbe5a 100644 ---- a/test/runtime/lcr/CMakeLists.txt -+++ b/test/runtime/lcr/CMakeLists.txt -@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt -index aaff39ef..8588fac3 100644 ---- a/test/services/execution/execute/execution_extend/CMakeLists.txt -+++ b/test/services/execution/execute/execution_extend/CMakeLists.txt -@@ -76,3 +76,4 @@ target_include_directories(${EXE} PUBLIC - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/services/execution/spec/CMakeLists.txt b/test/services/execution/spec/CMakeLists.txt -index e1aa680e..d4a9d9b0 100644 ---- a/test/services/execution/spec/CMakeLists.txt -+++ b/test/services/execution/spec/CMakeLists.txt -@@ -74,4 +74,6 @@ target_include_directories(${MOCK_EXE} PUBLIC - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) - target_link_libraries(${MOCK_EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) - add_test(NAME ${MOCK_EXE} COMMAND ${MOCK_EXE} --gtest_output=xml:${MOCK_EXE}-Results.xml) -+set_tests_properties(${MOCK_EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/services/execution/spec/selinux_label_ut.cc b/test/services/execution/spec/selinux_label_ut.cc -index c0e9ab1c..f9a73cca 100644 ---- a/test/services/execution/spec/selinux_label_ut.cc -+++ b/test/services/execution/spec/selinux_label_ut.cc -@@ -179,11 +179,13 @@ protected: - void SetUp() override - { - CreateTestedObjects(); -+ selinux_state_init(); - } - - void TearDown() override - { - ClearTestedObjects(); -+ selinux_state_free(); - } - - private: -diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt -index b730959c..c4b36c5f 100644 ---- a/test/specs/specs/CMakeLists.txt -+++ b/test/specs/specs/CMakeLists.txt -@@ -83,3 +83,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt -index 7d05deb4..06f46a37 100644 ---- a/test/specs/specs_extend/CMakeLists.txt -+++ b/test/specs/specs_extend/CMakeLists.txt -@@ -79,3 +79,4 @@ target_include_directories(${EXE} PUBLIC - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz) - add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) --- -2.25.1 - diff --git a/0015-fix-cpu-rt-disable-after-reboot-machine.patch b/0015-fix-cpu-rt-disable-after-reboot-machine.patch deleted file mode 100644 index 23cf10e..0000000 --- a/0015-fix-cpu-rt-disable-after-reboot-machine.patch +++ /dev/null @@ -1,850 +0,0 @@ -From adc75018f7eda4b205034999893634d1b98b2701 Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Thu, 16 Feb 2023 15:42:02 +0800 -Subject: [PATCH 15/19] fix cpu-rt disable after reboot machine - -1. ensure parent cgroup cpu-rt of container, should do in start container; -2. current do in create container, will cause failed of start container with cpu-rt after reboot machine - -Signed-off-by: haozi007 ---- - src/daemon/common/sysinfo.c | 48 ++++ - src/daemon/common/sysinfo.h | 2 + - src/daemon/executor/container_cb/execution.c | 177 +++++++++++++- - .../executor/container_cb/execution_create.c | 221 +----------------- - src/daemon/modules/api/specs_api.h | 2 +- - src/daemon/modules/spec/specs.c | 88 ++++--- - test/specs/specs/specs_ut.cc | 60 +++-- - 7 files changed, 306 insertions(+), 292 deletions(-) - -diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c -index e7b3807b..8ad92b2a 100644 ---- a/src/daemon/common/sysinfo.c -+++ b/src/daemon/common/sysinfo.c -@@ -1627,3 +1627,51 @@ free_out: - } - return minfos; - } -+ -+char *sysinfo_cgroup_controller_cpurt_mnt_path(void) -+{ -+ char *res = NULL; -+ int nret = 0; -+ char *mnt = NULL; -+ char *root = NULL; -+ char fpath[PATH_MAX] = { 0 }; -+ sysinfo_t *sysinfo = NULL; -+ -+ sysinfo = get_sys_info(true); -+ if (sysinfo == NULL) { -+ ERROR("Can not get system info"); -+ return NULL; -+ } -+ -+ if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { -+ ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); -+ isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); -+ return NULL; -+ } -+ -+ nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); -+ if (nret != 0 || mnt == NULL || root == NULL) { -+ ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); -+ isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); -+ goto out; -+ } -+ -+ // When iSulad is run inside docker, the root is based of the host cgroup. -+ // Replace root to "/" -+ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { -+ root[1] = '\0'; -+ } -+ -+ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); -+ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { -+ ERROR("Failed to print string"); -+ goto out; -+ } -+ -+ res = util_strdup_s(fpath); -+ -+out: -+ free(mnt); -+ free(root); -+ return res; -+} -\ No newline at end of file -diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h -index 8468e00a..027cb1de 100644 ---- a/src/daemon/common/sysinfo.h -+++ b/src/daemon/common/sysinfo.h -@@ -139,6 +139,8 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir); - - void free_mounts_info(mountinfo_t **minfos); - -+char *sysinfo_cgroup_controller_cpurt_mnt_path(void); -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c -index 68d0d8d6..4dd5f858 100644 ---- a/src/daemon/executor/container_cb/execution.c -+++ b/src/daemon/executor/container_cb/execution.c -@@ -17,7 +17,13 @@ - #include - #include - #include -+#include -+#include -+#include -+#include -+#include - #include -+ - #include - #include - #include -@@ -34,12 +40,11 @@ - #include - #include - #include --#include --#include --#include --#include -+#include - --#include "isula_libutils/log.h" -+#include "sysinfo.h" -+#include "isulad_config.h" -+#include "specs_api.h" - #include "container_api.h" - #include "execution_extend.h" - #include "execution_information.h" -@@ -297,6 +302,155 @@ static void pack_start_response(container_start_response *response, uint32_t cc, - } - } - -+static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, -+ int64_t cpu_rt_period, int64_t cpu_rt_runtime); -+ -+/* maybe create cpu realtime file */ -+static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) -+{ -+ int ret; -+ int fd = -1; -+ ssize_t nwrite; -+ char fpath[PATH_MAX] = { 0 }; -+ char buf[ISULAD_NUMSTRLEN64] = { 0 }; -+ -+ if (value == 0) { -+ return 0; -+ } -+ -+ ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); -+ if (ret != 0) { -+ ERROR("Failed to mkdir: %s", path); -+ return -1; -+ } -+ -+ ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); -+ if (ret < 0 || ret >= sizeof(fpath)) { -+ ERROR("Failed to print string"); -+ return -1; -+ } -+ ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); -+ if (ret < 0 || (size_t)ret >= sizeof(buf)) { -+ ERROR("Failed to print string"); -+ return -1; -+ } -+ -+ fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); -+ if (fd < 0) { -+ ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); -+ isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); -+ return -1; -+ } -+ nwrite = util_write_nointr(fd, buf, strlen(buf)); -+ if (nwrite < 0 || nwrite != strlen(buf)) { -+ ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); -+ isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); -+ close(fd); -+ return -1; -+ } -+ close(fd); -+ -+ return 0; -+} -+ -+static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, -+ int64_t cpu_rt_runtime) -+{ -+ int nret; -+ char *dup = NULL; -+ char *dirpath = NULL; -+ char fpath[PATH_MAX] = { 0 }; -+ -+ dup = util_strdup_s(path); -+ dirpath = dirname(dup); -+ nret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); -+ free(dup); -+ if (nret != 0) { -+ return -1; -+ } -+ -+ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); -+ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { -+ ERROR("Failed to print string"); -+ return -1; -+ } -+ -+ nret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); -+ if (nret != 0) { -+ return -1; -+ } -+ -+ return maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); -+} -+ -+/* init cgroups path */ -+static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, -+ int64_t cpu_rt_period, int64_t cpu_rt_runtime) -+{ -+ if ((recursive_depth + 1) > MAX_PATH_DEPTH) { -+ ERROR("Reach the max cgroup depth:%s", path); -+ return -1; -+ } -+ -+ if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { -+ return 0; -+ } -+ -+ // Recursively create cgroup to ensure that the system and all parent cgroups have values set -+ // for the period and runtime as this limits what the children can be set to. -+ if (recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime)) { -+ return -1; -+ } -+ -+ return 0; -+} -+ -+// TODO: maybe we should adapt to cgroup v2 -+static int cpurt_controller_init(const char *id, const host_config *host_spec) -+{ -+ int ret = 0; -+ int64_t cpu_rt_period = 0; -+ int64_t cpu_rt_runtime = 0; -+ char *dirpath = NULL; -+ char *mnt_root = NULL; -+ char *cgroups_path = NULL; -+ -+ if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { -+ return -1; -+ } -+ -+ if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { -+ return 0; -+ } -+ -+ cgroups_path = merge_container_cgroups_path(id, host_spec); -+ if (cgroups_path == NULL) { -+ ERROR("Merge cgroups path for container failed"); -+ ret = -1; -+ goto out; -+ } -+ if (strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { -+ goto out; -+ } -+ -+ mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path(); -+ if (mnt_root == NULL) { -+ ERROR("Failed to get cpu rt controller mnt root path"); -+ isulad_set_error_message("Failed to get cpu rt controller mnt root path"); -+ ret = -1; -+ goto out; -+ } -+ -+ dirpath = dirname(cgroups_path); -+ -+ ret = do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); -+ -+out: -+ free(mnt_root); -+ free(cgroups_path); -+ return ret; -+} -+ - static int container_start_prepare(container_t *cont, const container_start_request *request, int stdinfd, - struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler, - char **fifopath, char *fifos[], int *sync_fd, pthread_t *thread_id) -@@ -309,6 +463,19 @@ static int container_start_prepare(container_t *cont, const container_start_requ - return -1; - } - -+ // init cgroup path for cpu_rt_runtime and cpu_rt_period -+ // we should do this in start container, not create container -+ // because it will fail after the following steps: -+ // 1. enable cpu-rt of isulad; -+ // 2. then run container with --cpu-rt-runtime -+ // 3. then reboot machine; -+ // 4. finally, start before container, it will failed... -+ // cause of no one to set value into cgroup/isulad/cpu-rt-runtime and cpu-rt-period. -+ if (cpurt_controller_init(id, cont->hostconfig) != 0) { -+ isulad_set_error_message("Failed to init controller of cpu-rt for container \"%s\".", id); -+ return -1; -+ } -+ - if (prepare_start_io(cont, request, fifopath, fifos, stdinfd, stdout_handler, stderr_handler, sync_fd, thread_id) != - 0) { - return -1; -diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c -index 0d03afcb..a039ba6b 100644 ---- a/src/daemon/executor/container_cb/execution_create.c -+++ b/src/daemon/executor/container_cb/execution_create.c -@@ -19,6 +19,13 @@ - #include - #include - #include -+#include -+#include -+#include -+#include -+#include -+ -+#include - #include - #include - #include -@@ -28,14 +35,7 @@ - #include - #include - #include --#include --#include --#include --#include --#include --#include - --#include "isula_libutils/log.h" - #include "isulad_config.h" - #include "specs_api.h" - #include "verify.h" -@@ -61,9 +61,6 @@ - #include "opt_log.h" - #include "network_namespace_api.h" - --static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, -- int64_t cpu_rt_period, int64_t cpu_rt_runtime); -- - static int runtime_check(const char *name, bool *runtime_res) - { - int ret = 0; -@@ -1291,148 +1288,6 @@ static int save_container_config_before_create(const char *id, const char *runti - return 0; - } - --/* maybe create cpu realtime file */ --static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path) --{ -- int ret; -- int fd = -1; -- ssize_t nwrite; -- char fpath[PATH_MAX] = { 0 }; -- char buf[ISULAD_NUMSTRLEN64] = { 0 }; -- -- if (value == 0) { -- return 0; -- } -- -- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); -- if (ret != 0) { -- ERROR("Failed to mkdir: %s", path); -- return -1; -- } -- -- ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file); -- if (ret < 0 || ret >= sizeof(fpath)) { -- ERROR("Failed to print string"); -- return -1; -- } -- ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value); -- if (ret < 0 || (size_t)ret >= sizeof(buf)) { -- ERROR("Failed to print string"); -- return -1; -- } -- -- fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700); -- if (fd < 0) { -- ERROR("Failed to open file: %s: %s", fpath, strerror(errno)); -- isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno)); -- return -1; -- } -- nwrite = util_write_nointr(fd, buf, strlen(buf)); -- if (nwrite < 0) { -- ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno)); -- isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno)); -- close(fd); -- return -1; -- } -- close(fd); -- -- return 0; --} -- --static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period, -- int64_t cpu_rt_runtime) --{ -- int ret = 0; -- char *dup = NULL; -- char *dirpath = NULL; -- char fpath[PATH_MAX] = { 0 }; -- -- dup = util_strdup_s(path); -- dirpath = dirname(dup); -- ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime); -- free(dup); -- if (ret != 0) { -- return ret; -- } -- -- int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path); -- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { -- ERROR("Failed to print string"); -- ret = -1; -- goto out; -- } -- -- ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath); -- if (ret != 0) { -- goto out; -- } -- -- ret = maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath); -- if (ret != 0) { -- goto out; -- } -- --out: -- return ret; --} -- --/* init cgroups path */ --static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root, -- int64_t cpu_rt_period, int64_t cpu_rt_runtime) --{ -- if ((recursive_depth + 1) > MAX_PATH_DEPTH) { -- ERROR("Reach the max cgroup depth:%s", path); -- return -1; -- } -- -- if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) { -- return 0; -- } -- -- // Recursively create cgroup to ensure that the system and all parent cgroups have values set -- // for the period and runtime as this limits what the children can be set to. -- if (recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime)) { -- return -1; -- } -- -- return 0; --} -- --static char *get_cpurt_controller_mnt_path() --{ -- char *res = NULL; -- int nret = 0; -- char *mnt = NULL; -- char *root = NULL; -- char fpath[PATH_MAX] = { 0 }; -- -- nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root); -- if (nret != 0 || mnt == NULL || root == NULL) { -- ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'"); -- isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'"); -- goto out; -- } -- -- // When iSulad is run inside docker, the root is based of the host cgroup. -- // Replace root to "/" -- if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { -- root[1] = '\0'; -- } -- -- nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); -- if (nret < 0 || (size_t)nret >= sizeof(fpath)) { -- ERROR("Failed to print string"); -- goto out; -- } -- -- res = util_strdup_s(fpath); -- --out: -- free(mnt); -- free(root); -- return res; --} -- - static char *new_pod_sandbox_key(void) - { - int nret = 0; -@@ -1495,61 +1350,6 @@ err_out: - return -1; - } - --static int cpurt_controller_init(const char *cgroups_path) --{ -- int ret = 0; -- char *dup = NULL; -- char *dirpath = NULL; -- int64_t cpu_rt_period = 0; -- int64_t cpu_rt_runtime = 0; -- sysinfo_t *sysinfo = NULL; -- char *mnt_root = NULL; -- -- if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) { -- return 0; -- } -- -- if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) { -- return -1; -- } -- -- if (cpu_rt_period == 0 && cpu_rt_runtime == 0) { -- return 0; -- } -- -- sysinfo = get_sys_info(true); -- if (sysinfo == NULL) { -- ERROR("Can not get system info"); -- ret = -1; -- goto out; -- } -- -- if (!(sysinfo->cgcpuinfo.cpu_rt_period)) { -- ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); -- isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel"); -- ret = -1; -- goto out; -- } -- -- mnt_root = get_cpurt_controller_mnt_path(); -- if (mnt_root == NULL) { -- ERROR("Failed to get cpu rt controller mnt root path"); -- isulad_set_error_message("Failed to get cpu rt controller mnt root path"); -- ret = -1; -- goto out; -- } -- -- dup = util_strdup_s(cgroups_path); -- dirpath = dirname(dup); -- -- ret = do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime); -- --out: -- free(mnt_root); -- free(dup); -- return ret; --} -- - /* - * request -> host_spec + container_spec - * container_spec + image config -@@ -1707,13 +1507,6 @@ int container_create_cb(const container_create_request *request, container_creat - goto umount_channel; - } - -- // init cgroup path for cpu_rt_runtime and cpu_rt_period -- if (cpurt_controller_init(oci_spec->linux->cgroups_path) != 0) { -- ERROR("Unable to init CPU RT controller %s", oci_spec->linux->cgroups_path); -- cc = ISULAD_ERR_EXEC; -- goto umount_channel; -- } -- - if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) { - ERROR("Failed to merge container settings"); - cc = ISULAD_ERR_EXEC; -diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h -index c4ad79aa..0a594d81 100644 ---- a/src/daemon/modules/api/specs_api.h -+++ b/src/daemon/modules/api/specs_api.h -@@ -27,7 +27,7 @@ extern "C" { - - int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, - oci_runtime_spec *oci_spec); --int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec); -+char *merge_container_cgroups_path(const char *id, const host_config *host_spec); - int merge_global_config(oci_runtime_spec *oci_spec); - oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); - oci_runtime_spec *default_spec(bool system_container); -diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c -index 138a0989..f32ff911 100644 ---- a/src/daemon/modules/spec/specs.c -+++ b/src/daemon/modules/spec/specs.c -@@ -165,36 +165,45 @@ out: - return ret; - } - --static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) -+static char *do_get_container_cgroup_path(const host_config *host_spec) - { -- int ret = 0; -- char cleaned[PATH_MAX] = { 0 }; -- char *default_cgroup_parent = NULL; - char *path = NULL; - -- default_cgroup_parent = conf_get_isulad_cgroup_parent(); - if (host_spec->cgroup_parent != NULL) { -- path = host_spec->cgroup_parent; -- } else if (default_cgroup_parent != NULL) { -- path = default_cgroup_parent; -+ // first, use user setting -+ path = util_strdup_s(host_spec->cgroup_parent); -+ } else { -+ // second, if user donot set, use setting from daemon config -+ path = conf_get_isulad_cgroup_parent(); - } -+ - if (path == NULL) { -- path = "/isulad"; -+ // third, all faild, just use default '/isulad' -+ path = util_strdup_s("/isulad"); - } -+ -+ return path; -+} -+ -+static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec) -+{ -+ char cleaned[PATH_MAX] = { 0 }; -+ char *path = NULL; -+ -+ path = do_get_container_cgroup_path(host_spec); - if (util_clean_path(path, cleaned, sizeof(cleaned)) == NULL) { - ERROR("Failed to clean path: %s", path); -- ret = -1; -- goto out; -+ free(path); -+ return -1; - } -+ free(path); -+ - if (append_json_map_string_string(container_spec->annotations, "cgroup.dir", cleaned)) { - ERROR("Realloc annotations failed"); -- ret = -1; -- goto out; -+ return -1; - } - --out: -- free(default_cgroup_parent); -- return ret; -+ return 0; - } - - static int make_annotations_oom_score_adj(const container_config *container_spec, const host_config *host_spec) -@@ -2048,42 +2057,45 @@ out: - return ret; - } - --int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) -+char *merge_container_cgroups_path(const char *id, const host_config *host_spec) - { -- int ret = 0; -- char *default_cgroup_parent = NULL; - char *path = NULL; -+ char *ret = NULL; -+ -+ if (id == NULL || host_spec == NULL) { -+ ERROR("Invalid arguments"); -+ return NULL; -+ } -+ -+ path = do_get_container_cgroup_path(host_spec); -+ -+ ret = util_path_join(path, id); -+ free(path); -+ -+ return ret; -+} - -+static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec) -+{ - if (id == NULL || oci_spec == NULL || host_spec == NULL) { - ERROR("Invalid arguments"); -- ret = -1; -- goto out; -+ return -1; - } - - if (make_sure_oci_spec_linux(oci_spec) != 0) { - ERROR("Failed to make oci spec linux"); -- ret = -1; -- goto out; -+ return -1; - } - -- default_cgroup_parent = conf_get_isulad_cgroup_parent(); -- path = default_cgroup_parent; -- if (host_spec->cgroup_parent != NULL) { -- path = host_spec->cgroup_parent; -- } -+ free(oci_spec->linux->cgroups_path); - -- if (path == NULL) { -- free(oci_spec->linux->cgroups_path); -- oci_spec->linux->cgroups_path = util_path_join("/isulad", id); -- return 0; -+ oci_spec->linux->cgroups_path = merge_container_cgroups_path(id, host_spec); -+ if (oci_spec->linux->cgroups_path == NULL) { -+ ERROR("Failed to merge OCI spec cgroups path"); -+ return -1; - } - -- free(oci_spec->linux->cgroups_path); -- oci_spec->linux->cgroups_path = util_path_join(path, id); -- --out: -- free(default_cgroup_parent); -- return ret; -+ return 0; - } - - int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec, -diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc -index c4014e2e..a2bd3890 100644 ---- a/test/specs/specs/specs_ut.cc -+++ b/test/specs/specs/specs_ut.cc -@@ -232,41 +232,36 @@ char *invoke_conf_get_isulad_cgroup_parent() - return util_strdup_s("/var/lib/isulad/engines/lcr"); - } - --TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_1) -+TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1) - { -- ASSERT_EQ(merge_oci_cgroups_path(nullptr, nullptr, nullptr), -1); -+ ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr); - } - --TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2) -+TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2) - { -- oci_runtime_spec *oci_spec = nullptr; - host_config *host_spec = nullptr; -- -- oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); -- ASSERT_TRUE(oci_spec != nullptr); -+ char *merged_cp = nullptr; - - host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); - ASSERT_TRUE(host_spec != nullptr); - - EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); - -- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); -+ merged_cp = merge_container_cgroups_path("123", host_spec); -+ ASSERT_NE(merged_cp, nullptr); - -- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/isulad/123"); -+ ASSERT_STREQ(merged_cp, "/isulad/123"); - -- free_oci_runtime_spec(oci_spec); - free_host_config(host_spec); -+ free(merged_cp); - - testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); - } - --TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) -+TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3) - { -- oci_runtime_spec *oci_spec = nullptr; - host_config *host_spec = nullptr; -- -- oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); -- ASSERT_TRUE(oci_spec != nullptr); -+ char *merged_cp = nullptr; - - host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); - ASSERT_TRUE(host_spec != nullptr); -@@ -275,46 +270,42 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3) - - EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null)); - -- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); -+ merged_cp = merge_container_cgroups_path("123", host_spec); -+ ASSERT_NE(merged_cp, nullptr); - -- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); -+ ASSERT_STREQ(merged_cp, "/test/123"); - -- free_oci_runtime_spec(oci_spec); - free_host_config(host_spec); -+ free(merged_cp); - - testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); - } - --TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4) -+TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4) - { -- oci_runtime_spec *oci_spec = nullptr; - host_config *host_spec = nullptr; -- -- oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); -- ASSERT_TRUE(oci_spec != nullptr); -+ char *merged_cp = nullptr; - - host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); - ASSERT_TRUE(host_spec != nullptr); - - EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); - -- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); -+ merged_cp = merge_container_cgroups_path("123", host_spec); -+ ASSERT_NE(merged_cp, nullptr); - -- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/123"); -+ ASSERT_STREQ(merged_cp, "/var/lib/isulad/engines/lcr/123"); - -- free_oci_runtime_spec(oci_spec); - free_host_config(host_spec); -+ free(merged_cp); - - testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); - } - --TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) -+TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5) - { -- oci_runtime_spec *oci_spec = nullptr; - host_config *host_spec = nullptr; -- -- oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec)); -- ASSERT_TRUE(oci_spec != nullptr); -+ char *merged_cp = nullptr; - - host_spec = (host_config *)util_common_calloc_s(sizeof(host_config)); - ASSERT_TRUE(host_spec != nullptr); -@@ -323,12 +314,13 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5) - - EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent)); - -- ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0); -+ merged_cp = merge_container_cgroups_path("123", host_spec); -+ ASSERT_NE(merged_cp, nullptr); - -- ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123"); -+ ASSERT_STREQ(merged_cp, "/test/123"); - -- free_oci_runtime_spec(oci_spec); - free_host_config(host_spec); -+ free(merged_cp); - - testing::Mock::VerifyAndClearExpectations(&m_isulad_conf); - } --- -2.25.1 - diff --git a/0016-fix-code-style.patch b/0016-fix-code-style.patch deleted file mode 100644 index bfdfc15..0000000 --- a/0016-fix-code-style.patch +++ /dev/null @@ -1,681 +0,0 @@ -From 549e22a9fb7f6e2c36a4a7422223c90fa7c44bd6 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Fri, 17 Feb 2023 09:34:14 +0800 -Subject: [PATCH 16/19] fix code style - -Signed-off-by: zhangxiaoyu ---- - src/client/connect/protocol_type.c | 2 +- - .../connect/rest/rest_containers_client.c | 2 +- - src/cmd/isula/base/run.h | 2 +- - src/cmd/isula/client_console.c | 2 +- - src/cmd/isula/stream/attach.c | 2 +- - src/cmd/isulad-shim/process.c | 5 +- - .../connect/grpc/grpc_containers_service.cc | 2 +- - .../grpc/grpc_containers_service_private.cc | 8 +-- - .../cri_pod_sandbox_manager_service_impl.cc | 10 ++-- - src/daemon/executor/container_cb/execution.c | 2 +- - .../modules/image/image_rootfs_handler.c | 12 ++--- - .../modules/runtime/isula/isula_rt_ops.c | 8 +-- - src/daemon/modules/runtime/shim/shim_rt_ops.c | 2 +- - .../modules/service/service_container.c | 4 +- - src/utils/cutils/filters.c | 2 +- - src/utils/cutils/utils_mount_spec.c | 6 +-- - test/cgroup/cpu/cgroup_cpu_ut.cc | 2 +- - test/console/console_ut.cc | 4 +- - test/cutils/utils_file/utils_file_ut.cc | 50 +++++++++---------- - .../utils_mount_spec/utils_mount_spec_ut.cc | 6 ++- - test/cutils/utils_pwgr/utils_pwgr_ut.cc | 4 +- - test/cutils/utils_utils/utils_utils_ut.cc | 5 +- - test/cutils/utils_verify/utils_verify_ut.cc | 2 +- - .../oci/storage/layers/storage_driver_ut.cc | 3 +- - test/sha256/sha256_ut.cc | 2 +- - 25 files changed, 78 insertions(+), 71 deletions(-) - -diff --git a/src/client/connect/protocol_type.c b/src/client/connect/protocol_type.c -index a3aabe0e..64576d1e 100644 ---- a/src/client/connect/protocol_type.c -+++ b/src/client/connect/protocol_type.c -@@ -660,7 +660,7 @@ void isula_kill_response_free(struct isula_kill_response *response) - } - free(response->id); - response->id = NULL; -- -+ - free(response->errmsg); - response->errmsg = NULL; - free(response); -diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c -index 22aa1c9b..0a735a64 100644 ---- a/src/client/connect/rest/rest_containers_client.c -+++ b/src/client/connect/rest/rest_containers_client.c -@@ -1333,7 +1333,7 @@ static int unpack_kill_response(const struct parsed_http_message *message, void - if (cresponse->id != NULL) { - kill_response->id = util_strdup_s(cresponse->id); - } -- -+ - kill_response->server_errono = cresponse->cc; - if (cresponse->errmsg != NULL) { - kill_response->errmsg = util_strdup_s(cresponse->errmsg); -diff --git a/src/cmd/isula/base/run.h b/src/cmd/isula/base/run.h -index efca28a9..1084fae2 100644 ---- a/src/cmd/isula/base/run.h -+++ b/src/cmd/isula/base/run.h -@@ -34,7 +34,7 @@ extern "C" { - 'd', \ - &(cmdargs).detach, \ - "Run container in background and print container ID", \ -- NULL }, -+ NULL }, - - extern const char g_cmd_run_desc[]; - extern const char g_cmd_run_usage[]; -diff --git a/src/cmd/isula/client_console.c b/src/cmd/isula/client_console.c -index 8cba2047..1c087d24 100644 ---- a/src/cmd/isula/client_console.c -+++ b/src/cmd/isula/client_console.c -@@ -338,7 +338,7 @@ int delete_client_fifo_home_dir(const char *name) - WARN("Failed to delete client fifo home path:%s", client_fifo_home_dir); - } - free(client_fifo_home_dir); -- -+ - return 0; - } - -diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c -index 8f60eb44..02c67e30 100644 ---- a/src/cmd/isula/stream/attach.c -+++ b/src/cmd/isula/stream/attach.c -@@ -49,7 +49,7 @@ sem_t g_attach_waitopen_sem; - sem_t g_attach_waitexit_sem; - #endif - --struct client_arguments g_cmd_attach_args = { -+struct client_arguments g_cmd_attach_args = { - .time = INSPECT_TIMEOUT_SEC, - }; - -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index 66c604f3..5222629c 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -1203,7 +1203,8 @@ out: - return ret; - } - --static int try_wait_all_child(void) { -+static int try_wait_all_child(void) -+{ - if (waitpid(-1, NULL, WNOHANG) == -1 && errno == ECHILD) { - // all child handled - return 0; -@@ -1243,7 +1244,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept) - } - if (exit_shim) { - process_kill_all(p); -- -+ - // wait atmost 120 seconds - DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); - if (nret != 0) { -diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc -index d3b661a2..bc5c73f9 100644 ---- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc -+++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc -@@ -465,7 +465,7 @@ Status ContainerServiceImpl::Stop(ServerContext *context, const StopRequest *req - } - - (void)cb->container.stop(container_req, &container_res); -- -+ - stop_response_to_grpc(container_res, reply); - - free_container_stop_request(container_req); -diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc b/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc -index e934aa74..853336fe 100644 ---- a/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc -+++ b/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc -@@ -268,12 +268,12 @@ void ContainerServiceImpl::stop_response_to_grpc(const container_stop_response * - gresponse->set_cc(ISULAD_ERR_MEMOUT); - return; - } -- -+ - gresponse->set_cc(response->cc); - if (response->errmsg != nullptr) { - gresponse->set_errmsg(response->errmsg); - } -- -+ - if (response->id != nullptr) { - gresponse->set_id(response->id); - } -@@ -321,12 +321,12 @@ void ContainerServiceImpl::kill_response_to_grpc(const container_kill_response * - gresponse->set_cc(ISULAD_ERR_MEMOUT); - return; - } -- -+ - gresponse->set_cc(response->cc); - if (response->errmsg != nullptr) { - gresponse->set_errmsg(response->errmsg); - } -- -+ - if (response->id != nullptr) { - gresponse->set_id(response->id); - } -diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -index 3e89008a..2ebd800e 100644 ---- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -@@ -99,8 +99,8 @@ void PodSandboxManagerServiceImpl::ApplySandboxResources(const runtime::v1alpha2 - } - - --void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue(const google::protobuf::Map &annotations, -- host_config *hc) -+void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue( -+ const google::protobuf::Map &annotations, host_config *hc) - { - free(hc->network_mode); - -@@ -525,7 +525,7 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2:: - return; - } - stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, -- std::string(sandbox_key))); -+ std::string(sandbox_key))); - free(sandbox_key); - } - m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), -@@ -775,12 +775,12 @@ auto PodSandboxManagerServiceImpl::ClearCniNetwork(const std::string &realSandbo - // not return and make sure teardown network - } else { - stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, -- std::string(netnsPath))); -+ std::string(netnsPath))); - } - } - - m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, -- pluginErr); -+ pluginErr); - if (pluginErr.NotEmpty()) { - WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage()); - errlist.push_back(pluginErr.GetMessage()); -diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c -index 4dd5f858..4ca2eb8b 100644 ---- a/src/daemon/executor/container_cb/execution.c -+++ b/src/daemon/executor/container_cb/execution.c -@@ -465,7 +465,7 @@ static int container_start_prepare(container_t *cont, const container_start_requ - - // init cgroup path for cpu_rt_runtime and cpu_rt_period - // we should do this in start container, not create container -- // because it will fail after the following steps: -+ // because it will fail after the following steps: - // 1. enable cpu-rt of isulad; - // 2. then run container with --cpu-rt-runtime - // 3. then reboot machine; -diff --git a/src/daemon/modules/image/image_rootfs_handler.c b/src/daemon/modules/image/image_rootfs_handler.c -index c3964b2c..1275658e 100644 ---- a/src/daemon/modules/image/image_rootfs_handler.c -+++ b/src/daemon/modules/image/image_rootfs_handler.c -@@ -87,7 +87,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user * - struct passwd *pwbufp = NULL; - - if (f_passwd != NULL) { --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); - #else - errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); -@@ -105,7 +105,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user * - *matched_username = util_strdup_s(pwbufp->pw_name); - break; - } --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); - #else - errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); -@@ -215,7 +215,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user - return 0; - } - --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); - #else - errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); -@@ -226,7 +226,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user - if (search_group_list(gbufp, matched_username, puser) != 0) { - return -1; - } --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); - #else - errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); -@@ -240,7 +240,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user - puser->gid = gbufp->gr_gid; - *groupcnt = 1; - } --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); - #else - errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); -@@ -378,7 +378,7 @@ static int get_additional_groups(char **additional_groups, size_t additional_gro - struct group *gbufp = NULL; - struct group *groups = NULL; - --#if defined (__ANDROID__) || defined(__MUSL__) -+#if defined (__ANDROID__) || defined(__MUSL__) - while (f_group != NULL && util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) { - #else - while (f_group != NULL && fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) { -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index d198f92e..99f8be5e 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -599,10 +599,10 @@ static int runtime_call_simple(const char *workdir, const char *runtime, const c - if (!util_exec_cmd(runtime_exec_func, &rei, NULL, &stdout, &stderr)) { - ERROR("call runtime %s failed stderr %s", subcmd, stderr); - ret = -1; -- // additional handler for the stderr, -+ // additional handler for the stderr, - // this intend to change the ret val of this function -- // for example, if output string contains some specific content, -- // we consider the runtime call simple succeeded, -+ // for example, if output string contains some specific content, -+ // we consider the runtime call simple succeeded, - // even if the process exit with failure. - if (stderr != NULL && cb != NULL) { - ret = cb(stderr); -@@ -636,7 +636,7 @@ static int kill_output_check(const char *output) - - // kill success or kill_output_check succeed return 0, DO_RETRY_CALL will break; - // if kill failed, recheck on shim alive, if not alive, kill succeed, still return 0; --// else, return -1, DO_RETRY_CALL will call this again; -+// else, return -1, DO_RETRY_CALL will call this again; - static int runtime_call_kill_and_check(const char *workdir, const char *runtime, const char *id) - { - int ret = -1; -diff --git a/src/daemon/modules/runtime/shim/shim_rt_ops.c b/src/daemon/modules/runtime/shim/shim_rt_ops.c -index 1b497358..9afb9ce7 100644 ---- a/src/daemon/modules/runtime/shim/shim_rt_ops.c -+++ b/src/daemon/modules/runtime/shim/shim_rt_ops.c -@@ -289,7 +289,7 @@ int rt_shim_create(const char *id, const char *runtime, const rt_create_params_t - goto out; - } - close(fd); -- -+ - if (shim_bin_v2_create(runtime, id, params->bundle, NULL, addr, state_path) != 0) { - ERROR("%s: failed to create v2 shim", id); - ret = -1; -diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c -index 340dea07..f30a3f8e 100644 ---- a/src/daemon/modules/service/service_container.c -+++ b/src/daemon/modules/service/service_container.c -@@ -1094,7 +1094,7 @@ static int delete_client_fifo_home_dir(const char *name) - WARN("Failed to delete client fifo home path:%s", client_fifo_home_dir); - } - free(client_fifo_home_dir); -- -+ - return 0; - } - -@@ -1199,7 +1199,7 @@ out: - // when container is auto-remove, it will be deleted when stopped. - // isula has no suitable time to delete fifo dir, so isulad delete it here. - // Whether the delete container operation fails or not, delete the client's fifo dir to avoid it residual. -- // When isula and isulad use tcp to connect, fifo files will not be created. -+ // When isula and isulad use tcp to connect, fifo files will not be created. - // Because restart will set auto_remove to false, using auto_remove_bak to ensure delete Policy. - if (cont->hostconfig != NULL && cont->hostconfig->auto_remove_bak && delete_client_fifo_home_dir(id) != 0) { - WARN("Failed to delete client fifo home dir"); -diff --git a/src/utils/cutils/filters.c b/src/utils/cutils/filters.c -index 21f5db7c..43ccde20 100644 ---- a/src/utils/cutils/filters.c -+++ b/src/utils/cutils/filters.c -@@ -318,7 +318,7 @@ bool filters_args_valid_key(const char **accepted, size_t len, const char *field - if (accepted == NULL || field == NULL) { - return false; - } -- -+ - for (i = 0; i < len; i++) { - if (accepted[i] != NULL && strcmp(accepted[i], field) == 0) { - return true; -diff --git a/src/utils/cutils/utils_mount_spec.c b/src/utils/cutils/utils_mount_spec.c -index e9b23cf2..6793f93b 100644 ---- a/src/utils/cutils/utils_mount_spec.c -+++ b/src/utils/cutils/utils_mount_spec.c -@@ -465,15 +465,15 @@ int util_parse_mount_spec(char *mount_str, mount_spec **spec, char **errmsg_out) - return -1; - } - -- if(spec == NULL){ -+ if (spec == NULL) { - CACHE_ERRMSG(errmsg, "Invalid spec: can't be NULL"); - return -1; - } - -- if(errmsg_out == NULL){ -+ if (errmsg_out == NULL) { - CACHE_ERRMSG(errmsg, "Invalid errmsg_out: can't be NULL"); - return -1; -- } -+ } - - m = util_common_calloc_s(sizeof(mount_spec)); - if (m == NULL) { -diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc -index 1cfdf86f..8691db7c 100644 ---- a/test/cgroup/cpu/cgroup_cpu_ut.cc -+++ b/test/cgroup/cpu/cgroup_cpu_ut.cc -@@ -49,7 +49,7 @@ TEST(CgroupCpuUnitTest, test_conf_get_cgroup_cpu_rt) - ASSERT_EQ(conf_get_cgroup_cpu_rt(nullptr, nullptr), -1); - ASSERT_EQ(conf_get_cgroup_cpu_rt(&cpu_rt_period, nullptr), -1); - ASSERT_EQ(conf_get_cgroup_cpu_rt(nullptr, &cpu_rt_runtime), -1); -- -+ - struct service_arguments *args = new_args(cpu_rt_period, cpu_rt_runtime); - ASSERT_EQ(save_args_to_conf(args), 0); - ASSERT_EQ(conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime), 0); -diff --git a/test/console/console_ut.cc b/test/console/console_ut.cc -index 73479000..a31083b3 100644 ---- a/test/console/console_ut.cc -+++ b/test/console/console_ut.cc -@@ -26,7 +26,7 @@ TEST(utils_console, test_console_fifo_create) - - ASSERT_EQ(S_ISFIFO(buf.st_mode), true); - -- ret = access(FIFO_NAME, R_OK|W_OK); -+ ret = access(FIFO_NAME, R_OK | W_OK); - ASSERT_EQ(ret, 0); - - remove(FIFO_NAME); -@@ -47,7 +47,7 @@ TEST(utils_console, test_console_fifo_delete) - - memset(path_buf, 'a', LONGER_PATH_MAX); - path_buf[LONGER_PATH_MAX - 1] = 0; -- ASSERT_EQ(strlen(path_buf), LONGER_PATH_MAX-1)<< "strlen is " << strlen(path_buf); -+ ASSERT_EQ(strlen(path_buf), LONGER_PATH_MAX - 1) << "strlen is " << strlen(path_buf); - - ret = console_fifo_create(FIFO_NAME); - if (ret != 0) { -diff --git a/test/cutils/utils_file/utils_file_ut.cc b/test/cutils/utils_file/utils_file_ut.cc -index ccd60ba4..78c07e60 100644 ---- a/test/cutils/utils_file/utils_file_ut.cc -+++ b/test/cutils/utils_file/utils_file_ut.cc -@@ -40,12 +40,12 @@ TEST(utils_file, test_util_dir_exists) - TEST(utils_file, test_util_fileself_exists) - { - ASSERT_EQ(util_fileself_exists(nullptr), false); -- -+ - std::string path = "/tmp/test"; - std::string path_link = "/tmp/test/link"; - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); - ASSERT_EQ(util_fileself_exists(path_link.c_str()), false); -- ASSERT_EQ(symlink(path.c_str(),path_link.c_str()), 0); -+ ASSERT_EQ(symlink(path.c_str(), path_link.c_str()), 0); - ASSERT_EQ(util_fileself_exists(path_link.c_str()), true); - ASSERT_EQ(util_path_remove(path_link.c_str()), 0); - ASSERT_EQ(util_path_remove(path.c_str()), 0); -@@ -54,7 +54,7 @@ TEST(utils_file, test_util_fileself_exists) - TEST(utils_file, test_util_file_exists) - { - ASSERT_EQ(util_file_exists(nullptr), false); -- -+ - std::string path = "/tmp/test"; - ASSERT_EQ(util_file_exists(path.c_str()), false); - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); -@@ -65,13 +65,13 @@ TEST(utils_file, test_util_file_exists) - - TEST(utils_file, test_util_recursive_rmdir) - { -- ASSERT_EQ(util_recursive_rmdir(nullptr,0), -1); -- -+ ASSERT_EQ(util_recursive_rmdir(nullptr, 0), -1); -+ - std::string path = "/tmp/test"; - std::string path_link = "/tmp/test/link"; - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); - ASSERT_EQ(util_mkdir_p(path_link.c_str(), FILE_PERMISSION_TEST), 0); -- ASSERT_EQ(util_recursive_rmdir(path.c_str(),1), 0); -+ ASSERT_EQ(util_recursive_rmdir(path.c_str(), 1), 0); - ASSERT_EQ(util_file_exists(path.c_str()), false); - ASSERT_EQ(util_file_exists(path_link.c_str()), false); - } -@@ -80,11 +80,11 @@ TEST(utils_file, test_util_ensure_path) - { - char *rpath = NULL; - std::string path = "/tmp/test"; -- ASSERT_EQ(util_ensure_path(nullptr,path.c_str()), -1); -- ASSERT_EQ(util_ensure_path(&rpath,nullptr), -1); -+ ASSERT_EQ(util_ensure_path(nullptr, path.c_str()), -1); -+ ASSERT_EQ(util_ensure_path(&rpath, nullptr), -1); - - ASSERT_EQ(util_file_exists(path.c_str()), false); -- ASSERT_EQ(util_ensure_path(&rpath,path.c_str()), 0); -+ ASSERT_EQ(util_ensure_path(&rpath, path.c_str()), 0); - ASSERT_EQ(util_file_exists(rpath), true); - ASSERT_EQ(util_path_remove(path.c_str()), 0); - } -@@ -130,12 +130,12 @@ TEST(utils_file, test_util_add_path) - std::string path = "/tmp/test/"; - std::string add_path = "add"; - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); -- ASSERT_STREQ(util_add_path(path.c_str(),add_path.c_str()), "/tmp/test/add"); -+ ASSERT_STREQ(util_add_path(path.c_str(), add_path.c_str()), "/tmp/test/add"); - ASSERT_EQ(util_path_remove(path.c_str()), 0); -- -+ - std::string path1 = "/tmp/test"; - ASSERT_EQ(util_mkdir_p(path1.c_str(), FILE_PERMISSION_TEST), 0); -- ASSERT_STREQ(util_add_path(path1.c_str(),add_path.c_str()), "/tmp/add"); -+ ASSERT_STREQ(util_add_path(path1.c_str(), add_path.c_str()), "/tmp/add"); - ASSERT_EQ(util_path_remove(path1.c_str()), 0); - } - -@@ -143,7 +143,7 @@ TEST(utils_file, test_verify_file_and_get_real_path) - { - std::string path = "/tmp/test"; - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); -- ASSERT_STREQ(verify_file_and_get_real_path(path.c_str()),"/tmp/test"); -+ ASSERT_STREQ(verify_file_and_get_real_path(path.c_str()), "/tmp/test"); - ASSERT_EQ(util_path_remove(path.c_str()), 0); - } - -@@ -151,17 +151,17 @@ TEST(utils_file, test_look_path) - { - std::string path = "/usr/bin/nsenter"; - char *err = NULL; -- ASSERT_STREQ(look_path("nsenter",&err),path.c_str()); -+ ASSERT_STREQ(look_path("nsenter", &err), path.c_str()); - } - - TEST(utils_file, test_util_copy_file) - { - std::string path = "/tmp/test"; -- ASSERT_EQ(util_copy_file("/etc/hosts",path.c_str(),NETWORK_MOUNT_FILE_MODE), 0); -+ ASSERT_EQ(util_copy_file("/etc/hosts", path.c_str(), NETWORK_MOUNT_FILE_MODE), 0); - ASSERT_EQ(util_path_remove(path.c_str()), 0); - - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); -- ASSERT_EQ(util_copy_file("/etc/hosts",path.c_str(),NETWORK_MOUNT_FILE_MODE), -1); -+ ASSERT_EQ(util_copy_file("/etc/hosts", path.c_str(), NETWORK_MOUNT_FILE_MODE), -1); - ASSERT_EQ(util_path_remove(path.c_str()), 0); - } - -@@ -169,10 +169,10 @@ TEST(utils_file, test_utils_calculate_dir_size_without_hardlink) - { - std::string path = "/tmp/test"; - std::string hosts = "/etc/hosts"; -- ASSERT_EQ(util_copy_file(hosts.c_str(),path.c_str(),NETWORK_MOUNT_FILE_MODE), 0); -+ ASSERT_EQ(util_copy_file(hosts.c_str(), path.c_str(), NETWORK_MOUNT_FILE_MODE), 0); - int64_t total_size = 0; - int64_t total_inodes = 0; -- utils_calculate_dir_size_without_hardlink("/tmp/", &total_size,&total_inodes); -+ utils_calculate_dir_size_without_hardlink("/tmp/", &total_size, &total_inodes); - ASSERT_NE(total_size, 0); - ASSERT_NE(total_inodes, 0); - ASSERT_EQ(util_path_remove(path.c_str()), 0); -@@ -191,12 +191,12 @@ TEST(utils_file, test_util_proc_file_line_by_line) - std::string content = "hello world"; - int fd; - fd = util_open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, DEFAULT_SECURE_FILE_MODE); -- ASSERT_EQ(util_write_nointr_in_total(fd, content.c_str(), strlen(content.c_str())),11); -+ ASSERT_EQ(util_write_nointr_in_total(fd, content.c_str(), strlen(content.c_str())), 11); - FILE *fp = NULL; -- map_t *checked_layers = NULL; -+ map_t *checked_layers = NULL; - fp = util_fopen(path.c_str(), "r"); - checked_layers = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -- ASSERT_EQ(util_proc_file_line_by_line(fp, parse_checked_layer_cb, (void *)checked_layers),0); -+ ASSERT_EQ(util_proc_file_line_by_line(fp, parse_checked_layer_cb, (void *)checked_layers), 0); - fclose(fp); - ASSERT_EQ(util_path_remove(path.c_str()), 0); - } -@@ -204,7 +204,7 @@ TEST(utils_file, test_util_proc_file_line_by_line) - TEST(utils_file, test_util_recursive_remove_path) - { - ASSERT_EQ(util_recursive_remove_path(nullptr), -1); -- -+ - std::string path = "/tmp/test"; - std::string path_link = "/tmp/test/link"; - ASSERT_EQ(util_mkdir_p(path.c_str(), FILE_PERMISSION_TEST), 0); -@@ -212,16 +212,16 @@ TEST(utils_file, test_util_recursive_remove_path) - ASSERT_EQ(util_recursive_remove_path(path.c_str()), 0); - ASSERT_EQ(util_file_exists(path.c_str()), false); - ASSERT_EQ(util_file_exists(path_link.c_str()), false); -- -+ - } - - TEST(utils_file, test_util_copy_dir_recursive) - { -- char *path = (char*)"/tmp/test1/"; -+ char *path = (char*)"/tmp/test1/"; - char *src = (char*)"/tmp/test/"; - ASSERT_EQ(util_mkdir_p(path, FILE_PERMISSION_TEST), 0); - ASSERT_EQ(util_mkdir_p(src, FILE_PERMISSION_TEST), 0); -- ASSERT_EQ(util_copy_dir_recursive(path,src), 0); -+ ASSERT_EQ(util_copy_dir_recursive(path, src), 0); - ASSERT_EQ(util_recursive_remove_path(path), 0); - ASSERT_EQ(util_recursive_remove_path(src), 0); - } -diff --git a/test/cutils/utils_mount_spec/utils_mount_spec_ut.cc b/test/cutils/utils_mount_spec/utils_mount_spec_ut.cc -index 0f60d397..77b4752c 100644 ---- a/test/cutils/utils_mount_spec/utils_mount_spec_ut.cc -+++ b/test/cutils/utils_mount_spec/utils_mount_spec_ut.cc -@@ -18,7 +18,8 @@ - - TEST(utils_mount_spec, test_util_valid_mount_spec) - { -- char *base_valid = (char *)"type=bind,source=/home,target=/vol3,readonly=true,bind-selinux-opts=z,bind-propagation=rprivate"; -+ char *base_valid = (char *) -+ "type=bind,source=/home,target=/vol3,readonly=true,bind-selinux-opts=z,bind-propagation=rprivate"; - char *oci_valid = (char *)"type=tmpfs,dst=/tmpfs,tmpfs-size=1m,tmpfs-mode=1700"; - char *invalid1 = (char *)"type=volume,src=vol,dst=/vol,ro=true,red=false"; - char *invalid2 = (char *)"type,src,dst"; -@@ -35,7 +36,8 @@ TEST(utils_mount_spec, test_util_valid_mount_spec) - - TEST(utils_mount_spec, test_util_parse_mount_spec) - { -- char *base_valid = (char *)"type=bind,source=/home,target=/vol3,readonly=true,bind-selinux-opts=z,bind-propagation=rprivate"; -+ char *base_valid = (char *) -+ "type=bind,source=/home,target=/vol3,readonly=true,bind-selinux-opts=z,bind-propagation=rprivate"; - char *oci_valid = (char *)"type=tmpfs,dst=/tmpfs,tmpfs-size=1m,tmpfs-mode=1700"; - char *invalid1 = (char *)"type=volume,src=vol,dst=/vol,ro=true,red=false"; - char *invalid2 = (char *)"type,src,dst"; -diff --git a/test/cutils/utils_pwgr/utils_pwgr_ut.cc b/test/cutils/utils_pwgr/utils_pwgr_ut.cc -index 2a6b7e08..431372d3 100644 ---- a/test/cutils/utils_pwgr/utils_pwgr_ut.cc -+++ b/test/cutils/utils_pwgr/utils_pwgr_ut.cc -@@ -77,7 +77,9 @@ TEST(utils_pwgr, test_getgrent_r) - FILE *f_gr = fopen(path.c_str(), "r"); - ASSERT_NE(f_gr, nullptr); - -- struct group gr{0}; -+ struct group gr { -+ 0 -+ }; - struct group *pgr = nullptr; - struct group *pgr_alter = &gr; - char buf[BUFSIZ] = { 0 }; -diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc -index c8f38717..6c7ece24 100644 ---- a/test/cutils/utils_utils/utils_utils_ut.cc -+++ b/test/cutils/utils_utils/utils_utils_ut.cc -@@ -134,7 +134,7 @@ TEST(utils_utils, test_util_env_ops) - ASSERT_EQ(ret_len, 2); - ASSERT_STREQ(ret[1], second_val.c_str()); - -- -+ - got = util_env_get_val(ret, ret_len, key1.c_str(), key1.size()); - ASSERT_STREQ(got, "world"); - free(got); -@@ -272,7 +272,8 @@ TEST(utils_utils, test_convert_v2_runtime) - } - - int global_total = 0; --int retry_call_test(int success_idx) { -+int retry_call_test(int success_idx) -+{ - if (global_total == success_idx) { - return 0; - } -diff --git a/test/cutils/utils_verify/utils_verify_ut.cc b/test/cutils/utils_verify/utils_verify_ut.cc -index 9c4bf944..a68dfb66 100644 ---- a/test/cutils/utils_verify/utils_verify_ut.cc -+++ b/test/cutils/utils_verify/utils_verify_ut.cc -@@ -159,7 +159,7 @@ TEST(utils_verify, test_util_valid_file) - - ASSERT_EQ(util_path_remove(isulad_dir.c_str()), 0); - -- ASSERT_EQ(util_valid_file(nullptr, S_IFDIR),false); -+ ASSERT_EQ(util_valid_file(nullptr, S_IFDIR), false); - } - - TEST(utils_verify, test_util_valid_digest) -diff --git a/test/image/oci/storage/layers/storage_driver_ut.cc b/test/image/oci/storage/layers/storage_driver_ut.cc -index 650368d8..ae9f4df1 100644 ---- a/test/image/oci/storage/layers/storage_driver_ut.cc -+++ b/test/image/oci/storage/layers/storage_driver_ut.cc -@@ -144,7 +144,8 @@ protected: - + root_dir + "/overlay/9c27e219663c25e0f28493790cc0b88bc973ba3b1686355f221c38a36978ac63/work "; - ASSERT_EQ(system(mkdir.c_str()), 0); - -- struct storage_module_init_options *opts = (struct storage_module_init_options *)util_common_calloc_s(sizeof(struct storage_module_init_options)); -+ struct storage_module_init_options *opts = (struct storage_module_init_options *)util_common_calloc_s(sizeof( -+ struct storage_module_init_options)); - opts->storage_root = strdup(root_dir.c_str()); - opts->storage_run_root = strdup(run_dir.c_str()); - opts->driver_name = strdup("overlay"); -diff --git a/test/sha256/sha256_ut.cc b/test/sha256/sha256_ut.cc -index 746220d7..2313ef0d 100644 ---- a/test/sha256/sha256_ut.cc -+++ b/test/sha256/sha256_ut.cc -@@ -67,7 +67,7 @@ TEST(sha256, test_sha256_digest_str) - EXPECT_STREQ(digest, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); - - digest = sha256_digest_str( -- "^cvdgfdgghaswere3575676y&*`~cx,xfdgdvcvdfd][';./?.,<>|\\!@#$%^&*()_+=-090wvvs3sdfel33cxvdf***$"); -+ "^cvdgfdgghaswere3575676y&*`~cx,xfdgdvcvdfd][';./?.,<>|\\!@#$%^&*()_+=-090wvvs3sdfel33cxvdf***$"); - EXPECT_STREQ(digest, "899a57a99c14c047eab26f8d6719da256a0737f6c28728ba5777b4fc5398c657"); - } - --- -2.25.1 - diff --git a/0017-add-retry-for-read-write.patch b/0017-add-retry-for-read-write.patch deleted file mode 100644 index eec3caa..0000000 --- a/0017-add-retry-for-read-write.patch +++ /dev/null @@ -1,337 +0,0 @@ -From 82f9d555929be506738a2113bb0f0034e402ccd0 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Fri, 17 Feb 2023 16:07:53 +0800 -Subject: [PATCH 17/19] add retry for read/write - -Signed-off-by: zhangxiaoyu ---- - src/cmd/isulad-shim/common.c | 6 +++--- - src/cmd/isulad/main.c | 4 ++-- - src/daemon/common/selinux_label.c | 2 +- - src/daemon/entry/connect/grpc/grpc_containers_service.cc | 9 ++++++--- - src/daemon/entry/cri/sysctl_tools.c | 2 +- - src/daemon/executor/container_cb/execution.c | 2 +- - .../modules/container/container_gc/containers_gc.c | 3 ++- - src/daemon/modules/events_sender/event_sender.c | 2 +- - src/daemon/modules/image/oci/storage/storage.c | 4 +++- - src/daemon/modules/log/log_gather.c | 6 +++--- - src/daemon/modules/plugin/plugin.c | 2 +- - src/daemon/modules/runtime/isula/isula_rt_ops.c | 4 ++-- - src/daemon/modules/service/io_handler.c | 2 +- - src/daemon/modules/service/service_container.c | 2 +- - src/utils/cutils/utils.c | 2 +- - src/utils/cutils/utils_aes.c | 2 +- - src/utils/cutils/utils_file.c | 2 +- - src/utils/tar/util_archive.c | 4 ++-- - src/utils/tar/util_gzip.c | 2 +- - 19 files changed, 34 insertions(+), 28 deletions(-) - -diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c -index bb8464bb..0c345187 100644 ---- a/src/cmd/isulad-shim/common.c -+++ b/src/cmd/isulad-shim/common.c -@@ -196,7 +196,7 @@ int generate_random_str(char *id, size_t len) - } - for (i = 0; i < len; i++) { - int nret; -- if (read(fd, &num, sizeof(int)) < 0) { -+ if (read_nointr(fd, &num, sizeof(int)) < 0) { - close(fd); - return SHIM_ERR; - } -@@ -232,8 +232,8 @@ void write_message(int fd, const char *level, const char *fmt, ...) - va_end(arg_list); - - snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf); -- nwrite = write(fd, msg, strlen(msg)); -- if (nwrite != strlen(msg)) { -+ nwrite = write_nointr_in_total(fd, msg, strlen(msg)); -+ if (nwrite < 0 || (size_t)nwrite != strlen(msg)) { - return; - } - } -diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c -index d22ab407..6d121f33 100644 ---- a/src/cmd/isulad/main.c -+++ b/src/cmd/isulad/main.c -@@ -478,8 +478,8 @@ int check_and_save_pid(const char *fn) - goto out; - } - -- len = (int)write(fd, pidbuf, strlen(pidbuf)); -- if (len < 0) { -+ len = util_write_nointr(fd, pidbuf, strlen(pidbuf)); -+ if (len < 0 || (size_t)len != strlen(pidbuf)) { - ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno)); - ret = -1; - } -diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c -index 24294780..173f3acb 100644 ---- a/src/daemon/common/selinux_label.c -+++ b/src/daemon/common/selinux_label.c -@@ -310,7 +310,7 @@ static int get_random_value(unsigned int range, unsigned int *val) - return -1; - } - -- if (read(fd, &num, sizeof(int)) < 0) { -+ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { - ERROR("Failed to read urandom value\n"); - ret = -1; - goto out; -diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc -index bc5c73f9..f69613ce 100644 ---- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc -+++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc -@@ -384,7 +384,8 @@ Status ContainerServiceImpl::RemoteStart(ServerContext *context, - break; - } - const std::string &command = request.stdin(); -- if (write(read_pipe_fd[1], (void *)(command.c_str()), command.length()) < 0) { -+ int nret = util_write_nointr_in_total(read_pipe_fd[1], command.c_str(), command.length()); -+ if (nret < 0 || (size_t)nret != command.length()) { - ERROR("sub write over!"); - break; - } -@@ -668,7 +669,8 @@ public: - } - for (int i = 0; i < request.cmd_size(); i++) { - std::string command = request.cmd(i); -- if (write(m_read_pipe_fd, (void *)(command.c_str()), command.length()) < 0) { -+ int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length()); -+ if (nret < 0 || (size_t)nret != command.length()) { - ERROR("sub write over!"); - return; - } -@@ -946,7 +948,8 @@ Status ContainerServiceImpl::Attach(ServerContext *context, ServerReaderWriter= PATH_MAX - 1) { - ERROR("Invalid layer id: %s", lid); -@@ -1438,7 +1439,8 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers) - (void)memcpy(buf, lid, strlen(lid)); - buf[strlen(lid)] = '\n'; - // save checked layer ids into file -- if (util_write_nointr(fd, buf, strlen(lid) + 1) < 0) { -+ nret = util_write_nointr(fd, buf, strlen(lid) + 1); -+ if (nret < 0 || (size_t)nret != strlen(lid) + 1) { - ERROR("Write checked layer data failed: %s", strerror(errno)); - ret = -1; - goto out; -diff --git a/src/daemon/modules/log/log_gather.c b/src/daemon/modules/log/log_gather.c -index 49facaa2..414c9ad1 100644 ---- a/src/daemon/modules/log/log_gather.c -+++ b/src/daemon/modules/log/log_gather.c -@@ -183,9 +183,9 @@ static int write_into_file(const void *buf, size_t g_log_size) - return -1; - } - } -- ret = (int)write(g_log_fd, buf, g_log_size); -- if (ret <= 0) { -- return ret; -+ ret = util_write_nointr_in_total(g_log_fd, buf, g_log_size); -+ if (ret < 0 || (size_t)ret != g_log_size) { -+ return -1; - } - - write_size += ret; -diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c -index 53afeeaf..1c0af368 100644 ---- a/src/daemon/modules/plugin/plugin.c -+++ b/src/daemon/modules/plugin/plugin.c -@@ -618,7 +618,7 @@ static int process_plugin_events(int inotify_fd, const char *plugin_dir) - struct inotify_event *plugin_event = NULL; - char buffer[8192 + 1] = { 0 }; - int action = 0; -- events_length = read(inotify_fd, buffer, 8192); -+ events_length = util_read_nointr(inotify_fd, buffer, 8192); - - if (events_length <= 0) { - ERROR("Failed to wait events"); -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index 99f8be5e..e80aa9a4 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -1355,8 +1355,8 @@ int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resi - goto out; - } - -- count = write(fd, data, RESIZE_DATA_SIZE); -- if (count <= 0) { -+ count = util_write_nointr(fd, data, strlen(data)); -+ if (count < 0 || (size_t)count != strlen(data)) { - ERROR("write exec resize data error"); - ret = -1; - goto out; -diff --git a/src/daemon/modules/service/io_handler.c b/src/daemon/modules/service/io_handler.c -index 893733bc..98c763a4 100644 ---- a/src/daemon/modules/service/io_handler.c -+++ b/src/daemon/modules/service/io_handler.c -@@ -340,7 +340,7 @@ static ssize_t write_to_fd(void *context, const void *data, size_t len) - { - ssize_t ret; - ret = util_write_nointr(*(int *)context, data, len); -- if ((ret <= 0) || (ret != (ssize_t)len)) { -+ if (ret < 0 || (size_t)ret != len) { - ERROR("Failed to write: %s", strerror(errno)); - return -1; - } -diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c -index f30a3f8e..6dc06c83 100644 ---- a/src/daemon/modules/service/service_container.c -+++ b/src/daemon/modules/service/service_container.c -@@ -340,7 +340,7 @@ static int write_env_content(const char *env_path, const char **env, size_t env_ - goto out; - } - nret = util_write_nointr(fd, env_content, strlen(env_content)); -- if (nret < 0 || nret != len - 1) { -+ if (nret < 0 || (size_t)nret != strlen(env_content)) { - SYSERROR("Write env file failed"); - free(env_content); - ret = -1; -diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c -index b1db045a..2c3709ad 100644 ---- a/src/utils/cutils/utils.c -+++ b/src/utils/cutils/utils.c -@@ -1237,7 +1237,7 @@ int util_generate_random_str(char *id, size_t len) - } - for (i = 0; i < len; i++) { - int nret; -- if (read(fd, &num, sizeof(int)) < 0) { -+ if (util_read_nointr(fd, &num, sizeof(int)) < 0) { - ERROR("Failed to read urandom value"); - close(fd); - return -1; -diff --git a/src/utils/cutils/utils_aes.c b/src/utils/cutils/utils_aes.c -index 1e25ecd3..055a9538 100644 ---- a/src/utils/cutils/utils_aes.c -+++ b/src/utils/cutils/utils_aes.c -@@ -77,7 +77,7 @@ int util_aes_key(const char *key_file, bool create, unsigned char *aeskey) - goto out; - } - -- if (read(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { -+ if (util_read_nointr(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) { - ERROR("read key file %s failed: %s", key_file, strerror(errno)); - ret = -1; - goto out; -diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c -index 7b1d5cb6..4c62aaa6 100644 ---- a/src/utils/cutils/utils_file.c -+++ b/src/utils/cutils/utils_file.c -@@ -1012,7 +1012,7 @@ int util_file2str(const char *filename, char *buf, size_t len) - if (fd == -1) { - return -1; - } -- num_read = (int)read(fd, buf, len - 1); -+ num_read = (int)util_read_nointr(fd, buf, len - 1); - if (num_read <= 0) { - num_read = -1; - } else { -diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c -index da814c94..630ad8f8 100644 ---- a/src/utils/tar/util_archive.c -+++ b/src/utils/tar/util_archive.c -@@ -659,7 +659,7 @@ child_out: - if (ret != 0) { - ERROR("Wait archive_untar_handler failed with error:%s", strerror(errno)); - fcntl(pipe_stderr[0], F_SETFL, O_NONBLOCK); -- if (read(pipe_stderr[0], errbuf, BUFSIZ) < 0) { -+ if (util_read_nointr(pipe_stderr[0], errbuf, BUFSIZ) < 0) { - ERROR("read error message from child failed"); - } - } -@@ -1054,7 +1054,7 @@ child_out: - if (ret != 0) { - ERROR("tar failed"); - fcntl(pipe_for_read[0], F_SETFL, O_NONBLOCK); -- if (read(pipe_for_read[0], errbuf, BUFSIZ) < 0) { -+ if (util_read_nointr(pipe_for_read[0], errbuf, BUFSIZ) < 0) { - ERROR("read error message from child failed"); - } - } -diff --git a/src/utils/tar/util_gzip.c b/src/utils/tar/util_gzip.c -index 5c34d719..2f4750be 100644 ---- a/src/utils/tar/util_gzip.c -+++ b/src/utils/tar/util_gzip.c -@@ -212,7 +212,7 @@ int gzip(const char *filename, size_t len) - return -1; - } - -- size_read = read(pipefd[0], buffer, BUFSIZ); -+ size_read = util_read_nointr(pipefd[0], buffer, BUFSIZ); - close(pipefd[0]); - - if (size_read) { --- -2.25.1 - diff --git a/0018-add-crictl-timeout-and-sync-for-CI.patch b/0018-add-crictl-timeout-and-sync-for-CI.patch deleted file mode 100644 index 50b7226..0000000 --- a/0018-add-crictl-timeout-and-sync-for-CI.patch +++ /dev/null @@ -1,127 +0,0 @@ -From 4ca7a67fc6f9304239224e92d0ab15180cf48a10 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Mon, 20 Feb 2023 15:42:40 +0800 -Subject: [PATCH 18/19] add crictl timeout and sync for CI - -Signed-off-by: zhangxiaoyu ---- - CI/test_cases/container_cases/bigdata_stream.sh | 7 +++++++ - CI/test_cases/container_cases/bigdata_stream_runc.sh | 4 ++++ - CI/test_cases/helpers.sh | 5 ++++- - 3 files changed, 15 insertions(+), 1 deletion(-) - -diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/bigdata_stream.sh -index 6c669503..4b2022af 100755 ---- a/CI/test_cases/container_cases/bigdata_stream.sh -+++ b/CI/test_cases/container_cases/bigdata_stream.sh -@@ -124,6 +124,7 @@ function test_concurrent_bigdata_stream() - pids[${#pids[@]}]=$! - done - wait ${pids[*]// /|} -+ sync && sync - - for index in $(seq 1 5); do - ls -l /home/iocopy_stream_data_500M_$index -@@ -151,6 +152,7 @@ function test_concurrent_bigdata_stream_without_pty() - pids[${#pids[@]}]=$! - done - wait ${pids[*]// /|} -+ sync && sync - - for index in $(seq 1 5); do - ls -l /home/iocopy_stream_data_500M_$index -@@ -209,6 +211,7 @@ function test_stream_with_stop_client() - kill -18 $pid - - wait $pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -@@ -254,6 +257,7 @@ function test_stream_with_stop_attach() - kill -18 $pid - - wait $exec_pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -@@ -299,6 +303,7 @@ function test_stream_with_stop_lxc_monitor() - kill -18 $pid - - wait $exec_pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -@@ -348,6 +353,7 @@ function test_stream_with_stop_isulad() - kill -18 $(cat /var/run/isulad.pid) - - wait $pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -@@ -398,6 +404,7 @@ function test_stream_with_runc() - isula exec -it $RUNCID cat test_500M > /home/iocopy_stream_data_500M - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata" && ((ret++)) - -+ sync && sync - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) - [[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++)) - -diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh -index 6933bef0..3a384cd8 100755 ---- a/CI/test_cases/container_cases/bigdata_stream_runc.sh -+++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh -@@ -117,6 +117,7 @@ function test_cat_bigdata() - pids[${#pids[@]}]=$! - done - wait ${pids[*]// /|} -+ sync && sync - - for index in $(seq 1 5); do - ls -l /home/iocopy_stream_data_500M_$index -@@ -144,6 +145,7 @@ function test_cat_bigdata_without_pty() - pids[${#pids[@]}]=$! - done - wait ${pids[*]// /|} -+ sync && sync - - for index in $(seq 1 5); do - ls -l /home/iocopy_stream_data_500M_$index -@@ -173,6 +175,7 @@ function test_stream_with_stop_client() - kill -18 $pid - - wait $pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -@@ -217,6 +220,7 @@ function test_stream_with_stop_isulad() - kill -18 $(cat /var/run/isulad.pid) - - wait $pid -+ sync && sync - - ls -l /home/iocopy_stream_data_500M - total_size=$(stat -c"%s" /home/iocopy_stream_data_500M) -diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh -index 5097217c..a11e676c 100755 ---- a/CI/test_cases/helpers.sh -+++ b/CI/test_cases/helpers.sh -@@ -80,7 +80,10 @@ function wait_container() { - - function crictl() { - CRICTL=$(which crictl) -- "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock "$@" -+ # Default timeout is 2s. -+ # In some high IO testcase, isulad handle CRI request time maybe more than 2s. -+ # And the crictl will print error message "context deadline exceeded" -+ "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock --timeout 5s "$@" - } - - function msg_ok() --- -2.25.1 - diff --git a/0019-unlock-m_podsLock-if-new-failed.patch b/0019-unlock-m_podsLock-if-new-failed.patch deleted file mode 100644 index b332e39..0000000 --- a/0019-unlock-m_podsLock-if-new-failed.patch +++ /dev/null @@ -1,27 +0,0 @@ -From d7c76d137f60b4adf0eba46e0db67182fca04e69 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Wed, 22 Feb 2023 10:43:52 +0800 -Subject: [PATCH 19/19] unlock m_podsLock if new failed - -Signed-off-by: zhangxiaoyu ---- - src/daemon/entry/cri/network_plugin.cc | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc -index 00be975e..7a957de3 100644 ---- a/src/daemon/entry/cri/network_plugin.cc -+++ b/src/daemon/entry/cri/network_plugin.cc -@@ -347,6 +347,9 @@ void PluginManager::Lock(const std::string &fullPodName, Errors &error) - auto tmpLock = std::unique_ptr(new (std::nothrow) PodLock()); - if (tmpLock == nullptr) { - error.SetError("Out of memory"); -+ if (pthread_mutex_unlock(&m_podsLock) != 0) { -+ error.SetError("plugin manager unlock failed"); -+ } - return; - } - lock = tmpLock.get(); --- -2.25.1 - diff --git a/0020-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch b/0020-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch deleted file mode 100644 index 2cb2098..0000000 --- a/0020-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch +++ /dev/null @@ -1,220 +0,0 @@ -From e3e5cf6d2a6858f9f83ee42f8ceeaaef4752ff1b Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Thu, 2 Mar 2023 14:27:01 +0800 -Subject: [PATCH 20/26] fix CRI SetupPod and TearDownPod deadlock - -Signed-off-by: zhangxiaoyu ---- - src/daemon/entry/cri/cni_network_plugin.cc | 44 +++++++++++++------ - .../cri_pod_sandbox_manager_service_impl.cc | 2 + - src/daemon/entry/cri/network_plugin.cc | 39 +++++++++++----- - 3 files changed, 61 insertions(+), 24 deletions(-) - -diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc -index 9b03bea1..02e75ffe 100644 ---- a/src/daemon/entry/cri/cni_network_plugin.cc -+++ b/src/daemon/entry/cri/cni_network_plugin.cc -@@ -110,9 +110,12 @@ void CniNetworkPlugin::SetDefaultNetwork(std::unique_ptr network, st - if (network == nullptr) { - return; - } -- WLockNetworkMap(err); -- if (err.NotEmpty()) { -- ERROR("%s", err.GetCMessage()); -+ -+ Errors tmpErr; -+ WLockNetworkMap(tmpErr); -+ if (tmpErr.NotEmpty()) { -+ ERROR("%s", tmpErr.GetCMessage()); -+ err.AppendError(tmpErr.GetCMessage()); - return; - } - m_defaultNetwork = std::move(network); -@@ -120,9 +123,10 @@ void CniNetworkPlugin::SetDefaultNetwork(std::unique_ptr network, st - - DEBUG("Update new cni network: \"%s\"", m_defaultNetwork->GetName().c_str()); - -- UnlockNetworkMap(err); -- if (err.NotEmpty()) { -- ERROR("%s", err.GetCMessage()); -+ UnlockNetworkMap(tmpErr); -+ if (tmpErr.NotEmpty()) { -+ ERROR("%s", tmpErr.GetCMessage()); -+ err.AppendError(tmpErr.GetCMessage()); - } - } - -@@ -132,8 +136,11 @@ void CniNetworkPlugin::UpdateMutlNetworks(std::vectorGetName()] = std::move(*iter); - } - -- UnlockNetworkMap(err); -+ UnlockNetworkMap(tmpErr); -+ if (tmpErr.NotEmpty()) { -+ err.AppendError(tmpErr.GetCMessage()); -+ } - } - - CniNetworkPlugin::CniNetworkPlugin(std::vector &binDirs, const std::string &confDir, -@@ -336,13 +346,20 @@ free_out: - - void CniNetworkPlugin::CheckInitialized(Errors &err) - { -- RLockNetworkMap(err); -- if (err.NotEmpty()) { -- ERROR("%s", err.GetCMessage()); -+ Errors tmpErr; -+ RLockNetworkMap(tmpErr); -+ if (tmpErr.NotEmpty()) { -+ ERROR("%s", tmpErr.GetCMessage()); -+ err.AppendError(tmpErr.GetCMessage()); - return; - } - bool inited = (m_defaultNetwork != nullptr); -- UnlockNetworkMap(err); -+ -+ UnlockNetworkMap(tmpErr); -+ if (tmpErr.NotEmpty()) { -+ err.AppendError(tmpErr.GetCMessage()); -+ } -+ - if (!inited) { - err.AppendError("cni config uninitialized"); - } -@@ -527,6 +544,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, - } - } - -+ err.Clear(); - RLockNetworkMap(err); - if (err.NotEmpty()) { - ERROR("%s", err.GetCMessage()); -diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -index 2ebd800e..7ff545db 100644 ---- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -@@ -642,6 +642,7 @@ auto PodSandboxManagerServiceImpl::GetRealSandboxIDToStop(const std::string &pod - if (status->linux().namespaces().has_options()) { - hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE); - } -+ // if metadata is invalid, don't return -1 and continue stopping pod - if (status->has_metadata()) { - name = status->metadata().name(); - ns = status->metadata().namespace_(); -@@ -779,6 +780,7 @@ auto PodSandboxManagerServiceImpl::ClearCniNetwork(const std::string &realSandbo - } - } - -+ pluginErr.Clear(); - m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, - pluginErr); - if (pluginErr.NotEmpty()) { -diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc -index 7a957de3..9933b584 100644 ---- a/src/daemon/entry/cri/network_plugin.cc -+++ b/src/daemon/entry/cri/network_plugin.cc -@@ -422,21 +422,27 @@ void PluginManager::GetPodNetworkStatus(const std::string &ns, const std::string - const std::string &interfaceName, const std::string &podSandboxID, - PodNetworkStatus &status, Errors &error) - { -+ Errors tmpErr; - std::string fullName = name + "_" + ns; - -- Lock(fullName, error); -- if (error.NotEmpty()) { -+ Lock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); - return; - } - if (m_plugin != nullptr) { -- Errors tmpErr; - m_plugin->GetPodNetworkStatus(ns, name, interfaceName, podSandboxID, status, tmpErr); - if (tmpErr.NotEmpty()) { - error.Errorf("NetworkPlugin %s failed on the status hook for pod %s: %s", m_plugin->Name().c_str(), - fullName.c_str(), tmpErr.GetCMessage()); - } - } -- Unlock(fullName, error); -+ -+ tmpErr.Clear(); -+ Unlock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); -+ } - } - - void PluginManager::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, -@@ -447,20 +453,26 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con - return; - } - -+ Errors tmpErr; - std::string fullName = name + "_" + ns; -- Lock(fullName, error); -- if (error.NotEmpty()) { -+ Lock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); - return; - } - INFO("Calling network plugin %s to set up pod %s", m_plugin->Name().c_str(), fullName.c_str()); - -- Errors tmpErr; - m_plugin->SetUpPod(ns, name, interfaceName, podSandboxID, annotations, options, tmpErr); - if (tmpErr.NotEmpty()) { - error.Errorf("NetworkPlugin %s failed to set up pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(), - tmpErr.GetCMessage()); - } -- Unlock(fullName, error); -+ -+ tmpErr.Clear(); -+ Unlock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); -+ } - } - - void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, -@@ -469,8 +481,9 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, - { - Errors tmpErr; - std::string fullName = name + "_" + ns; -- Lock(fullName, error); -- if (error.NotEmpty()) { -+ Lock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); - return; - } - if (m_plugin == nullptr) { -@@ -484,7 +497,11 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, - fullName.c_str(), tmpErr.GetCMessage()); - } - unlock: -- Unlock(fullName, error); -+ tmpErr.Clear(); -+ Unlock(fullName, tmpErr); -+ if (tmpErr.NotEmpty()) { -+ error.AppendError(tmpErr.GetCMessage()); -+ } - } - - void NoopNetworkPlugin::Init(const std::string &hairpinMode, const std::string &nonMasqueradeCIDR, int mtu, --- -2.25.1 - diff --git a/0021-support-pull-image-with-digest.patch b/0021-support-pull-image-with-digest.patch deleted file mode 100644 index 8c35f5f..0000000 --- a/0021-support-pull-image-with-digest.patch +++ /dev/null @@ -1,316 +0,0 @@ -From ea3ffc300224b2fee31a5c9f761386cb830ce7bc Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Thu, 16 Feb 2023 18:22:02 +0800 -Subject: [PATCH 21/26] support pull image with digest - -usage: isula pull busybox@sha256:907ca53d7e2947e849b839b1cd258c98fd3916c60f2e6e70c30edbf741ab6754 - -Signed-off-by: haozi007 ---- - src/daemon/executor/container_cb/execution.c | 2 +- - src/daemon/executor/image_cb/image_cb.c | 8 ++++ - src/daemon/modules/image/oci/oci_pull.c | 23 ++++++---- - .../modules/image/oci/registry/registry.c | 2 +- - .../oci/storage/image_store/image_store.c | 7 +++ - src/daemon/modules/image/oci/utils_images.c | 45 +++++++++++++++---- - src/daemon/modules/image/oci/utils_images.h | 2 + - src/utils/cutils/utils_verify.c | 25 ++++++++--- - src/utils/cutils/utils_verify.h | 3 ++ - 9 files changed, 92 insertions(+), 25 deletions(-) - -diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c -index 735c1bff..92c34b09 100644 ---- a/src/daemon/executor/container_cb/execution.c -+++ b/src/daemon/executor/container_cb/execution.c -@@ -405,7 +405,7 @@ static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, con - return 0; - } - --// TODO: maybe we should adapt to cgroup v2 -+// maybe we should adapt to cgroup v2 - static int cpurt_controller_init(const char *id, const host_config *host_spec) - { - int ret = 0; -diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c -index 5beda5f4..c087a679 100644 ---- a/src/daemon/executor/image_cb/image_cb.c -+++ b/src/daemon/executor/image_cb/image_cb.c -@@ -556,6 +556,14 @@ static int trans_one_image(image_list_images_response *response, size_t image_in - out_image->name = util_strdup_s(im_image->repo_tags[repo_index]); - } - -+ if (out_image->name == NULL && im_image->repo_digests != NULL && im_image->repo_digests_len > 0) { -+ // repo digest must valid, so just get lastest @ -+ char *pod = strrchr(im_image->repo_digests[0], '@'); -+ if (pod != NULL) { -+ out_image->name = util_sub_string(im_image->repo_digests[0], 0, (size_t)(pod - im_image->repo_digests[0])); -+ } -+ } -+ - out_image->target = util_common_calloc_s(sizeof(image_descriptor)); - if (out_image->target == NULL) { - ERROR("Out of memory"); -diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c -index d8c874a8..c39cab22 100644 ---- a/src/daemon/modules/image/oci/oci_pull.c -+++ b/src/daemon/modules/image/oci/oci_pull.c -@@ -119,10 +119,19 @@ static int pull_image(const im_pull_request *request, char **name) - options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce; - insecure_registries = oci_image_data->insecure_registries; - -+ // key of image which save in image-store -+ options->dest_image_name = oci_normalize_image_name(request->image); -+ -+ // add default tag if required -+ with_tag = oci_default_tag(request->image); -+ - host = oci_get_host(request->image); - if (host != NULL) { -- options->image_name = oci_default_tag(request->image); -- options->dest_image_name = oci_normalize_image_name(request->image); -+ // 1. image_name use for split host/tag/name -+ // 2. user for tag of log -+ options->image_name = with_tag; -+ with_tag = NULL; -+ - update_option_insecure_registry(options, insecure_registries, host); - ret = registry_pull(options); - if (ret != 0) { -@@ -143,13 +152,12 @@ static int pull_image(const im_pull_request *request, char **name) - } - host = oci_host_from_mirror(*mirror); - update_option_insecure_registry(options, insecure_registries, host); -- with_tag = oci_default_tag(request->image); -+ // add current mirror to image name -+ free(options->image_name); - options->image_name = oci_add_host(host, with_tag); -- free(with_tag); -- with_tag = NULL; - free(host); - host = NULL; -- options->dest_image_name = oci_normalize_image_name(request->image); -+ - ret = registry_pull(options); - if (ret != 0) { - continue; -@@ -161,10 +169,9 @@ static int pull_image(const im_pull_request *request, char **name) - *name = util_strdup_s(options->dest_image_name); - - out: -+ free(with_tag); - free(host); -- host = NULL; - free_registry_pull_options(options); -- options = NULL; - - return ret; - } -diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c -index e3efbb7c..14e84f81 100644 ---- a/src/daemon/modules/image/oci/registry/registry.c -+++ b/src/daemon/modules/image/oci/registry/registry.c -@@ -1855,7 +1855,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio - } - - if (!util_valid_image_name(options->dest_image_name)) { -- ERROR("Invalid dest image name %s", options->image_name); -+ ERROR("Invalid dest image name %s", options->dest_image_name); - isulad_try_set_error_message("Invalid image name"); - return -1; - } -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c -index 9dab66fd..7e1a5373 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.c -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c -@@ -1980,6 +1980,7 @@ static int resort_image_names(const char **names, size_t names_len, char **first - MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH); - } - -+ // maybe should support other digest - if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) { - if (util_array_append(image_digests, names[i]) != 0) { - ERROR("Failed to append image to digest: %s", names[i]); -@@ -2173,6 +2174,7 @@ static int get_image_repo_digests(char ***old_repo_digests, char **image_tags, i - goto out; - } - -+ // get repo digest from images which with tag - if (pack_repo_digest(old_repo_digests, (const char **)image_tags, digest, repo_digests) != 0) { - ERROR("Failed to pack repo digest"); - ret = -1; -@@ -2195,12 +2197,17 @@ static int pack_image_tags_and_repo_digest(image_t *img, imagetool_image *info) - char *image_digest = NULL; - char **repo_digests = NULL; - -+ // get names from image-store names: -+ // 1. image names with tag; -+ // 2. image names with digests; -+ // 3. get first image name, current unused; - if (resort_image_names((const char **)img->simage->names, img->simage->names_len, &name, &tags, &digests) != 0) { - ERROR("Failed to resort image names"); - ret = -1; - goto out; - } - -+ // update repo digests from tags - if (get_image_repo_digests(&digests, tags, img, &image_digest, &repo_digests) != 0) { - ERROR("Failed to get image repo digests"); - ret = -1; -diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c -index 80a25502..794f0d16 100644 ---- a/src/daemon/modules/image/oci/utils_images.c -+++ b/src/daemon/modules/image/oci/utils_images.c -@@ -42,6 +42,26 @@ - // nanos of 2038-01-19T03:14:07, the max valid linux time - #define MAX_NANOS 2147483647000000000 - -+char *oci_image_digest_pos(const char *name) -+{ -+ char *pos = NULL; -+ -+ if (name == NULL) { -+ return NULL; -+ } -+ -+ pos = strrchr(name, '@'); -+ if (pos == NULL) { -+ return NULL; -+ } -+ -+ if (util_reg_match(__DIGESTPattern, pos) != 0) { -+ return NULL; -+ } -+ -+ return pos; -+} -+ - char *get_last_part(char **parts) - { - char *last_part = NULL; -@@ -98,6 +118,7 @@ char *oci_default_tag(const char *name) - } - - last_part = get_last_part(parts); -+ // will pass image name with digest and with tag - if (last_part != NULL && strrchr(last_part, ':') == NULL) { - add_default_tag = DEFAULT_TAG; - } -@@ -181,9 +202,9 @@ char *oci_normalize_image_name(const char *name) - return result; - } - --int oci_split_image_name(const char *image_name, char **host, char **name, char **tag) -+int oci_split_image_name(const char *image_name, char **host, char **name, char **tag_digest) - { -- char *tag_pos = NULL; -+ char *tag_digest_pos = NULL; - char *name_pos = NULL; - char *tmp_image_name = NULL; - -@@ -193,18 +214,24 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char - } - - tmp_image_name = util_strdup_s(image_name); -- tag_pos = util_tag_pos(tmp_image_name); -- if (tag_pos != NULL) { -- *tag_pos = 0; -- tag_pos++; -- if (tag != NULL) { -- *tag = util_strdup_s(tag_pos); -+ -+ // check digest first -+ tag_digest_pos = oci_image_digest_pos(tmp_image_name); -+ if (tag_digest_pos == NULL) { -+ tag_digest_pos = util_tag_pos(tmp_image_name); -+ } -+ -+ if (tag_digest_pos != NULL) { -+ *tag_digest_pos = '\0'; -+ tag_digest_pos++; -+ if (tag_digest != NULL) { -+ *tag_digest = util_strdup_s(tag_digest_pos); - } - } - - name_pos = strchr(tmp_image_name, '/'); - if (name_pos != NULL) { -- *name_pos = 0; -+ *name_pos = '\0'; - name_pos++; - if (name != NULL) { - *name = util_strdup_s(name_pos); -diff --git a/src/daemon/modules/image/oci/utils_images.h b/src/daemon/modules/image/oci/utils_images.h -index 2fa8b29d..53bce4e0 100644 ---- a/src/daemon/modules/image/oci/utils_images.h -+++ b/src/daemon/modules/image/oci/utils_images.h -@@ -56,6 +56,8 @@ char *oci_get_isulad_tmpdir(const char *root_dir); - int makesure_isulad_tmpdir_perm_right(const char *root_dir); - char *get_hostname_to_strip(); - -+char *oci_image_digest_pos(const char *name); -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c -index 713e72c3..ea43a40a 100644 ---- a/src/utils/cutils/utils_verify.c -+++ b/src/utils/cutils/utils_verify.c -@@ -359,7 +359,7 @@ cleanup: - bool util_valid_image_name(const char *name) - { - char *copy = NULL; -- char *tag_pos = NULL; -+ char *check_pos = NULL; - bool bret = false; - - if (name == NULL) { -@@ -372,13 +372,26 @@ bool util_valid_image_name(const char *name) - } - - copy = util_strdup_s(name); -- tag_pos = util_tag_pos(copy); -- if (tag_pos != NULL) { -- if (util_reg_match(__TagPattern, tag_pos)) { -+ -+ // 1. first, check digest or not -+ check_pos = strrchr(copy, '@'); -+ if (check_pos != NULL) { -+ // image name with digest -+ if (util_reg_match(__DIGESTPattern, check_pos)) { - goto cleanup; - } -- -- *tag_pos = '\0'; -+ *check_pos = '\0'; -+ } else { -+ // image name without digest -+ // 2. check tag or not -+ check_pos = util_tag_pos(copy); -+ if (check_pos != NULL) { -+ if (util_reg_match(__TagPattern, check_pos)) { -+ goto cleanup; -+ } -+ -+ *check_pos = '\0'; -+ } - } - - if (util_reg_match(__NamePattern, copy)) { -diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h -index 87976299..7d954475 100644 ---- a/src/utils/cutils/utils_verify.h -+++ b/src/utils/cutils/utils_verify.h -@@ -33,6 +33,9 @@ extern "C" { - "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" \ - "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \ - "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$" -+ -+#define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}" -+ - #define VALID_VOLUME_NAME "[a-zA-Z0-9][a-zA-Z0-9_.-]{1,63}" - - extern const char *g_all_caps[]; --- -2.25.1 - diff --git a/0022-isulad-shim-support-execSync-with-timeout.patch b/0022-isulad-shim-support-execSync-with-timeout.patch deleted file mode 100644 index 80694ba..0000000 --- a/0022-isulad-shim-support-execSync-with-timeout.patch +++ /dev/null @@ -1,186 +0,0 @@ -From 7d5be830bfae9f9908b3b7b323975b65fc7dd856 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 13 Feb 2023 15:36:58 +0800 -Subject: [PATCH 22/26] isulad-shim support execSync with timeout - -Signed-off-by: zhongtao ---- - .../executor/container_cb/execution_stream.c | 2 +- - .../modules/runtime/isula/isula_rt_ops.c | 37 +++++++++++++++--- - src/utils/cutils/utils.c | 39 +++++++++++++++++++ - src/utils/cutils/utils.h | 5 +++ - 4 files changed, 77 insertions(+), 6 deletions(-) - -diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c -index fde0335e..1a7353b5 100644 ---- a/src/daemon/executor/container_cb/execution_stream.c -+++ b/src/daemon/executor/container_cb/execution_stream.c -@@ -161,7 +161,7 @@ static int container_exec_cb(const container_exec_request *request, container_ex - - if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { - ret = -1; -- goto out; -+ goto pack_err_response; - } - - goto out; -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index e80aa9a4..51a72c4f 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -205,6 +205,10 @@ static void show_shim_runtime_errlog(const char *workdir) - char buf1[SHIM_LOG_SIZE] = { 0 }; - char buf2[SHIM_LOG_SIZE] = { 0 }; - -+ if (g_isulad_errmsg != NULL) { -+ return; -+ } -+ - get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); - get_err_message(buf2, sizeof(buf2), workdir, "log.json"); - ERROR("shim-log: %s", buf1); -@@ -678,8 +682,29 @@ static int status_to_exit_code(int status) - return exit_code; - } - -+static int try_wait_pid(pid_t pid) -+{ -+ if (waitpid(pid, NULL, WNOHANG) == pid) { -+ return 0; -+ } -+ -+ return 1; -+} -+ -+static void kill_and_show_err(pid_t pid) -+{ -+ int nret = 0; -+ kill(pid, SIGKILL); -+ // wait atmost 0.5 seconds -+ DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); -+ if (nret != 0) { -+ WARN("Fail to wait isulad-shim"); -+ } -+ isulad_set_error_message("Exec container error;exec timeout"); -+} -+ - static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, -- int *exit_code) -+ int *exit_code, const int64_t timeout) - { - pid_t pid = 0; - int exec_fd[2] = { -1, -1 }; -@@ -770,7 +795,7 @@ realexec: - goto out; - } - -- status = util_wait_for_pid_status(pid); -+ status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); - if (status < 0) { - ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); - ret = -1; -@@ -784,7 +809,9 @@ realexec: - out: - if (ret != 0) { - show_shim_runtime_errlog(workdir); -- kill(pid, SIGKILL); /* can kill other process? */ -+ if (timeout <= 0) { -+ kill(pid, SIGKILL); /* can kill other process? */ -+ } - } - - return ret; -@@ -893,7 +920,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ - } - - get_runtime_cmd(runtime, &cmd); -- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL); -+ ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); - if (ret != 0) { - runtime_call_delete_force(workdir, runtime, id); - ERROR("%s: failed create shim process", id); -@@ -1165,7 +1192,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - } - - get_runtime_cmd(runtime, &cmd); -- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code); -+ ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); - if (ret != 0) { - ERROR("%s: failed create shim process for exec %s", id, exec_id); - goto errlog_out; -diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c -index 2c3709ad..9173cd14 100644 ---- a/src/utils/cutils/utils.c -+++ b/src/utils/cutils/utils.c -@@ -313,6 +313,45 @@ rep: - return 0; - } - -+/* -+ * If timeout <= 0, blocking wait pid. -+ * If timeout > 0, non-blocking wait pid with timeout. -+ * When waitpid timeout, calling handle_timeout_callback_t. -+ */ -+int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) -+{ -+ int nret = 0; -+ time_t start_time = time(NULL); -+ time_t end_time; -+ double interval; -+ -+ if (timeout <= 0) { -+ return util_wait_for_pid_status(pid); -+ } -+ -+ for (;;) { -+ nret = waitpid(pid, NULL, WNOHANG); -+ if (nret == pid) { -+ break; -+ } -+ if (nret == -1 && errno != EINTR) { -+ return -1; -+ } -+ end_time = time(NULL); -+ interval = difftime(end_time, start_time); -+ if (nret == 0 && interval >= timeout) { -+ INFO("Wait %d timeout", pid); -+ if (cb != NULL) { -+ cb(pid); -+ } -+ return -1; -+ } -+ // sleep some time instead to avoid cpu full running and then retry. -+ sleep(0.1); -+ } -+ return 0; -+} -+ - int util_wait_for_pid_status(pid_t pid) - { - int st; -diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h -index fec6d879..d14d048e 100644 ---- a/src/utils/cutils/utils.h -+++ b/src/utils/cutils/utils.h -@@ -301,6 +301,9 @@ typedef struct _proc_t { - processor; /* current (or most recent?) CPU */ - } proc_t; - -+// handle waitpid timeout. -+typedef void(*handle_timeout_callback_t)(pid_t pid); -+ - struct signame { - int num; - const char *name; -@@ -328,6 +331,8 @@ char *util_strdup_s(const char *src); - - int util_wait_for_pid(pid_t pid); - -+int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb); -+ - void util_contain_errmsg(const char *errmsg, int *exit_code); - - char *util_short_digest(const char *digest); --- -2.25.1 - diff --git a/0023-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch b/0023-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch deleted file mode 100644 index 9a08535..0000000 --- a/0023-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch +++ /dev/null @@ -1,26 +0,0 @@ -From e9bd090e5d6755eacaa1f8710c32386aba5190f2 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 6 Mar 2023 09:54:44 +0800 -Subject: [PATCH 23/26] change sleep() to usleep() to avoid lossing of accuracy - -Signed-off-by: zhongtao ---- - src/utils/cutils/utils.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c -index 9173cd14..64d7e9f9 100644 ---- a/src/utils/cutils/utils.c -+++ b/src/utils/cutils/utils.c -@@ -347,7 +347,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c - return -1; - } - // sleep some time instead to avoid cpu full running and then retry. -- sleep(0.1); -+ usleep(0.1); - } - return 0; - } --- -2.25.1 - diff --git a/0024-adapt-to-repo-of-openeuler-url-changed.patch b/0024-adapt-to-repo-of-openeuler-url-changed.patch deleted file mode 100644 index 84a99ef..0000000 --- a/0024-adapt-to-repo-of-openeuler-url-changed.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 7a2f218550735403319a0bea6c47a0c334838a12 Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Mon, 6 Mar 2023 14:38:58 +0800 -Subject: [PATCH 24/26] adapt to repo of openeuler url changed - -Signed-off-by: haozi007 ---- - CI/pr-gateway.sh | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh -index 604ec6cb..93b07c44 100755 ---- a/CI/pr-gateway.sh -+++ b/CI/pr-gateway.sh -@@ -18,7 +18,15 @@ if [ $# -eq 1 ]; then - tbranch=$1 - fi - -+sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /etc/yum.repos.d/openEuler.repo -+ -+dnf update -y -+ - dnf install -y gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath -+if [ $? -ne 0 ]; then -+ echo "install dependences failed" -+ exit 1 -+fi - - # dnf install -y cargo rust rust-packaging - --- -2.25.1 - diff --git a/0025-modify-sleep-time.patch b/0025-modify-sleep-time.patch deleted file mode 100644 index 01f44e3..0000000 --- a/0025-modify-sleep-time.patch +++ /dev/null @@ -1,26 +0,0 @@ -From c2bf76c3b6af0d88d84a76cd5680caf0aa22e321 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 6 Mar 2023 15:34:05 +0800 -Subject: [PATCH 25/26] modify sleep time - -Signed-off-by: zhongtao ---- - src/utils/cutils/utils.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c -index 64d7e9f9..7f36d019 100644 ---- a/src/utils/cutils/utils.c -+++ b/src/utils/cutils/utils.c -@@ -347,7 +347,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c - return -1; - } - // sleep some time instead to avoid cpu full running and then retry. -- usleep(0.1); -+ usleep(100); - } - return 0; - } --- -2.25.1 - diff --git a/0026-change-goto-branch.patch b/0026-change-goto-branch.patch deleted file mode 100644 index f4075a7..0000000 --- a/0026-change-goto-branch.patch +++ /dev/null @@ -1,27 +0,0 @@ -From e0c800749961cf9f97b6a767ea3f7628a568a33d Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 6 Mar 2023 17:22:16 +0800 -Subject: [PATCH 26/26] change goto branch - -Signed-off-by: zhongtao ---- - src/daemon/executor/container_cb/execution_stream.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c -index 1a7353b5..7e928cf7 100644 ---- a/src/daemon/executor/container_cb/execution_stream.c -+++ b/src/daemon/executor/container_cb/execution_stream.c -@@ -161,7 +161,8 @@ static int container_exec_cb(const container_exec_request *request, container_ex - - if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) { - ret = -1; -- goto pack_err_response; -+ // pack err response in exec_container, there is no need to pack here. -+ goto out; - } - - goto out; --- -2.25.1 - diff --git a/0027-modifying-cpurt-file-permissions.patch b/0027-modifying-cpurt-file-permissions.patch deleted file mode 100644 index 52914f2..0000000 --- a/0027-modifying-cpurt-file-permissions.patch +++ /dev/null @@ -1,72 +0,0 @@ -From eb46344fd8b7d42e6268353bfc801a1a9c8cb9a3 Mon Sep 17 00:00:00 2001 -From: songbuhuang <544824346@qq.com> -Date: Thu, 9 Mar 2023 19:38:09 +0800 -Subject: [PATCH 27/46] modifying cpurt file permissions - -Signed-off-by: songbuhuang <544824346@qq.com> ---- - src/common/constants.h | 4 ++++ - src/daemon/common/sysinfo.c | 3 +-- - src/daemon/executor/container_cb/execution.c | 3 ++- - 3 files changed, 7 insertions(+), 3 deletions(-) - -diff --git a/src/common/constants.h b/src/common/constants.h -index 93a069de..1a4cb7c4 100644 ---- a/src/common/constants.h -+++ b/src/common/constants.h -@@ -22,6 +22,10 @@ extern "C" { - - /* mode of file and directory */ - -+#define DEFAULT_CGROUP_FILE_MODE 0644 -+ -+#define DEFAULT_CGROUP_DIR_MODE 0755 -+ - #define DEFAULT_SECURE_FILE_MODE 0640 - - #define DEFAULT_SECURE_DIRECTORY_MODE 0750 -diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c -index 8ad92b2a..cb02bee3 100644 ---- a/src/daemon/common/sysinfo.c -+++ b/src/daemon/common/sysinfo.c -@@ -24,6 +24,7 @@ - #include - #include - -+#include "constants.h" - #include "err_msg.h" - #include "isula_libutils/log.h" - #include "utils.h" -@@ -71,8 +72,6 @@ - - #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" - #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" --#define DEFAULT_CGROUP_DIR_MODE 0755 --#define DEFAULT_CGROUP_FILE_MODE 0644 - #define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" - #define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" - #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" -diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c -index 92c34b09..130bdaa4 100644 ---- a/src/daemon/executor/container_cb/execution.c -+++ b/src/daemon/executor/container_cb/execution.c -@@ -46,6 +46,7 @@ - #include "isulad_config.h" - #include "specs_api.h" - #include "container_api.h" -+#include "constants.h" - #include "execution_extend.h" - #include "execution_information.h" - #include "execution_stream.h" -@@ -318,7 +319,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const - return 0; - } - -- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); -+ ret = util_mkdir_p(path, DEFAULT_CGROUP_DIR_MODE); - if (ret != 0) { - ERROR("Failed to mkdir: %s", path); - return -1; --- -2.25.1 - diff --git a/0028-add-design-docs-for-cri-manager.patch b/0028-add-design-docs-for-cri-manager.patch deleted file mode 100644 index 56c65d3..0000000 --- a/0028-add-design-docs-for-cri-manager.patch +++ /dev/null @@ -1,492 +0,0 @@ -From 5cab44aa7709c743a1968ea5ac44863e6cf5434c Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Thu, 16 Mar 2023 11:41:43 +0800 -Subject: [PATCH 28/46] add design docs for cri manager - -Signed-off-by: haozi007 ---- - docs/cri_pod_manager_design.md | 432 ++++++++++++++++++ - docs/design/isulad_cri_listcontainerstats.svg | 1 + - docs/design/isulad_cri_remove_pod.svg | 1 + - docs/design/isulad_cri_run_pod.svg | 1 + - docs/design/isulad_cri_stop_pod.svg | 1 + - 5 files changed, 436 insertions(+) - create mode 100644 docs/cri_pod_manager_design.md - create mode 100644 docs/design/isulad_cri_listcontainerstats.svg - create mode 100644 docs/design/isulad_cri_remove_pod.svg - create mode 100644 docs/design/isulad_cri_run_pod.svg - create mode 100644 docs/design/isulad_cri_stop_pod.svg - -diff --git a/docs/cri_pod_manager_design.md b/docs/cri_pod_manager_design.md -new file mode 100644 -index 00000000..aa7ac165 ---- /dev/null -+++ b/docs/cri_pod_manager_design.md -@@ -0,0 +1,432 @@ -+| Author | 刘昊 | -+| ------ | ------------------------------------------ | -+| Date | 2023-03-06 | -+| Email | [liuhao27@huawei.com](liuhao27@huawei.com) | -+ -+本文主要对 `iSulad` 的 `CRI` 实现部分进行源码级别的学习和研究，为后续想了解 `CRI` 或者 `iSulad` 的同学提供帮助。 -+ -+*注：本文以iSulad 2.0.18版本为分析对象。* -+ -+本文主要是针对 `CRI` 的 `RuntimeService` 部分，涉及如下接口： -+- `RunPodSandbox` -+- `StopPodSandbox` -+- `RemovePodSandbox` -+- `PodSandboxStatus` -+- `ListPodSandbox` -+- `UpdateRuntimeConfig` -+- `Status` -+- `ContainerStats` -+- `ListContainerStats` -+ -+## CRI 代码结构 -+ -+CRI 涉及代码文件如下： -+ -+```c -+// CRI proto文件 -+src/api/services/cri/api.proto -+// proto 文件解析生成入口 -+cmake/protoc.cmake -+ -+// grpc服务注册入口 -+src/daemon/entry/connect/grpc/grpc_service.cc -+// cri 镜像部分的封装类 -+src/daemon/entry/connect/grpc/runtime_image_service.h -+// cri runtime 部分的封装类 -+src/daemon/entry/connect/grpc/runtime_runtime_service.h -+ -+// cri 具体实现目录 -+src/daemon/entry/cri/ -+``` -+ -+### CRI gRPC 服务注册和封装 -+ -+iSulad `gRPC` 模式下，直接在 `gRPC` 服务初始化时，注册了 `CRI` 服务： -+ -+```c -+// src/daemon/entry/connect/grpc/grpc_service.cc -+int Init(const struct service_arguments *args) -+{ -+ // 注册CRI runtime服务 -+ m_builder.RegisterService(&m_runtimeRuntimeService); -+ // 注册CRI 镜像服务 -+ m_builder.RegisterService(&m_runtimeImageService); -+} -+``` -+ -+`CRI` 服务初始化： -+ -+```c -+// src/daemon/entry/connect/grpc/runtime_runtime_service.cc -+void RuntimeRuntimeServiceImpl::Init(Network::NetworkPluginConf mConf, isulad_daemon_configs *config, Errors &err) -+{ -+ ... -+ // 初始化 CNI 网络插件 -+ Network::ProbeNetworkPlugins(mConf.GetPluginConfDir(), mConf.GetPluginBinDir(), &plugins); -+ Network::InitNetworkPlugin(&plugins, mConf.GetPluginName(), mConf.GetHairpinMode(), mConf.GetNonMasqueradeCIDR(), mConf.GetMTU(), &chosen, err); -+ -+ // 初始化，CRI 服务具体实现的类 -+ RuntimeVersionerService *runtimeVersioner = new RuntimeVersionerServiceImpl(cb); -+    ContainerManagerService *containerManager = new ContainerManagerServiceImpl(cb); -+    PodSandboxManagerService *podSandboxManager = new PodSandboxManagerServiceImpl(podSandboxImage, cb, pluginManager); -+    RuntimeManagerService *runtimeManager = new RuntimeManagerServiceImpl(cb, pluginManager); -+    std::unique_ptr service( -+        new CRIRuntimeServiceImpl(runtimeVersioner, containerManager, podSandboxManager, runtimeManager)); -+    rService = std::move(service); -+    // 初始化websocket服务 -+    websocket_server_init(err); -+ ... -+} -+``` -+ -+由于 `CRI` 具体实现类，定义在 `src/daemon/entry/cri/` 目录中；iSulad 通过封装的方式，在 `grpc` 服务提供了统一的入口，包括镜像和runtime两个封装类： -+- src/daemon/entry/connect/grpc/runtime_image_service.cc -+- src/daemon/entry/connect/grpc/runtime_runtime_service.cc -+ -+## PodSandbox 相关接口 -+ -+第一部分，对 `CRI` 的新增的 `Pod` 相关的接口进行详细的探讨。 -+ -+### RunPodSandbox -+ -+`RunPodSandbox` 负责 `Pod` 的创建、启动，以及网络环境准备等操作。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call RunPod -+ CRI-isula ->> iSulad: create container -+ CRI-isula ->> iSulad: inspect container -+ CRI-isula ->> cni plugin: prepare network -+ cni plugin -->> CRI-isula: return -+ CRI-isula ->> iSulad: start container -+ iSulad -->> CRI-isula: return -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+// src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -+auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler, Errors &error) -> std::string -+{ -+ ... -+ // Step1：根据Pod镜像策略，如果镜像不存在，是否下载 -+ EnsureSandboxImageExists() -+ // Step2：根据Pod配置，生成Pod对应容器的配置； -+ // 根据runtime选择对应的容器运行时； -+ CreateSandboxContainer() -+ // Step3：标识Pod网络状态为false -+ SetNetworkReady() -+ // Step4：通过inspect获取Pod信息，用于后续流程； -+ // 以及判定Pod状态是否符合预期； -+ InspectContainer() -+ // Step5：判断网络模式为file，是则先创建网络命名空间； -+ // 并且调用CNI插件，设置网络设备； -+ util_mount_namespace() -+ SetupSandboxNetwork() -+ // Step6：启动Pod -+ StartSandboxContainer() -+ // Step7：非file网络模式，则使用容器启动创建的网络名空间； -+ // 然后调用CNI插件，设置网络设备； -+ SetupSandboxNetwork() -+ ... -+} -+``` -+ -+详细的代码走读，代码流程图如下： -+ -+runpod -+ -+### StopPodSandbox -+ -+`StopPodSandbox` 负责 `Pod` 的停止，以及网络资源的清理。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call StopPod -+ CRI-isula ->> iSulad: get container info -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: stop all containers in pod -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> cni plugin: clear cni networks -+ cni plugin -->> CRI-isula: return -+ CRI-isula ->> iSulad: stop pod -+ iSulad -->> CRI-isula: return -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+void PodSandboxManagerServiceImpl::StopPodSandbox(const std::string &podSandboxID, Errors &error) -+{ -+ // Step1: get real pod ID -+ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); -+ // Step2: stop all containers in pod -+ StopAllContainersInSandbox(realSandboxID, error); -+ // Step3: clear cni network -+ ClearCniNetwork(realSandboxID, hostNetwork, ns, name, errlist, stdAnnos, error); -+ // Step4: stop pod -+ StopContainerHelper(realSandboxID, error); -+} -+``` -+ -+详细的代码走读，代码流程图如下： -+ -+stoppod -+ -+### RemovePodSandbox -+ -+`RemovePodSandbox` 负责删除 `Pod` ，包括对应容器的删除和其他数据的清理。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call RemovePod -+ CRI-isula ->> iSulad: get container info -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: remove all containers in pod -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: remove pod -+ iSulad -->> CRI-isula: return -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+void PodSandboxManagerServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Errors &error) -+{ -+ // Step1: get real pod ID -+ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); -+ // Step2: remove all containers in pod -+ RemoveAllContainersInSandbox(realSandboxID, errors); -+ // Step3: remove pod -+ DoRemovePodSandbox(realSandboxID, errors); -+} -+``` -+ -+详细的代码走读，代码流程图如下： -+ -+removepod -+ -+### PodSandboxStatus -+ -+`PodSandboxStatus` 负责获取 `Pod` 的状态信息，包括网络信息、创建时间、state、meta-data等等数据。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call Pod status -+ CRI-isula ->> iSulad: get pod real ID -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: call Inspect Pod -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set data from inspect response -+ CRI-isula ->> CNI: call network status -+ CNI -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set network ip and so on -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+std::unique_ptr -+PodSandboxManagerServiceImpl::PodSandboxStatus(const std::string &podSandboxID, Errors &error) -+{ -+ // Step1: get real pod ID -+ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); -+ // Step2: inspect pod sandbox -+ InspectContainer(realSandboxID, error, true); -+ // Step3: set status from inspect reponse -+ PodSandboxStatusToGRPC(inspect, realSandboxID, podStatus, error); -+ // Step 3.1 set timestamp -+ podStatus->set_created_at(createdAt); -+ // Step 3.2 set state -+ podStatus->set_state(runtime::v1alpha2::SANDBOX_READY); -+ // Step 3.3 set labels and annotations -+ CRIHelpers::ExtractLabels(inspect->config->labels, *podStatus->mutable_labels()); -+ CRIHelpers::ExtractAnnotations(inspect->config->annotations, *podStatus->mutable_annotations()); -+ // Step 3.4 set options -+ options->set_network(SharesHostNetwork(inspect)); -+ options->set_pid(SharesHostPid(inspect)); -+ options->set_ipc(SharesHostIpc(inspect)); -+ // Step 3.5 set network status -+ SetSandboxStatusNetwork(inspect, podSandboxID, podStatus, error); -+} -+``` -+ -+详细的代码走读，代码流程图在 [StopPod代码走读中](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 -+ -+## 通用接口 -+ -+### UpdateRuntimeConfig -+ -+`UpdateRuntimeConfig` 负责对运行配置进行更新，当前只有 `CIDR` 的配置更新。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call update runtime config -+ CRI-isula ->> network plugin: update -+ network plugin -->> cni plugin: update -+ cni plugin ->> network plugin: return -+ network plugin -->> CRI-isula: return -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+// src/daemon/entry/cri/cri_runtime_manager_service_impl.cc -+void RuntimeManagerServiceImpl::UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors & /*error*/) -+{ -+ // 设置用户传入的新CIDR值 -+ events[NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = config.network_config().pod_cidr(); -+ // 调用网络插件 --> CNI 网络模块，刷新CIDR配置 -+ m_pluginManager->Event(NET_PLUGIN_EVENT_POD_CIDR_CHANGE, events); -+} -+``` -+ -+代码结构比较简单，只是更新了类的一个成员的值；因此，不做详细的分析。 -+ -+### Status -+ -+`Status` 接口，主要用于获取容器引擎的 和 网络模块的状态是否正常。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call Status -+ CRI-isula ->> iSulad: call version -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set state of runtime -+ CRI-isula ->> network plugin: Status -+ network plugin -->> cni plugin: Status -+ cni plugin ->> network plugin: return -+ network plugin -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set state of network -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+// src/daemon/entry/cri/cri_runtime_manager_service_impl.cc -+auto RuntimeManagerServiceImpl::Status(Errors &error) -> std::unique_ptr -+{ -+ // 获取isulad的版本信息 -+ m_cb->container.version(nullptr, &response); -+ // 设置对应的状态 -+ runtimeReady->set_status(true/false); -+ // 调用网络插件的Status -+ m_pluginManager->Status(error); -+ // 设置对应的状态信息 -+ networkReady->set_status(true/false); -+} -+``` -+ -+代码结构比较简单，只是获取两个模块的状态，然后设置返回值；因此，不做详细的分析。 -+ -+## 容器相关接口 -+ -+### ContainerStats -+ -+`ContainerStats` 负责获取容器的度量数据。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call ContainerStats -+ CRI-isula ->> iSulad: call container.stats -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: call inspect -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set meta-data -+ CRI-isula ->> CRI-isula: set labels and annotations -+ CRI-isula ->> CRI-isula: set file usage -+ CRI-isula ->> CRI-isula: set memory usage -+ CRI-isula ->> CRI-isula: set cpu usage -+ CRI-isula -->> kubelet: return -+``` -+ -+源码实现，概要逻辑如下： -+ -+```c -+// src/daemon/entry/cri/cri_container_manager_service_impl.cc -+auto ContainerManagerServiceImpl::ContainerStats(const std::string &containerID, Errors &error) -+-> std::unique_ptr -+{ -+ // 获取容器的stats信息 -+ m_cb->container.stats(request, &response); -+ // 转换返回信息为CRI 对应的结果 -+ ContainerStatsToGRPC(response, &contStatsVec, error); -+ // inspect容器，获取meta-data信息，设置到返回值中 -+ PackContainerStatsAttributes(response->container_stats[i]->id, container, error); -+ // 获取容器使用的文件系统大小等度量数据 -+ PackContainerStatsFilesystemUsage(response->container_stats[i]->id, response->container_stats[i]->image_type, timestamp, container); -+ // 设置内存度量数据 -+ container->mutable_memory()->mutable_working_set_bytes()->set_value(workingset); -+ container->mutable_memory()->set_timestamp(timestamp); -+ // 设置CPU度量数据 -+ container->mutable_cpu()->mutable_usage_core_nano_seconds()->set_value(response->container_stats[i]->cpu_use_nanos); -+ container->mutable_cpu()->set_timestamp(timestamp); -+} -+``` -+ -+ -+由于 `ContainerStats` 和 `ListContainerStats` 实现基本一致，因此，详细分析部分放到一起分析，具体见 `ListContainerStats` 章节。 -+ -+### ListContainerStats -+ -+`ListContainerStats` 负责获取多容器的度量数据。 -+ -+相关组件的序列图如下： -+ -+```mermaid -+sequenceDiagram -+ kubelet ->> CRI-isula: call ContainerStats -+ CRI-isula ->> iSulad: call container.stats with filter -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> iSulad: call inspect -+ iSulad -->> CRI-isula: return -+ CRI-isula ->> CRI-isula: set meta-data -+ CRI-isula ->> CRI-isula: set labels and annotations -+ CRI-isula ->> CRI-isula: set file usage -+ CRI-isula ->> CRI-isula: set memory usage -+ CRI-isula ->> CRI-isula: set cpu usage -+ CRI-isula -->> kubelet: return -+``` -+ -+ 跟 `ContainerStats` 唯一的区别是，调用 `container->stats()` 的参数不同，`ListContainerStats` 根据过滤器去获取需要采集度量数据的容器列表，而 `ContainerStats` 只采集确定容器ID 的度量数据。 -+ -+源码实现，概要逻辑如下： -+ -+```c -+// src/daemon/entry/cri/cri_container_manager_service_impl.cc -+void ContainerManagerServiceImpl::ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter, std::vector> *containerstats, Errors &error) -+{ -+ // 根据用户参数，设置stats的过滤条件 -+ PackContainerStatsFilter(filter, request, error); -+ // 获取容器的stats信息 -+ m_cb->container.stats(request, &response); -+ // 转换返回信息为CRI 对应的结果 --> 与ContainerStats逻辑相同 -+ ContainerStatsToGRPC(response, &contStatsVec, error); -+} -+``` -+ -+详细的代码走读，代码流程图如下： -+ -+listcontainerstats -+ -+ -diff --git a/docs/design/isulad_cri_listcontainerstats.svg b/docs/design/isulad_cri_listcontainerstats.svg -new file mode 100644 -index 00000000..8781c39a ---- /dev/null -+++ b/docs/design/isulad_cri_listcontainerstats.svg -@@ -0,0 +1 @@ -+
ListContainerStats
PackContainerStatsFilter
call container.stats()
error?
ContainerStatsToGRPC
No
return error
Yes
sandbox id
empty?
add sandbox id filter
Yes
add type container lable
foreach selectors
finish?
add selector label
No
error?
return error
Yes
No
Yes
Yes
foreach return stats list
finish?
PackContainerStatsAttributes
No
error?
PackContainerStatsFilesystemUsage
No
mem_used
!= 0
set memory stats
nano_cpu
!= 0
set cpu stats
Yes
Yes
No
No
ContainerStatus
GetRealContainerOrSandboxID
InspectContainer
return error
Yes
return
return null
?
Yes
error?
Yes
ContainerStatusToGRPC
No
Yes
call container.get_id()
error?
Yes
got real sandbox ID
No
GetContainerTimeStamps
error?
Yes
set container start/create/finish time
No
PackContainerImageToStatus
UpdateBaseStatusFromInspect
PackLabelsToStatus
ParseContainerName
return
null?
Yes
Yes
No
has
meta?
No
add meta-data
Yes
No
label size
> 0
add labels
Yes
annotation
size > 0
add annotations
BUG：应该直接检查error
ConvertMountsToStatus
Yes
No
error?
ContainerStats
set containerID filter
 -\ No newline at end of file -diff --git a/docs/design/isulad_cri_remove_pod.svg b/docs/design/isulad_cri_remove_pod.svg -new file mode 100644 -index 00000000..f470bd17 ---- /dev/null -+++ b/docs/design/isulad_cri_remove_pod.svg -@@ -0,0 +1 @@ -+
RemovePodSandbox
GetRealContainerOrSandboxID
error?
not found
error?
Yes
return error
No
RemoveAllContainersInSandbox
No
Yes
call  container.list()
foreach return container list
RemoveContainer
DoRemovePodSandbox
error?
add error message
GetRealContainerOrSandboxID
error?
Yes
RemoveContainerLogSymlink
No
GetContainerLogPath
InspectContainer
error?
remove log path
No
call container.remove()
Yes
Yes
error?
Yes
No
cal container.remove()
pod-container
error?
ClearNetworkReady
No
not found
error?
Yes
Yes
return error
No
return
 -\ No newline at end of file -diff --git a/docs/design/isulad_cri_run_pod.svg b/docs/design/isulad_cri_run_pod.svg -new file mode 100644 -index 00000000..87ffad73 ---- /dev/null -+++ b/docs/design/isulad_cri_run_pod.svg -@@ -0,0 +1 @@ -+
RunPodSandbox
EnsureSandboxImageExists
ImageStatus
exist?
PullImage
NO
CreateSandboxContainer
Yes
MakeSandboxIsuladConfig
GenerateSandboxCreateContainerRequest
add sandbox
add pod name
add container type
add sandbox namespace 
add sandbox name
add UID
add  ATTEMPT
labels
annotations
has metadata?
Yes
set network mode
No
has linux options?
ApplySandboxLinuxOptions
Yes
ApplySandboxResources
No
has security options?
add security_opt
into hostconfig
Yes
CreateCheckpoint
save into annotation of pod
PackCreateContainerRequest
MakeSandboxName
parse runtime handler
set image
set host config
set custom config
ApplySandboxSecurityContext
run_as_user?
Yes
set user
set privileged
set readonly rootfs
No
has capabilities?
has cap
add?
set cap_adds in hostconfig
Yes
has cap
drop?
No
set cap_drops in hostconfig
Yes
Yes
add no-new-privileged into security opts
no new
privs?
Yes
No
has supplemental
groups?
No
add group_add into hostconfig
Yes
error?
No
ModifySandboxNamespaceOptions
No
return error
Yes
set pid-mode
set ipc-mode
set uts-mode
set network-mode
set cgroup_parent
cgroup parent?
Yes
sysctl size
> 0
No
add sysctls into hostconfig
Yes
No
No
call executor->create()
error?
return error
Yes
SetNetworkReady false
No
InspectContainer
ns is
file?
create new netns
Yes
SetupSandboxNetwork
StartSandboxContainer
No
ns is
file?
return response
Yes
No
SetNetworkReady true
error?
No
Yes
SetupSandboxNetwork
error?
No
StopContainerHelper
Yes
has dns?
SetupSandboxFiles
Yes
is NODE
network?
return
Yes
No
ns is
file?
No
add sandbox_key into annotations
call NetworkPlugin
SetUpPod
No
Yes
Lock Pod
call CNI SetUpPod
error?
No
return error
UnLock pod
cni inited?
No
Yes
AddToNetwork loop network
Yes
error?
Yes
SetupMultNetworks
No
TearDownMultNetworks
error?
Yes
foreach
AddToNetwork
user defined network
AddToNetwork
default network
return
No
SetupSandboxNetwork
call cni_add_network_list
 -\ No newline at end of file -diff --git a/docs/design/isulad_cri_stop_pod.svg b/docs/design/isulad_cri_stop_pod.svg -new file mode 100644 -index 00000000..b0549958 ---- /dev/null -+++ b/docs/design/isulad_cri_stop_pod.svg -@@ -0,0 +1 @@ -+
 
StopPodSandbox
empty
podID?
return
GetRealSandboxIDToStop
Yes
No
PodSandboxStatus
GetRealContainerOrSandboxID
call cb->get_id
error?
return
Yes
check input id is prefix of return id
valid
No
No
InspectContainer
return podID


PodSandboxStatusToGRPC


fill podStatus from inspect response:

1. fill pod ID;
2. fill created time;
3. fill state;
4. fill labels;
5. fill annotations;
6. fill meta-data: name, namespace, uid, attempt;
7. fill options: network, pid, ipc;
SetSandboxStatusNetwork
GetIPs
size of ip
list > 0
StopAllContainersInSandbox
No
error?
Yes
No found
error?
set real PodID to input ID
return error
No
Yes
has options
?
No
return hostNetwork
has meta
data?
return ns and name
Yes
return annotations
No
FiltersAddLabel
call container.list()
foreach return containers
error?
return error
Yes
No
StopContainer
GetRealContainerOrSandboxID
error?
Yes
error?
Yes
call container.stop()
No
No
ClearCniNetwork
StopContainerHelper
host
network?
network
ready?
No
InspectContainer
Yes
TearDownPod
SetNetworkReady false
network
type is file?
clear network namespace
Yes
Lock()
failed?
CNI TearDownPod
Unlock()
No
Yes
return 0
Yes
No
ignore error,
success forever
inited?
No
GetNetNSPath
RLockNetworkMap
error?
Yes
Yes
TearDownMultNetworks
UnlockNetworkMap
No
get user defined network list
error?
foreach user network


DeleteFromNetwork


DeleteFromNetwork
default network
Yes
No
BuildCNIRuntimeConf
error?
Get CNI bin Paths
null?
return
No
Yes
Yes
cni_del_network_list
call CNI plugin
DeleteFromNetwork
call container.stop()
finfish
StopPodSandbox
add_additional_ips
Yes
 -\ No newline at end of file --- -2.25.1 - diff --git a/0029-improve-check-of-process-failure.patch b/0029-improve-check-of-process-failure.patch deleted file mode 100644 index 05fc259..0000000 --- a/0029-improve-check-of-process-failure.patch +++ /dev/null @@ -1,43 +0,0 @@ -From 5a6112a35daa7229ffb03d0dbb1df0bd1dba3469 Mon Sep 17 00:00:00 2001 -From: haozi007 -Date: Thu, 16 Mar 2023 19:16:30 +0800 -Subject: [PATCH 29/46] improve check of process failure - -1. fix docs error; -2. check error to decrease Unnecessary process; - -Signed-off-by: haozi007 ---- - docs/cri_pod_manager_design.md | 2 +- - src/daemon/entry/cri/cri_container_manager_service_impl.cc | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/docs/cri_pod_manager_design.md b/docs/cri_pod_manager_design.md -index aa7ac165..91d74f5a 100644 ---- a/docs/cri_pod_manager_design.md -+++ b/docs/cri_pod_manager_design.md -@@ -263,7 +263,7 @@ PodSandboxManagerServiceImpl::PodSandboxStatus(const std::string &podSandboxID, - } - ``` - --详细的代码走读，代码流程图在 [StopPod代码走读中](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 -+详细的代码走读，代码流程图在 [StopPod代码走读](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 - - ## 通用接口 - -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index 93b939c9..0fc1884f 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -730,7 +730,7 @@ void ContainerManagerServiceImpl::PackContainerStatsAttributes( - - container->mutable_attributes()->set_id(id); - auto status = ContainerStatus(std::string(id), error); -- if (status == nullptr) { -+ if (error.NotEmpty()) { - return; - } - --- -2.25.1 - diff --git a/0030-support-isula-update-when-runtime-is-runc.patch b/0030-support-isula-update-when-runtime-is-runc.patch deleted file mode 100644 index 0be335e..0000000 --- a/0030-support-isula-update-when-runtime-is-runc.patch +++ /dev/null @@ -1,251 +0,0 @@ -From 0752a4324e7a8f54e4ebe5efb403221388b483d3 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 6 Mar 2023 11:31:56 +0800 -Subject: [PATCH 30/46] support isula update when runtime is runc - -Signed-off-by: zhongtao ---- - .../executor/container_cb/execution_extend.c | 1 + - src/daemon/modules/api/runtime_api.h | 1 + - .../modules/runtime/isula/isula_rt_ops.c | 167 +++++++++++++++++- - 3 files changed, 163 insertions(+), 6 deletions(-) - -diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c -index b0da705e..58303f80 100644 ---- a/src/daemon/executor/container_cb/execution_extend.c -+++ b/src/daemon/executor/container_cb/execution_extend.c -@@ -1129,6 +1129,7 @@ static int do_update_resources(const container_update_request *request, containe - if (container_is_running(cont->state)) { - params.rootpath = cont->root_path; - params.hostconfig = hostconfig; -+ params.state = cont->state_path; - if (runtime_update(id, cont->runtime, ¶ms)) { - ERROR("Update container %s failed", id); - ret = -1; -diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h -index c5e05ca2..a8dfdeae 100644 ---- a/src/daemon/modules/api/runtime_api.h -+++ b/src/daemon/modules/api/runtime_api.h -@@ -164,6 +164,7 @@ typedef struct _rt_attach_params_t { - typedef struct _rt_update_params_t { - const char *rootpath; - const host_config *hostconfig; -+ const char *state; - } rt_update_params_t; - - typedef struct _rt_listpids_params_t { -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index 51a72c4f..4553fa90 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -39,6 +39,7 @@ - #include "constants.h" - #include "isula_libutils/shim_client_process_state.h" - #include "isula_libutils/shim_client_runtime_stats.h" -+#include "isula_libutils/shim_client_cgroup_resources.h" - #include "isula_libutils/oci_runtime_state.h" - #include "isulad_config.h" - #include "utils_string.h" -@@ -54,6 +55,9 @@ - #define RESIZE_DATA_SIZE 100 - #define PID_WAIT_TIME 120 - -+// file name formats of cgroup resources json -+#define RESOURCE_FNAME_FORMATS "%s/resources.json" -+ - // handle string from stderr output. - typedef int(*handle_output_callback_t)(const char *output); - -@@ -725,18 +729,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - runtime_exec_param_dump(params); - - if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { -- ERROR("failed make shim-pid full path"); -+ ERROR("Failed make shim-pid full path"); - return -1; - } - - if (pipe2(exec_fd, O_CLOEXEC) != 0) { -- ERROR("failed to create pipe for shim create"); -+ ERROR("Failed to create pipe for shim create"); - return -1; - } - - pid = fork(); - if (pid < 0) { -- ERROR("failed fork for shim parent %s", strerror(errno)); -+ ERROR("Failed fork for shim parent %s", strerror(errno)); - close(exec_fd[0]); - close(exec_fd[1]); - return -1; -@@ -1256,13 +1260,164 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ - return -1; - } - --int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) -+static int to_engine_resources(const host_config *hostconfig, shim_client_cgroup_resources *cr) -+{ -+ uint64_t period = 0; -+ int64_t quota = 0; -+ -+ if (hostconfig == NULL || cr == NULL) { -+ return -1; -+ } -+ -+ cr->block_io = util_common_calloc_s(sizeof(shim_client_cgroup_resources_block_io)); -+ if (cr->block_io == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ -+ cr->cpu = util_common_calloc_s(sizeof(shim_client_cgroup_resources_cpu)); -+ if (cr->cpu == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ -+ cr->memory = util_common_calloc_s(sizeof(shim_client_cgroup_resources_memory)); -+ if (cr->memory == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ -+ cr->block_io->weight = hostconfig->blkio_weight; -+ cr->cpu->shares = (uint64_t)hostconfig->cpu_shares; -+ cr->cpu->period = (uint64_t)hostconfig->cpu_period; -+ cr->cpu->quota = hostconfig->cpu_quota; -+ cr->cpu->cpus = util_strdup_s(hostconfig->cpuset_cpus); -+ cr->cpu->mems = util_strdup_s(hostconfig->cpuset_mems); -+ cr->memory->limit = (uint64_t)hostconfig->memory; -+ cr->memory->swap = (uint64_t)hostconfig->memory_swap; -+ cr->memory->reservation = (uint64_t)hostconfig->memory_reservation; -+ cr->memory->kernel = (uint64_t)hostconfig->kernel_memory; -+ cr->cpu->realtime_period = hostconfig->cpu_realtime_period; -+ cr->cpu->realtime_runtime = hostconfig->cpu_realtime_runtime; -+ -+ // when --cpus=n is set, nano_cpus = n * 1e9. -+ if (hostconfig->nano_cpus > 0) { -+ // in the case, period will be set to the default value of 100000(0.1s). -+ period = (uint64_t)(100 * Time_Milli / Time_Micro); -+ // set quota = period * n, in order to let container process fully occupy n cpus. -+ if ((hostconfig->nano_cpus / 1e9) > (INT64_MAX / (int64_t)period)) { -+ ERROR("Overflow of quota"); -+ return -1; -+ } -+ quota = hostconfig->nano_cpus / 1e9 * (int64_t)period; -+ cr->cpu->period = period; -+ cr->cpu->quota = quota; -+ } -+ -+ return 0; -+} -+ -+static int create_resources_json_file(const char *workdir, const shim_client_cgroup_resources *cr, char *fname, -+ size_t fname_size) - { -- ERROR("isula update not support on isulad-shim"); -- isulad_set_error_message("isula update not support on isulad-shim"); -+ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; -+ parser_error perr = NULL; -+ char *data = NULL; -+ int retcode = 0; -+ int nret = 0; -+ -+ nret = snprintf(fname, fname_size, RESOURCE_FNAME_FORMATS, workdir); -+ if (nret < 0 || (size_t)nret >= fname_size) { -+ ERROR("Failed make resources.json full path"); -+ return -1; -+ } -+ -+ data = shim_client_cgroup_resources_generate_json(cr, &ctx, &perr); -+ if (data == NULL) { -+ retcode = -1; -+ ERROR("Failed generate json for resources.json error=%s", perr); -+ goto out; -+ } -+ -+ if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { -+ retcode = -1; -+ ERROR("Failed write resources.json"); -+ goto out; -+ } -+ -+out: -+ UTIL_FREE_AND_SET_NULL(perr); -+ UTIL_FREE_AND_SET_NULL(data); -+ -+ return retcode; -+} -+ -+// show std error msg, always return -1. -+static int show_stderr(const char *err) -+{ -+ isulad_set_error_message(err); - return -1; - } - -+int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) -+{ -+ int ret = 0; -+ char workdir[PATH_MAX] = { 0 }; -+ char resources_fname[PATH_MAX] = { 0 }; -+ const char *opts[2] = { 0 }; -+ shim_client_cgroup_resources *cr = NULL; -+ -+ if (id == NULL || runtime == NULL || params == NULL || params->state == NULL || strlen(params->state) == 0) { -+ ERROR("Nullptr arguments not allowed"); -+ return -1; -+ } -+ -+ ret = snprintf(workdir, sizeof(workdir), "%s/%s/update", params->state, id); -+ if (ret < 0 || (size_t)ret >= sizeof(workdir)) { -+ ERROR("Failed join update full path"); -+ return -1; -+ } -+ -+ ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE); -+ if (ret < 0) { -+ ERROR("Failed mkdir update workdir %s", workdir); -+ return ret; -+ } -+ -+ cr = util_common_calloc_s(sizeof(shim_client_cgroup_resources)); -+ if (cr == NULL) { -+ ERROR("Out of memory"); -+ goto del_out; -+ } -+ -+ ret = to_engine_resources(params->hostconfig, cr); -+ if (ret < 0) { -+ ERROR("Failed to get resources for update"); -+ goto del_out; -+ } -+ -+ ret = create_resources_json_file(workdir, cr, resources_fname, sizeof(resources_fname)); -+ if (ret != 0) { -+ ERROR("%s: failed create update json file", id); -+ goto del_out; -+ } -+ -+ opts[0] = "--resources"; -+ opts[1] = resources_fname; -+ -+ if (runtime_call_simple(workdir, runtime, "update", opts, 2, id, show_stderr) != 0) { -+ ERROR("Call runtime update id failed"); -+ ret = -1; -+ } -+ -+del_out: -+ if (util_recursive_rmdir(workdir, 0)) { -+ ERROR("Rmdir %s failed", workdir); -+ } -+ free_shim_client_cgroup_resources(cr); -+ return ret; -+} -+ - int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params) - { - char workdir[PATH_MAX] = { 0 }; --- -2.25.1 - diff --git a/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch b/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch deleted file mode 100644 index bd5c270..0000000 --- a/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch +++ /dev/null @@ -1,80 +0,0 @@ -From dd36a6031c3f25171d34c9f8cc483af01a4dace1 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 20 Mar 2023 14:49:07 +0800 -Subject: [PATCH 31/46] when calling runc start, unset NOTIFY_ SOCKET - -Signed-off-by: sailorvii ---- - src/daemon/modules/runtime/isula/isula_rt_ops.c | 14 ++++++++------ - 1 file changed, 8 insertions(+), 6 deletions(-) - -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index 4553fa90..e974964a 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -469,6 +469,12 @@ static void runtime_exec_func(void *arg) - _exit(EXIT_FAILURE); - } - -+ // clear NOTIFY_SOCKET from the env to adapt runc start -+ if (strcmp(rei->subcmd, "start") == 0 && unsetenv("NOTIFY_SOCKET") != 0) { -+ dprintf(STDERR_FILENO, "unset env NOTIFY_SOCKET failed %s", strerror(errno)); -+ _exit(EXIT_FAILURE); -+ } -+ - execvp(rei->cmd, rei->params); - dprintf(STDERR_FILENO, "exec %s %s %s failed", rei->cmd, rei->subcmd, rei->id); - _exit(EXIT_FAILURE); -@@ -941,7 +947,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t - char shim_pid_file_name[PATH_MAX] = { 0 }; - pid_t pid = 0; - pid_t shim_pid = -1; -- int ret = 0; -+ int ret = -1; - int splice_ret = 0; - proc_t *proc = NULL; - proc_t *p_proc = NULL; -@@ -963,28 +969,24 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t - - pid = get_container_process_pid(workdir); - if (pid < 0) { -- ret = -1; - ERROR("%s: failed wait init pid", id); - goto out; - } - - file_read_int(shim_pid_file_name, &shim_pid); - if (shim_pid < 0) { -- ret = -1; - ERROR("%s: failed to read isulad shim pid", id); - goto out; - } - - proc = util_get_process_proc_info(pid); - if (proc == NULL) { -- ret = -1; - ERROR("%s: failed to read pidinfo", id); - goto out; - } - - p_proc = util_get_process_proc_info(shim_pid); - if (p_proc == NULL) { -- ret = -1; - ERROR("%s: failed to read isulad shim pidinfo", id); - goto out; - } -@@ -996,10 +998,10 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t - - if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { - ERROR("call runtime start id failed"); -- ret = -1; - goto out; - } - -+ ret = 0; - out: - if (ret != 0) { - show_shim_runtime_errlog(workdir); --- -2.25.1 - diff --git a/0032-add-CRI-container-design-doc.patch b/0032-add-CRI-container-design-doc.patch deleted file mode 100644 index cc929c9..0000000 --- a/0032-add-CRI-container-design-doc.patch +++ /dev/null @@ -1,850 +0,0 @@ -From 5f75bec5b213eb85eda117ac912ddf4f94ff3277 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Mon, 20 Mar 2023 11:59:21 +0800 -Subject: [PATCH 32/46] add CRI container design doc - -Signed-off-by: zhangxiaoyu ---- - docs/cri_container_manager_design.md | 634 ++++++++++++++++++ - .../CRI/Attach_Flow_Chart.excalidraw.svg | 5 + - .../ContainerStatus_Flow_Chart.excalidraw.svg | 5 + - .../CreateContainer_Flow_Chart.excalidraw.svg | 5 + - .../CRI/ExecSync_Flow_Chart.excalidraw.svg | 5 + - .../design/CRI/Exec_Flow_Chart.excalidraw.svg | 5 + - .../ListContainers_Flow_Chart.excalidraw.svg | 5 + - .../RemoveContainer_Flow_Chart.excalidraw.svg | 5 + - .../StartContainer_Flow_Chart.excalidraw.svg | 5 + - .../StopContainer_Flow_Chart.excalidraw.svg | 5 + - ...ntainerResources_Flow_Chart.excalidraw.svg | 5 + - .../CRI/Websocket_Flow_Chart.excalidraw.svg | 5 + - ...Websocket_Server_Flow_Chart.excalidraw.svg | 5 + - ...Websocket_Stream_Flow_Chart.excalidraw.svg | 5 + - docs/design/CRI/Websocket_UML.excalidraw.svg | 5 + - 15 files changed, 704 insertions(+) - create mode 100644 docs/cri_container_manager_design.md - create mode 100644 docs/design/CRI/Attach_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/Exec_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg - create mode 100644 docs/design/CRI/Websocket_UML.excalidraw.svg - -diff --git a/docs/cri_container_manager_design.md b/docs/cri_container_manager_design.md -new file mode 100644 -index 00000000..5e8758b0 ---- /dev/null -+++ b/docs/cri_container_manager_design.md -@@ -0,0 +1,634 @@ -+| Author | 张晓雨 | -+| ------ | -----------------------------------------------------| -+| Date | 2023-03-20 | -+| Email | [zhangxiaoyu58@huawei.com](zhangxiaoyu58@huawei.com) | -+ -+本文主要对 `iSulad` 的 `CRI` 实现部分进行源码级别的学习和研究，为后续想了解 `CRI` 或者 `iSulad` 的同学提供帮助。 -+本文涉及`CRI`模块`Container`相关的接口。关于`Pod`相关接口的梳理请参考[cri_pod_manager_design.md](./cri_pod_manager_design.md)。 -+ -+*注：本文以iSulad 2.0.18版本为分析对象。* -+ -+本文主要是针对 `CRI` 的 `RuntimeService` 部分，涉及如下接口： -+- `CreateContainer` -+- `StartContainer` -+- `StopContainer` -+- `RemoveContainer` -+- `ListContainers` -+- `ContainerStatus` -+- `UpdateContainerResources` -+- `ExecSync` -+- `Exec` -+- `Attach` -+ -+## CreateContainer -+#### interface -+``` -+// CreateContainer creates a new container in specified PodSandbox -+rpc CreateContainer(CreateContainerRequest) returns (CreateContainerResponse) {} -+``` -+ -+#### Flow chart -+CreateContainer_Flow_Chart -+ -+#### Detail -+- GetRealContainerOrSandboxID -+get container from *g_containers_store* by (short) id, and filter container with label *cri.isulad.type=podSandbox* -+- GetContainerOrSandboxRuntime -+get podSandbox runtime -+- GenerateCreateContainerRequest -+set *container_create_request*, include -+ ```c -+ // name -+ std::string cname = CRINaming::MakeContainerName(podSandboxConfig, containerConfig) -+ // runtime -+ request->runtime = util_strdup_s(podSandboxRuntime.c_str()) -+ // image -+ request->image = util_strdup_s(containerConfig.image().image().c_str()) -+ // cgroup parent -+ hostconfig->cgroup_parent = util_strdup_s(podSandboxConfig.linux().cgroup_parent().c_str()) -+ // labels -+ custom_config->labels = CRIHelpers::MakeLabels(containerConfig.labels(), error) -+ // cri.isulad.type=container -+ append_json_map_string_string(custom_config->labels, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY.c_str(), CRIHelpers::Constants::CONTAINER_TYPE_LABEL_CONTAINER.c_str() -+ // log path -+ append_json_map_string_string(custom_config->labels, CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str(), real_logpath) -+ // annotations -+ CRIHelpers::MakeAnnotations(containerConfig.annotations(), error) -+ // mount -+ CRIHelpers::GenerateMountBindings(containerConfig.mounts(), hostconfig, error) -+ // device -+ PackCreateContainerHostConfigDevices(containerConfig, hostconfig, error) -+ // security context -+ PackCreateContainerHostConfigSecurityContext(containerConfig, hostconfig, error) -+ ... -+ ``` -+ -+## StartContainer -+#### interface -+``` -+// StartContainer starts the container. -+rpc StartContainer(StartContainerRequest) returns (StartContainerResponse) {} -+``` -+ -+#### Flow chart -+StartContainer_Flow_Chart -+ -+#### Detail -+- GetRealContainerOrSandboxID -+get container from *g_containers_store* by (short) id, and filter container with label *cri.isulad.type=container* -+- CreateContainerLogSymlink -+*inspect_container* , get *cri.container.logpath* annotation and container *log_path*, then create a symbolic from *log_path* to *cri.container.logpath* -+ -+## StopContainer -+#### interface -+``` -+// StopContainer stops a running container with a grace period (i.e., timeout). -+// This call is idempotent, and must not return an error if the container has -+// already been stopped. -+// The runtime must forcibly kill the container after the grace period is -+// reached. -+rpc StopContainer(StopContainerRequest) returns (StopContainerResponse) {} -+``` -+ -+#### Flow chart -+StopContainer_Flow_Chart -+ -+#### Detail -+- Timeout -+*timeout* in seconds to wait for the container to stop before forcibly terminating it, default value is 0 -+ -+## RemoveContainer -+#### interface -+``` -+// RemoveContainer removes the container. If the container is running, the -+// container must be forcibly removed. -+// This call is idempotent, and must not return an error if the container has -+// already been removed. -+rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse) {} -+``` -+ -+#### Flow chart -+RemoveContainer_Flow_Chart -+ -+#### Detail -+- RemoveContainerLogSymlink -+*inspect_container* , get *cri.container.logpath* annotation and container *log_path*, then delete *cri.container.logpath* -+ -+## ListContainers -+#### interface -+``` -+// ListContainers lists all containers by filters. -+rpc ListContainers(ListContainersRequest) returns (ListContainersResponse) {} -+``` -+ -+#### Flow chart -+ListContainers_Flow_Chart -+ -+#### Detail -+- ListContainersFromGRPC -+convert filter from GRPC request, and add *cri.isulad.type=container* to filter -+ ```c -+ // Add filter to get only non-sandbox containers -+ CRIHelpers::FiltersAddLabel((*request)->filters, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_CONTAINER) -+ // convert filter -+ // id -+ CRIHelpers::FiltersAdd((*request)->filters, "id", filter->id()) -+ // status -+ CRIHelpers::FiltersAdd((*request)->filters, "status", CRIHelpers::ToIsuladContainerStatus(filter->state())) -+ // sandbox id -+ CRIHelpers::FiltersAddLabel((*request)->filters, CRIHelpers::Constants::SANDBOX_ID_LABEL_KEY, filter->pod_sandbox_id()) -+ // label selector -+ for (auto &iter : filter->label_selector()) { -+ CRIHelpers::FiltersAddLabel((*request)->filters, iter.first, iter.second) -+ } -+ ``` -+- ListContainersToGRPC -+convert containers information to GRPC response, include -+ ```c -+ for (size_t i {}; i < response->containers_len; i++) { -+ // id -+ container->set_id(response->containers[i]->id) -+ // labels -+ CRIHelpers::ExtractLabels(response->containers[i]->labels, *container->mutable_labels()) -+ // annotations -+ CRIHelpers::ExtractAnnotations(response->containers[i]->annotations, *container->mutable_annotations()) -+ // name -+ CRINaming::ParseContainerName(container->annotations(), container->mutable_metadata(), error) -+ // image -+ image->set_image(response->containers[i]->image) -+ std::string imageID = CRIHelpers::ToPullableImageID(response->containers[i]->image, response->containers[i]->image_ref) -+ container->set_image_ref(imageID) -+ // runtime -+ CRIHelpers::ContainerStatusToRuntime(Container_Status(response->containers[i]->status)) -+ ... -+ } -+ ``` -+ -+## ContainerStatus -+#### interface -+``` -+// ContainerStatus returns status of the container. If the container is not -+// present, returns an error. -+rpc ContainerStatus(ContainerStatusRequest) returns (ContainerStatusResponse) {} -+``` -+ -+#### Flow chart -+ContainerStatus_Flow_Chart -+ -+#### Detail -+- ContainerStatusToGRPC -+add container information to GRPC response, include -+ ```c -+ // timestamps -+ CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error) -+ // image -+ PackContainerImageToStatus(inspect, contStatus, error) -+ // container status -+ UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus) -+ // labels -+ PackLabelsToStatus(inspect, contStatus) -+ // mount -+ ConvertMountsToStatus(inspect, contStatus); -+ ``` -+ -+## UpdateContainerResources -+#### interface -+``` -+// UpdateContainerResources updates ContainerConfig of the container synchronously. -+// If runtime fails to transactionally update the requested resources, an error is returned. -+rpc UpdateContainerResources(UpdateContainerResourcesRequest) returns (UpdateContainerResourcesResponse) {} -+``` -+ -+#### Flow chart -+UpdateContainerResources_Flow_Chart -+ -+#### Detail -+- Generate HostConfig -+set resources, include -+ ```c -+ // cpu -+ hostconfig->cpu_period = resources.cpu_period() -+ hostconfig->cpu_quota = resources.cpu_quota() -+ hostconfig->cpu_shares = resources.cpu_shares() -+ hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()) -+ hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()) -+ // memory -+ hostconfig->memory_swap_limit_in_bytes = resources.memory_swap_limit_in_bytes() -+ hostconfig->memory = resources.memory_limit_in_bytes() -+ // cgroupv2 unified -+ for (auto &iter : resources.unified()) { -+ append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) -+ } -+ // huge page -+ for (int i = 0; i < resources.hugepage_limits_size(); i++) { -+ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()) -+ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit() -+ hostconfig->hugetlbs_len++ -+ } -+ ``` -+ -+## ExecSync -+#### interface -+``` -+// ExecSync runs a command in a container synchronously. -+rpc ExecSync(ExecSyncRequest) returns (ExecSyncResponse) {} -+``` -+ -+#### Flow chart -+ExecSync_Flow_Chart -+ -+ -+#### Detail -+- ExecSyncFromGRPC -+set *container_exec_request*, include -+ ```c -+ (*request)->tty = false -+ (*request)->attach_stdin = false -+ (*request)->attach_stdout = true -+ (*request)->attach_stderr = true -+ (*request)->timeout = timeout -+ (*request)->container_id = util_strdup_s(containerID.c_str()) -+ // execution command -+ for (int i = 0; i < cmd.size(); i++) { -+ (*request)->argv[i] = util_strdup_s(cmd[i].c_str()) -+ (*request)->argv_len++ -+ } -+ // generate random suffix -+ (*request)->suffix = CRIHelpers::GenerateExecSuffix() -+ ``` -+- StringWriter -+set stdoutWriter and stderrWriter, write string to reply *stdout/stderr* -+ ```c -+ StdoutstringWriter.context = (void *)reply->mutable_stdout() -+ StdoutstringWriter.write_func = WriteToString -+ -+ StderrstringWriter.context = (void *)reply->mutable_stderr() -+ StderrstringWriter.write_func = WriteToString -+ -+ static auto WriteToString(void *context, const void *data, size_t len) -> ssize_t -+ { -+ if (len == 0) { -+ return 0; -+ } -+ -+ std::string *str = reinterpret_cast(context); -+ -+ str->append(reinterpret_cast(data), len); -+ return (ssize_t)len; -+ } -+ ``` -+ -+## Exec -+#### interface -+``` -+// Exec prepares a streaming endpoint to execute a command in the container. -+rpc Exec(ExecRequest) returns (ExecResponse) {} -+``` -+ -+#### Flow chart -+Exec_Flow_Chart -+ -+#### Detail -+- ValidateExecRequest -+1. check container is running -+2. check container is not paused -+3. check tty and stderr not both true -+4. check one of stdin, stdout, stderr must be true -+- GarbageCollection -+traverse from back of the *CacheEntry list* and remove the *expired requests*, also delete them in *token hash map* -+ ```c -+ void RequestCache::GarbageCollection() -+ { -+ auto now = std::chrono::system_clock::now(); -+ while (!m_ll.empty()) { -+ auto oldest = m_ll.back(); -+ if (now < oldest.expireTime) { -+ return; -+ } -+ if (oldest.req != nullptr) { -+ delete oldest.req; -+ oldest.req = nullptr; -+ } -+ m_ll.pop_back(); -+ m_tokens.erase(oldest.token); -+ } -+ } -+ ``` -+- UniqueToken -+`token=base64(random string))`. If token is exist in *token hash map*, retry generate token -+- Save -+save containerID, request, token in *CacheEntry list* and *token hash map* -+ -+## Attach -+#### interface -+``` -+// Attach prepares a streaming endpoint to attach to a running container. -+rpc Attach(AttachRequest) returns (AttachResponse) {} -+``` -+ -+#### Flow chart -+Attach_Flow_Chart -+ -+#### Detail -+- ValidateAttachRequest -+1. check tty and stderr not both true -+2. check one of stdin, stdout, stderr must be true -+ -+## Websocket -+ -+#### UML diagram -+Websocket_UML -+ -+### Start/Stop Websocket Server -+#### Flow chart -+Websocket_Server_Flow_Chart -+ -+#### Detail -+- CreateContext -+create websocket server context -+ ```c -+ /* -+ context->lws_lookup is allocated ( sizeof(struct lws *) * max_fds ) spaces, -+ In general, max_fds should be the process maximum number of open file descriptor. -+ If WS_ULIMIT_FDS set too large, context->lws_lookup will cost too much memory. -+ If WS_ULIMIT_FDS set too small, maybe fd > max_fds and context->lws_lookup[fd] will overflow. -+ */ -+ const size_t WS_ULIMIT_FDS { 10240 } -+ -+ info.port = m_listenPort -+ info.iface = "127.0.0.1" -+ info.protocols = m_protocols -+ info.ssl_cert_filepath = nullptr -+ info.ssl_private_key_filepath = nullptr -+ info.gid = -1 -+ info.uid = -1 -+ info.options = LWS_SERVER_OPTION_VALIDATE_UTF8 | LWS_SERVER_OPTION_DISABLE_IPV6 -+ info.max_http_header_pool = MAX_HTTP_HEADER_POOL -+ info.extensions = nullptr -+ -+ m_context = lws_create_context(&info) -+ ``` -+ -+### Websocket Main Callback -+#### Flow chart -+Websocket_Flow_Chart -+ -+#### Detail -+- LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION -+called when the handshake has been received and parsed from the client, but the response is not sent yet. Return non-zero to disallow the connection. -+ ```c -+ // Write Lock -+ WriteGuard lock(m_mutex) -+ -+ // get TOKEN URL -+ lws_hdr_copy(wsi, buf, sizeof(buf), WSI_TOKEN_GET_URI) -+ -+ // Validate URL -+ // URL format: "/cri/" + method + "/" + token + "/" + arg(container=cmd) -+ auto vec = CXXUtils::Split(buf + 1, '/'); -+ // method: exec and attach -+ // token is exist in token hash map -+ if vec.size() >= 3 && m_handler.IsValidMethod(vec.at(1)) && cache->IsValidToken(vec.at(2)) -+ -+ // socketID already exist ? -+ int socketID = lws_get_socket_fd(wsi) -+ if m_wsis.count(socketID) == 0 -+ -+ // too many connection sessions ? -+ if m_wsis.size() <= MAX_SESSION_NUM(128) -+ -+ // generate SessionData -+ GenerateSessionData(session, containerID) -+ -+ // save session -+ m_wsis.insert(std::make_pair(socketID, session)) -+ -+ // create StreamTask -+ std::thread streamTh([ = ]() { -+ StreamTask(&m_handler, session, vec.at(1), vec.at(2)).Run(); -+ }) -+ streamTh.detach() -+ -+ // Trigger polling in LWS_CALLBACK_SERVER_WRITEABLE -+ lws_callback_on_writable(wsi) -+ ``` -+- LWS_CALLBACK_SERVER_WRITEABLE -+If you call lws_callback_on_writable() on a connection, you will get one of these callbacks coming when the connection socket is able to accept another write packet without blocking. -+If it already was able to take another packet without blocking, you'll get this callback at the next call to the service loop function. -+ ```c -+ // Read Lock -+ ReadGuard lock(m_mutex) -+ -+ // get SessionData -+ int socketID = lws_get_socket_fd(wsi) -+ auto it = m_wsis.find(socketID) -+ if it != m_wsis.end() -+ -+ // record session closed -+ auto sessionClosed = it->second->IsClosed() -+ -+ // write message -+ while (!it->second->buffer.empty()) { -+ auto *message = it->second->FrontMessage() -+ // send success! free it and erase for list -+ WebsocketServer::GetInstance()->Wswrite(wsi, const_cast(message)) -+ free(message) -+ it->second->PopMessage() -+ } -+ -+ // return non-zero if session closed -+ if (sessionClosed) { -+ DEBUG("websocket session disconnected") -+ return -1 -+ } -+ -+ // return zero and callback with the reason of LWS_CALLBACK_SERVER_WRITEABLE again -+ lws_callback_on_writable(wsi) -+ ``` -+- LWS_CALLBACK_RECEIVE -+data has appeared for this server endpoint from a remote client, it can be found at in and is len bytes long -+ ```c -+ // Read Lock -+ ReadGuard lock(m_mutex) -+ -+ // get SessionData -+ auto it = m_wsis.find(socketID) -+ if it != m_wsis.end() -+ -+ // Last Stdin Uncomplete -+ if (!it->second->IsStdinComplete()) { -+ util_write_nointr_in_total(m_wsis[socketID]->pipes.at(1), (char *)in, len) -+ } -+ -+ // RESIZECHANNEL -+ if (*static_cast(in) == WebsocketChannel::RESIZECHANNEL) { -+ ParseTerminalSize(jsonData, len, width, height) -+ int ret = cb->container.resize(req, &res) -+ } -+ -+ // STDINCHANNEL -+ if (*static_cast(in) == WebsocketChannel::STDINCHANNEL) { -+ util_write_nointr_in_total(m_wsis[socketID]->pipes.at(1), (char *)in + 1, len - 1) -+ } -+ -+ // SetStdinComplete -+ size_t bytesLen = lws_remaining_packet_payload(wsi) -+ it->second->SetStdinComplete(bytesLen == 0) -+ ``` -+- LWS_CALLBACK_CLOSED -+when the websocket session ends -+ ```c -+ void WebsocketServer::CloseWsSession(int socketID) -+ { -+ // write lock -+ m_mutex.wrlock(); -+ auto it = m_wsis.find(socketID); -+ if (it == m_wsis.end()) { -+ m_mutex.unlock(); -+ return; -+ } -+ -+ auto session = it->second; -+ it->second = nullptr; -+ // delete session in map -+ m_wsis.erase(it); -+ m_mutex.unlock(); -+ -+ std::thread([session]() { -+ prctl(PR_SET_NAME, "WSSessionGC"); -+ session->CloseSession(); -+ session->EraseAllMessage(); -+ // close the pipe write endpoint first, make sure io copy thread exit, -+ // otherwise epoll will trigger EOF -+ if (session->pipes.at(1) >= 0) { -+ close(session->pipes.at(1)); -+ session->pipes.at(1) = -1; -+ } -+ (void)sem_wait(session->syncCloseSem); -+ (void)sem_destroy(session->syncCloseSem); -+ delete session->syncCloseSem; -+ session->syncCloseSem = nullptr; -+ close(session->pipes.at(0)); -+ delete session->sessionMutex; -+ session->sessionMutex = nullptr; -+ delete session; -+ }).detach(); -+ } -+ ``` -+ -+### Websocket Stream -+#### Flow chart -+Websocket_Stream_Flow_Chart -+ -+#### Detail -+- ConsumeRequest -+ ```c -+ // Consume the token (remove it from the cache) and return the cached request, if found. -+ ::google::protobuf::Message *RequestCache::ConsumeRequest(const std::string &token) -+ { -+ std::lock_guard lock(m_mutex); -+ -+ if (m_tokens.count(token) == 0) { -+ ERROR("Invalid token"); -+ return nullptr; -+ } -+ -+ CacheEntry ele = m_tokens[token]; -+ for (auto it = m_ll.begin(); it != m_ll.end(); it++) { -+ if (it->token == token) { -+ m_ll.erase(it); -+ break; -+ } -+ } -+ m_tokens.erase(token); -+ if (std::chrono::system_clock::now() > ele.expireTime) { -+ return nullptr; -+ } -+ -+ return ele.req; -+ } -+ ``` -+- AttachServe::SetContainerStreamRequest -+set stdin, stdout and stderr -+ ```c -+ m_request->attach_stdin = grequest->stdin() -+ m_request->attach_stdout = grequest->stdout() -+ m_request->attach_stderr = grequest->stderr() -+ ``` -+- AttachServe::ExecuteStreamCommand -+ ```c -+ // StringWriter -+ // write stdout to client if attach stdout is true -+ stdoutContext.attachWriter = m_request->attach_stdout ? WsWriteStdoutToClient : WsDoNotWriteStdoutToClient -+ // sem_post attachSem -+ stdoutstringWriter.close_func = AttachConnectClosed -+ // write stderr to client if attach stderr is true -+ stderrContext.attachWriter = m_request->attach_stderr ? WsWriteStderrToClient : WsDoNotWriteStderrToClient -+ stderrstringWriter.close_func = nullptr -+ -+ // call attach -+ // nonblocking -+ int ret = cb->container.attach(m_request, &m_response, m_request->attach_stdin ? lwsCtx->pipes.at(0) : -1, &stdoutstringWriter, &stderrstringWriter) -+ -+ // handle attach return value -+ if (ret != 0) { -+ // join io copy thread in attach callback -+ ERROR("Failed to attach container: %s", m_request->container_id); -+ -+ std::string message; -+ if (m_response != nullptr && m_response->errmsg != nullptr) { -+ message = m_response->errmsg; -+ } else { -+ message = "Failed to call attach container callback. "; -+ } -+ WsWriteStdoutToClient(lwsCtx, message.c_str(), message.length()); -+ } else { -+ // wait io copy thread complete -+ (void)sem_wait(&attachSem); -+ } -+ ``` -+- ExecServe::SetContainerStreamRequest -+set tty, stdin, stdout, stderr, container_id, args and suffix -+ ```c -+ m_request->tty = grequest->tty() -+ m_request->attach_stdin = grequest->stdin() -+ m_request->attach_stdout = grequest->stdout() -+ m_request->attach_stderr = grequest->stderr() -+ m_request->container_id = util_strdup_s(grequest->container_id().c_str()) -+ m_request->suffix = util_strdup_s(suffix.c_str()) -+ -+ // args -+ if (grequest->cmd_size() > 0) { -+ m_request->argv = (char **)util_smart_calloc_s(sizeof(char *), grequest->cmd_size()); -+ for (int i = 0; i < grequest->cmd_size(); i++) { -+ m_request->argv[i] = util_strdup_s(grequest->cmd(i).c_str()); -+ } -+ m_request->argv_len = static_cast(grequest->cmd_size()); -+ } -+ ``` -+- ExecServe::ExecuteStreamCommand -+ ```c -+ // StringWriter -+ StdoutstringWriter.write_func = WsWriteStdoutToClient -+ StdoutstringWriter.close_func = nullptr -+ StderrstringWriter.write_func = WsWriteStderrToClient -+ StderrstringWriter.close_func = nullptr -+ -+ // call exec -+ int ret = cb->container.exec(m_request, &m_response, m_request->attach_stdin ? lwsCtx->pipes.at(0) : -1, m_request->attach_stdout ? &StdoutstringWriter : nullptr, m_request->attach_stderr ? &StderrstringWriter : nullptr) -+ -+ // handle exec return value -+ if (ret != 0) { -+ std::string message; -+ if (m_response != nullptr && m_response->errmsg != nullptr) { -+ message = m_response->errmsg; -+ } else { -+ message = "Failed to call exec container callback. "; -+ } -+ WsWriteStdoutToClient(lwsCtx, message.c_str(), message.length()); -+ } -+ if (m_response != nullptr && m_response->exit_code != 0) { -+ std::string exit_info = "Exit code :" + std::to_string((int)m_response->exit_code) + "\n"; -+ WsWriteStdoutToClient(lwsCtx, exit_info.c_str(), exit_info.length()); -+ } -+ ``` -diff --git a/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg b/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..eaf33c61 ---- /dev/null -+++ b/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ AttachGetRealContainerOrSandboxIDlabel: cri.isulad.type=containergenerate urlreturn urlget container ID ?Ygenerate urlGarbageCollectionUniqueTokenlock(m_mutex)saveURLws://localhost:10350/cri/attach/<token>CacheEntry list overflow(1000) ?Nunlock(m_mutex)bugcheck container running?ValidateAttachRequestsuccess -\ No newline at end of file -diff --git a/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg b/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..c8daa2de ---- /dev/null -+++ b/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ ContainerStatusGetRealContainerOrSandboxIDlabel: cri.isulad.type=containerContainerStatusToGRPCreturnget container ID ?Yget inspect data?YInspectContainer -\ No newline at end of file -diff --git a/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..1fc622e9 ---- /dev/null -+++ b/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ CreateContainerGetRealContainerOrSandboxIDGetContainerOrSandboxRuntimelabel: cri.isulad.type=podsandboxGenerateCreateContainerRequestm_cb->container.createreturn response idget pod ID ?Yreturn error if not get runtimeORuse new variable to ignore error -\ No newline at end of file -diff --git a/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg b/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..b3462c59 ---- /dev/null -+++ b/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ ExecSyncGetRealContainerOrSandboxIDExecSyncFromGRPClabel: cri.isulad.type=containerm_cb->container.execset exit codeget container ID ?YStdoutstringWriterStderrstringWriter -\ No newline at end of file -diff --git a/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg b/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..463a9cce ---- /dev/null -+++ b/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ ExecGetRealContainerOrSandboxIDlabel: cri.isulad.type=containerget container ID ?Yreturn urlgenerate urlgenerate urlGarbageCollectionUniqueTokenlock(m_mutex)saveURLws://localhost:10350/cri/exec/<token>CacheEntry list overflow(1000) ?Nunlock(m_mutex)get container state ?YInspectContainerStateValidateExecRequest ?success -\ No newline at end of file -diff --git a/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg b/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..9955aefa ---- /dev/null -+++ b/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ ListContainersListContainersFromGRPCListContainersToGRPCm_cb->container.list -\ No newline at end of file -diff --git a/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..0ade7967 ---- /dev/null -+++ b/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ RemoveContainerGetRealContainerOrSandboxIDRemoveContainerLogSymlinklabel: cri.isulad.type=containercb->container.removeget container ID ?Yforce remove -\ No newline at end of file -diff --git a/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..e84bd261 ---- /dev/null -+++ b/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ StartContainerGetRealContainerOrSandboxIDm_cb->container.startlabel: cri.isulad.type=containerCreateContainerLogSymlinkget container ID ?Y -\ No newline at end of file -diff --git a/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..ab619ecc ---- /dev/null -+++ b/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ StopContainerGetRealContainerOrSandboxIDset id and timeoutlabel: cri.isulad.type=containerm_cb->container.stopget container ID ?Y -\ No newline at end of file -diff --git a/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg b/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..78ca3360 ---- /dev/null -+++ b/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ UpdateContainerResourcesGetRealContainerOrSandboxIDgenerate hostconfiglabel: cri.isulad.type=containerm_cb->container.updateget container ID ?Y -\ No newline at end of file -diff --git a/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..8e3e589a ---- /dev/null -+++ b/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ WebsocketServer::Callback(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len)switch (reason)read lock m_mutexIs Last StdinComplete ?write((char *)in)LWS_CALLBACK_RECEIVESetStdinCompleteswitch *(char *)in ?write((char *)in + 1)WebsocketChannel::STDINCHANNELParseTerminalSize((char *)in + 1)WebsocketChannel::RESIZECHANNELcb->container.resizereturn 0unlock m_mutexwrite lock m_mutexLWS_CALLBACK_CLOSEDdelete session in mapreturn 0unlock m_mutexcreate std::threadCloseSessionEraseAllMessageclose pipe[1]sem_wait(session->syncCloseSem)close pipe[0]deleteget SessionData by socket_fd ?(disable an http request)return -1get token URLLWS_CALLBACK_HTTPLWS_CALLBACK_FILTER_PROTOCOL_CONNECTIONwrite lock m_mutexvalidate URL ?too many connection sessions ?GenerateSessionDatasave sessionstd::thread StreamTasklws_callback_on_writableLWS_CALLBACK_ESTABLISHEDDebug messagereturn 0unlock m_mutexreturn 0read lock m_mutexbuffer.empty() ?message=FrontMessage()lws_write(message)PopMessageLWS_CALLBACK_SERVER_WRITEABLEget SessionData by socket_fd ?sessionClosed=IsClosed()sessionClosed ?lws_callback_on_writableunlock m_mutexreturn 0socketID already exist ?YNNNYYreturn nonzero from the user callback to close the connection and callback with the reason of LWS_CALLBACK_CLOSEDYNNYYNothersDebugInvalid dataYNNYget SessionData by socket_fd ?WSSessionGCunlock m_mutexreturn 0return 0othersYNthreadWebsocket Stream -\ No newline at end of file -diff --git a/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..ac784cc6 ---- /dev/null -+++ b/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ WebsocketServer::Startconf_get_websocket_server_listening_portWebsocketServer::CreateContextWebsocketServerlws_servicem_forceExit ?start isuladcreate threadreturnWebsocketServer::Waitwrite lock(m_mutex)unlock(m_mutex)join threadfor each SessionDatain m_wsislws_context_destroymain threaddelete SessionDataWebsocketServer::Shutdownset m_forceExitlws_cancel_servicereceive signal SIGINT/SIGTERMmainloop callback -\ No newline at end of file -diff --git a/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg -new file mode 100644 -index 00000000..9a1c60b7 ---- /dev/null -+++ b/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ sem_post(lwsCtx->syncCloseSem)bugcloseWsConnect?AttachServeget request by token ?earseAttachServe::SetContainerStreamRequeststdoutstringWriterstderrstringWritercb->container.attachattach return 0 ?write error messagesem_wait(&attachSem)CloseSessionsem_post(lwsCtx->syncCloseSem)YNexpired request ?YConsumeRequestNYAttachServe::ExecuteStreamCommanderrorExecServeget request by token ?earseExecServe::SetContainerStreamRequeststdoutstringWriterstderrstringWritercb->container.execexec return 0 ?write error messagewrite exit codeCloseSessionsem_post(lwsCtx->syncCloseSem)YNexpired request ?ConsumeRequestNYExecServe::ExecuteStreamCommandY -\ No newline at end of file -diff --git a/docs/design/CRI/Websocket_UML.excalidraw.svg b/docs/design/CRI/Websocket_UML.excalidraw.svg -new file mode 100644 -index 00000000..d1248c67 ---- /dev/null -+++ b/docs/design/CRI/Websocket_UML.excalidraw.svg -@@ -0,0 +1,5 @@ -+ -+ -+ -+ -+ WebsocketServer+ Start+ Wait+ Shutdown+ ...StreamingServeInterfaceinterface+ Execute+ ...AttachServe+ SetContainerStreamRequest+ ExecuteStreamCommand+ CloseConnect+ ...ExecServe+ SetContainerStreamRequest+ ExecuteStreamCommand+ CloseConnect+ ...RouteCallbackRegister+ RegisterCallback+ HandleCallback+ ...useuseWebsocket ImplStreamTask+ Run+ ...useuseRuntimeManagerService+ UpdateRuntimeConfig+ Status+ ...CRI IMPLuse -\ No newline at end of file --- -2.25.1 - diff --git a/0033-fix-util_getgrent_r-overflow.patch b/0033-fix-util_getgrent_r-overflow.patch deleted file mode 100644 index 619f367..0000000 --- a/0033-fix-util_getgrent_r-overflow.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 2890d9c08a60e45fa76ae0a8e72e0a37deac6a89 Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Tue, 21 Mar 2023 18:52:43 +0800 -Subject: [PATCH 33/46] fix util_getgrent_r overflow - -Signed-off-by: zhangxiaoyu ---- - src/utils/cutils/utils_pwgr.c | 3 +++ - test/cutils/utils_pwgr/long_sample | 1 + - test/cutils/utils_pwgr/utils_pwgr_ut.cc | 28 +++++++++++++++++++++++++ - 3 files changed, 32 insertions(+) - create mode 100644 test/cutils/utils_pwgr/long_sample - -diff --git a/src/utils/cutils/utils_pwgr.c b/src/utils/cutils/utils_pwgr.c -index 75efadb2..064bae1f 100644 ---- a/src/utils/cutils/utils_pwgr.c -+++ b/src/utils/cutils/utils_pwgr.c -@@ -157,6 +157,9 @@ static char **hold_string_list(char **line, char *buf_start, char *buf_end, cons - } - - out: -+ if ((char *)(walker + 2) > buf_end) { -+ return NULL; -+ } - *walker = NULL; - return result; - } -diff --git a/test/cutils/utils_pwgr/long_sample b/test/cutils/utils_pwgr/long_sample -new file mode 100644 -index 00000000..7e991ffd ---- /dev/null -+++ b/test/cutils/utils_pwgr/long_sample -@@ -0,0 +1 @@ -+adm:x:4:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -\ No newline at end of file -diff --git a/test/cutils/utils_pwgr/utils_pwgr_ut.cc b/test/cutils/utils_pwgr/utils_pwgr_ut.cc -index 431372d3..20de8a89 100644 ---- a/test/cutils/utils_pwgr/utils_pwgr_ut.cc -+++ b/test/cutils/utils_pwgr/utils_pwgr_ut.cc -@@ -140,3 +140,31 @@ TEST(utils_pwgr, test_getgrent_r) - - fclose(f_gr); - } -+ -+TEST(utils_pwgr, test_long_getgrent_r) -+{ -+ std::string path = "../../../../test/cutils/utils_pwgr/long_sample"; -+ FILE *f_gr = fopen(path.c_str(), "r"); -+ ASSERT_NE(f_gr, nullptr); -+ -+ struct group gr { -+ 0 -+ }; -+ struct group *pgr = nullptr; -+ char mark_before[BUFSIZ] = { 0 }; -+ char buf[BUFSIZ] = { 0 }; -+ char mark_after[BUFSIZ] = { 0 }; -+ -+ for (int num = 0; num < BUFSIZ; num++) { -+ mark_before[num] = 0xEE; -+ mark_after[num] = 0xDD; -+ } -+ -+ (void)util_getgrent_r(f_gr, &gr, buf, sizeof(buf), &pgr); -+ for (int num = 0; num < BUFSIZ; num++) { -+ ASSERT_EQ(mark_before[num], (char)0xEE); -+ ASSERT_EQ(mark_after[num], (char)0xDD); -+ } -+ -+ fclose(f_gr); -+} --- -2.25.1 - diff --git a/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch b/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch deleted file mode 100644 index c58e315..0000000 --- a/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch +++ /dev/null @@ -1,44 +0,0 @@ -From be989b6beff994f1b06e885ef766f01a6a3c0969 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Mon, 20 Mar 2023 14:17:00 +0800 -Subject: [PATCH 34/46] modify the return value of the - util_waitpid_with_timeout to status - -Signed-off-by: zhongtao ---- - src/utils/cutils/utils.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c -index 7f36d019..3cede76a 100644 ---- a/src/utils/cutils/utils.c -+++ b/src/utils/cutils/utils.c -@@ -321,6 +321,7 @@ rep: - int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) - { - int nret = 0; -+ int st; - time_t start_time = time(NULL); - time_t end_time; - double interval; -@@ -330,7 +331,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c - } - - for (;;) { -- nret = waitpid(pid, NULL, WNOHANG); -+ nret = waitpid(pid, &st, WNOHANG); - if (nret == pid) { - break; - } -@@ -349,7 +350,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c - // sleep some time instead to avoid cpu full running and then retry. - usleep(100); - } -- return 0; -+ return st; - } - - int util_wait_for_pid_status(pid_t pid) --- -2.25.1 - diff --git a/0035-fix-inspect-data-memleak.patch b/0035-fix-inspect-data-memleak.patch deleted file mode 100644 index 587a9a3..0000000 --- a/0035-fix-inspect-data-memleak.patch +++ /dev/null @@ -1,39 +0,0 @@ -From d4f524ef82ffdf2aed4847fa25795894a7bb7eda Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Tue, 28 Mar 2023 10:14:22 +0800 -Subject: [PATCH 35/46] fix inspect data memleak - -Signed-off-by: zhangxiaoyu ---- - src/daemon/entry/cri/cni_network_plugin.cc | 2 +- - src/daemon/entry/cri/cri_container_manager_service_impl.cc | 1 + - 2 files changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc -index 02e75ffe..8cad0126 100644 ---- a/src/daemon/entry/cri/cni_network_plugin.cc -+++ b/src/daemon/entry/cri/cni_network_plugin.cc -@@ -478,7 +478,7 @@ auto CniNetworkPlugin::GetNetNS(const std::string &podSandboxID, Errors &err) -> - - container_inspect *inspect_data = CRIHelpers::InspectContainer(podSandboxID, err, false); - if (inspect_data == nullptr) { -- goto cleanup; -+ return result; - } - if (inspect_data->state->pid == 0) { - err.Errorf("cannot find network namespace for the terminated container %s", podSandboxID.c_str()); -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index 0fc1884f..d406496e 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -1100,6 +1100,7 @@ ContainerManagerServiceImpl::ContainerStatus(const std::string &containerID, Err - ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1alpha2::ContainerStatus); - if (contStatus == nullptr) { - error.SetError("Out of memory"); -+ free_container_inspect(inspect); - return nullptr; - } - --- -2.25.1 - diff --git a/0036-containers-in-same-sandbox-should-have-same-process-.patch b/0036-containers-in-same-sandbox-should-have-same-process-.patch deleted file mode 100644 index 8b24921..0000000 --- a/0036-containers-in-same-sandbox-should-have-same-process-.patch +++ /dev/null @@ -1,193 +0,0 @@ -From 12b64931a90594e837e1f3dc9a0dde5292bb73b5 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sat, 8 Apr 2023 15:42:30 +0800 -Subject: [PATCH 36/46] containers in same sandbox should have same process - labels - -Signed-off-by: zhongtao ---- - .../cri/cri_container_manager_service_impl.cc | 71 ++++++++++++++++++- - .../cri/cri_container_manager_service_impl.h | 5 +- - src/daemon/entry/cri/cri_helpers.cc | 31 ++++++++ - src/daemon/entry/cri/cri_helpers.h | 3 + - 4 files changed, 107 insertions(+), 3 deletions(-) - -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index d406496e..a64d222c 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -126,8 +126,66 @@ auto ContainerManagerServiceImpl::PackCreateContainerHostConfigSecurityContext( - return 0; - } - -+auto ContainerManagerServiceImpl::DoUsePodLevelSELinuxConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, -+ host_config *hostconfig, const std::string &realPodSandboxID, Errors &error) -> int -+{ -+ int ret = -1; -+ size_t newSize = 0; -+ size_t oldSize = 0; -+ container_inspect *inspect = nullptr; -+ std::vector selinuxLabelOpts; -+ char **tmp_security_opt = nullptr; -+ std::string tmp_str; -+ -+ inspect = CRIHelpers::InspectContainer(realPodSandboxID, error, true); -+ if (error.NotEmpty()) { -+ return -1; -+ } -+ -+ if (inspect->process_label == nullptr) { -+ ret = 0; -+ goto cleanup; -+ } -+ -+ tmp_str = std::string(inspect->process_label); -+ selinuxLabelOpts = CRIHelpers::GetSELinuxLabelOpts(tmp_str, error); -+ if (error.NotEmpty()) { -+ ERROR("Failed to get SELinuxLabelOpts for container %s", containerConfig.metadata().name().c_str()); -+ goto cleanup; -+ } -+ if (selinuxLabelOpts.empty()) { -+ error.Errorf("SElinuxLabelOpts for container %s is empty", containerConfig.metadata().name().c_str()); -+ goto cleanup; -+ } -+ if (selinuxLabelOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { -+ ERROR("Out of memory"); -+ error.Errorf("Out of memory"); -+ goto cleanup; -+ } -+ newSize = (hostconfig->security_opt_len + selinuxLabelOpts.size()) * sizeof(char *); -+ oldSize = hostconfig->security_opt_len * sizeof(char *); -+ ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); -+ if (ret != 0) { -+ ERROR("Out of memory"); -+ error.Errorf("Out of memory"); -+ goto cleanup; -+ } -+ hostconfig->security_opt = tmp_security_opt; -+ for (const auto &securityOpt : selinuxLabelOpts) { -+ hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); -+ hostconfig->security_opt_len++; -+ } -+ -+cleanup: -+ free_container_inspect(inspect); -+ return ret; -+} -+ -+ -+ - auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( -- const runtime::v1alpha2::ContainerConfig &containerConfig, Errors &error) -> host_config * -+ const runtime::v1alpha2::ContainerConfig &containerConfig, -+ const std::string &realPodSandboxID, Errors &error) -> host_config * - { - host_config *hostconfig = (host_config *)util_common_calloc_s(sizeof(host_config)); - if (hostconfig == nullptr) { -@@ -159,6 +217,15 @@ auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( - goto cleanup; - } - -+ // If selinux label is not specified in container config, use pod level SELinux config -+ if (!containerConfig.linux().has_security_context() || -+ !containerConfig.linux().security_context().has_selinux_options()) { -+ if (DoUsePodLevelSELinuxConfig(containerConfig, hostconfig, realPodSandboxID, error) != 0) { -+ error.SetError("Failed to security context to host config"); -+ goto cleanup; -+ } -+ } -+ - return hostconfig; - - cleanup: -@@ -367,7 +434,7 @@ ContainerManagerServiceImpl::GenerateCreateContainerRequest(const std::string &r - - container_config *custom_config { nullptr }; - -- host_config *hostconfig = GenerateCreateContainerHostConfig(containerConfig, error); -+ host_config *hostconfig = GenerateCreateContainerHostConfig(containerConfig, realPodSandboxID, error); - if (error.NotEmpty()) { - goto cleanup; - } -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.h b/src/daemon/entry/cri/cri_container_manager_service_impl.h -index 6ac1cbc4..dcb2cd44 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.h -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.h -@@ -83,7 +83,7 @@ private: - const std::string &podSandboxRuntime, - Errors &error) -> container_create_request *; - auto GenerateCreateContainerHostConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, -- Errors &error) -> host_config *; -+ const std::string &realPodSandboxID, Errors &error) -> host_config *; - auto GenerateCreateContainerCustomConfig(const std::string &containerName, const std::string &realPodSandboxID, - const runtime::v1alpha2::ContainerConfig &containerConfig, - const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, -@@ -92,6 +92,9 @@ private: - host_config *hostconfig, Errors &error) -> int; - auto PackCreateContainerHostConfigSecurityContext(const runtime::v1alpha2::ContainerConfig &containerConfig, - host_config *hostconfig, Errors &error) -> int; -+ auto DoUsePodLevelSELinuxConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, -+ host_config *hostconfig, -+ const std::string &realPodSandboxID, Errors &error) -> int; - void MakeContainerConfig(const runtime::v1alpha2::ContainerConfig &config, container_config *cConfig, - Errors &error); - void CreateContainerLogSymlink(const std::string &containerID, Errors &error); -diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc -index 711196ba..2bc6bed7 100644 ---- a/src/daemon/entry/cri/cri_helpers.cc -+++ b/src/daemon/entry/cri/cri_helpers.cc -@@ -619,6 +619,37 @@ auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, E - return seccompSecurityOpts; - } - -+auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -+-> std::vector -+{ -+ // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) -+ // New version '=' , old version ':', iSulad cri is based on v18.09, so iSulad cri use new version separator -+ const char securityOptSep { '=' }; -+ // LabeSep is consistent with the separator used when parsing labels -+ const char labeSep { ':' }; -+ std::vector selinuxOpts { }; -+ char **labelArr = nullptr; -+ size_t labelArrLen = 0; -+ std::vector opts = {"user", "role", "type", "level"}; -+ std::vector vect; -+ -+ labelArr = util_string_split_n(selinuxLabel.c_str(), labeSep, 4); -+ if (labelArr == nullptr) { -+ error.Errorf("Invalid selinux label: %s", selinuxLabel.c_str()); -+ return vect; -+ } -+ -+ labelArrLen = util_array_len((const char **)labelArr); -+ for (size_t i {}; i < labelArrLen; i++) { -+ iSuladOpt tmp = { "label", opts[i] + std::string(1, labeSep) + std::string(labelArr[i]), "" }; -+ selinuxOpts.push_back(tmp); -+ } -+ -+ util_free_array(labelArr); -+ -+ return fmtiSuladOpts(selinuxOpts, securityOptSep); -+} -+ - auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string - { - cri_checkpoint *criCheckpoint { nullptr }; -diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h -index d50759ad..4ef227d7 100644 ---- a/src/daemon/entry/cri/cri_helpers.h -+++ b/src/daemon/entry/cri/cri_helpers.h -@@ -127,6 +127,9 @@ auto ToIsuladContainerStatus(const runtime::v1alpha2::ContainerStateValue &state - auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) - -> std::vector; - -+auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -+-> std::vector; -+ - auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string; - - void GetCheckpoint(const std::string &jsonCheckPoint, CRI::PodSandboxCheckpoint &checkpoint, Errors &error); --- -2.25.1 - diff --git a/0037-clean-container-process-after-execSync-timeout-exit.patch b/0037-clean-container-process-after-execSync-timeout-exit.patch deleted file mode 100644 index 405581f..0000000 --- a/0037-clean-container-process-after-execSync-timeout-exit.patch +++ /dev/null @@ -1,434 +0,0 @@ -From 53935dc36121b10975e047438ae8135ffa702920 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Tue, 4 Apr 2023 16:30:54 +0800 -Subject: [PATCH 37/46] clean container process after execSync timeout exit - -Signed-off-by: zhongtao ---- - src/cmd/isulad-shim/common.c | 30 ++++ - src/cmd/isulad-shim/common.h | 11 ++ - src/cmd/isulad-shim/main.c | 15 +- - src/cmd/isulad-shim/process.c | 158 +++++++++++++----- - src/cmd/isulad-shim/process.h | 2 +- - .../modules/runtime/isula/isula_rt_ops.c | 40 ++--- - 6 files changed, 188 insertions(+), 68 deletions(-) - -diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c -index 0c345187..f188da1e 100644 ---- a/src/cmd/isulad-shim/common.c -+++ b/src/cmd/isulad-shim/common.c -@@ -26,6 +26,7 @@ - #include - #include - #include -+#include - - int set_fd_no_inherited(int fd) - { -@@ -316,3 +317,32 @@ int open_no_inherit(const char *path, int flag, mode_t mode) - - return fd; - } -+ -+static bool is_invalid_error_str(const char *err_str, const char *numstr) -+{ -+ return err_str == NULL || err_str == numstr || *err_str != '\0'; -+} -+ -+int shim_util_safe_uint64(const char *numstr, uint64_t *converted) -+{ -+ char *err_str = NULL; -+ uint64_t ull; -+ -+ if (numstr == NULL || converted == NULL) { -+ return -EINVAL; -+ } -+ -+ errno = 0; -+ ull = strtoull(numstr, &err_str, 0); -+ if (errno > 0) { -+ return -errno; -+ } -+ -+ if (is_invalid_error_str(err_str, numstr)) { -+ return -EINVAL; -+ } -+ -+ *converted = (uint64_t)ull; -+ return 0; -+} -+ -diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h -index d06c5256..91808295 100644 ---- a/src/cmd/isulad-shim/common.h -+++ b/src/cmd/isulad-shim/common.h -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - - #ifdef __cplusplus - extern "C" { -@@ -31,6 +32,14 @@ extern "C" { - #define SHIM_ERR (-1) - #define SHIM_ERR_WAIT (-2) - #define SHIM_ERR_NOT_REQUIRED (-3) -+#define SHIM_ERR_TIMEOUT (-4) -+ -+// common exit code is defined in stdlib.h -+// EXIT_FAILURE 1 : Failing exit status. -+// EXIT_SUCCESS 0 : Successful exit status. -+// custom shim exit code -+// SHIM_EXIT_TIMEOUT 2: Container process timeout exit code -+#define SHIM_EXIT_TIMEOUT 2 - - #define INFO_MSG "info" - #define WARN_MSG "warn" -@@ -68,6 +77,8 @@ void close_fd(int *pfd); - - int open_no_inherit(const char *path, int flag, mode_t mode); - -+int shim_util_safe_uint64(const char *numstr, uint64_t *converted); -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c -index eedd8fda..68e99e53 100644 ---- a/src/cmd/isulad-shim/main.c -+++ b/src/cmd/isulad-shim/main.c -@@ -62,7 +62,8 @@ static int set_subreaper() - return SHIM_OK; - } - --static int parse_args(int argc, char **argv, char **cid, char **bundle, char **rt_name, char **log_level) -+static int parse_args(int argc, char **argv, char **cid, char **bundle, char **rt_name, char **log_level, -+ uint64_t *timeout) - { - if (argc < 4) { - return SHIM_ERR; -@@ -82,6 +83,12 @@ static int parse_args(int argc, char **argv, char **cid, char **bundle, char **r - } - } - -+ if (argc > 5) { -+ if (shim_util_safe_uint64(strdup(argv[5]), timeout) != 0) { -+ return SHIM_ERR; -+ } -+ } -+ - return SHIM_OK; - } - -@@ -99,6 +106,8 @@ int main(int argc, char **argv) - int efd = -1; - process_t *p = NULL; - pthread_t tid_accept; -+ // execSync timeout -+ uint64_t timeout = 0; - - g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); - if (g_log_fd < 0) { -@@ -117,7 +126,7 @@ int main(int argc, char **argv) - exit(EXIT_FAILURE); - } - -- ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level); -+ ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level, &timeout); - if (ret != SHIM_OK) { - write_message(g_log_fd, ERR_MSG, "parse args failed:%d", ret); - exit(EXIT_FAILURE); -@@ -167,5 +176,5 @@ int main(int argc, char **argv) - - released_timeout_exit(); - -- return process_signal_handle_routine(p, tid_accept); -+ return process_signal_handle_routine(p, tid_accept, timeout); - } -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index 5222629c..02609911 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -1213,69 +1213,145 @@ static int try_wait_all_child(void) - return 1; - } - --int process_signal_handle_routine(process_t *p, const pthread_t tid_accept) -+static int waitpid_with_timeout(int ctr_pid, int *status, const int64_t timeout) - { -- int ret = SHIM_ERR; -- bool exit_shim = false; - int nret = 0; -- int i; -- struct timespec ts; -+ time_t start_time = time(NULL); -+ time_t end_time; -+ double interval; -+ int st; - - for (;;) { -- int status; -- ret = reap_container(p->ctr_pid, &status); -+ nret = waitpid(-1, &st, WNOHANG); -+ if (nret == ctr_pid) { -+ break; -+ } -+ end_time = time(NULL); -+ interval = difftime(end_time, start_time); -+ if (nret == 0 && interval >= timeout) { -+ return SHIM_ERR_TIMEOUT; -+ } -+ // sleep some time instead to avoid cpu full running and then retry. -+ usleep(1000); -+ } -+ -+ if (WIFSIGNALED(st)) { -+ *status = EXIT_SIGNAL_OFFSET + WTERMSIG(st); -+ } else { -+ *status = WEXITSTATUS(st); -+ } -+ -+ if (*status == CONTAINER_ACTION_REBOOT) { -+ nret = setenv("CONTAINER_ACTION", "reboot", 1); -+ if (nret != SHIM_OK) { -+ write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); -+ } -+ } else if (*status == CONTAINER_ACTION_SHUTDOWN) { -+ nret = setenv("CONTAINER_ACTION", "shutdown", 1); -+ if (nret != SHIM_OK) { -+ write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); -+ } -+ } -+ return SHIM_OK; -+} -+ -+/* -+ * If timeout <= 0, blocking wait in reap_container. -+ * If timeout > 0, non-blocking wait pid with timeout. -+ */ -+static int wait_container_process_with_timeout(process_t *p, const unsigned int timeout, int *status) -+{ -+ int ret = SHIM_ERR; -+ -+ if (timeout > 0) { -+ return waitpid_with_timeout(p->ctr_pid, status, timeout); -+ } -+ -+ for (;;) { -+ ret = reap_container(p->ctr_pid, status); - if (ret == SHIM_OK) { -- exit_shim = true; -- if (status == CONTAINER_ACTION_REBOOT) { -+ if (*status == CONTAINER_ACTION_REBOOT) { - ret = setenv("CONTAINER_ACTION", "reboot", 1); - if (ret != SHIM_OK) { - write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); - } -- } else if (status == CONTAINER_ACTION_SHUTDOWN) { -+ } else if (*status == CONTAINER_ACTION_SHUTDOWN) { - ret = setenv("CONTAINER_ACTION", "shutdown", 1); - if (ret != SHIM_OK) { - write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); - } - } -- } else if (ret == SHIM_ERR_WAIT) { -+ return SHIM_OK; -+ } -+ -+ if (ret == SHIM_ERR_WAIT) { - /* avoid thread entering the infinite loop */ - usleep(1000); -+ } -+ -+ if (ret == SHIM_ERR) { -+ // if the child process is not expected, retry. - continue; - } -- if (exit_shim) { -- process_kill_all(p); -+ } - -- // wait atmost 120 seconds -- DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); -- if (nret != 0) { -- write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds"); -- } -+} - -- process_delete(p); -- if (p->exit_fd > 0) { -- (void)write_nointr(p->exit_fd, &status, sizeof(int)); -- } -- // wait for task_console_accept thread termination. In order to make sure that -- // the io_copy connection is established and io_thread is not used by multiple threads. -- if (p->state->terminal) { -- if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { -- write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); -- nret = pthread_join(tid_accept, NULL); -- } else { -- // Set the maximum waiting time to 60s to prevent stuck. -- ts.tv_sec += 60; -- nret = pthread_timedjoin_np(tid_accept, NULL, &ts); -- } -+int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout) -+{ -+ int i; -+ int nret = 0; -+ int ret = 0; -+ int status = 0; -+ struct timespec ts; - -- if (nret != 0) { -- write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); -- } -- } -+ ret = wait_container_process_with_timeout(p, timeout, &status); -+ if (ret == SHIM_ERR_TIMEOUT) { -+ // kill container process to ensure process_kill_all effective -+ nret = kill(p->ctr_pid, SIGKILL); -+ if (nret < 0 && errno != ESRCH) { -+ write_message(g_log_fd, ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid); -+ exit(EXIT_FAILURE); -+ } -+ } - -- for (i = 0; i < 3; i++) { -- destroy_io_thread(p, i); -- } -- return status; -+ process_kill_all(p); -+ -+ // wait atmost 120 seconds -+ DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); -+ if (nret != 0) { -+ write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds"); -+ } -+ -+ process_delete(p); -+ if (p->exit_fd > 0) { -+ (void)write_nointr(p->exit_fd, &status, sizeof(int)); -+ } -+ // wait for task_console_accept thread termination. In order to make sure that -+ // the io_copy connection is established and io_thread is not used by multiple threads. -+ if (p->state->terminal) { -+ if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { -+ write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); -+ nret = pthread_join(tid_accept, NULL); -+ } else { -+ // Set the maximum waiting time to 60s to prevent stuck. -+ ts.tv_sec += 60; -+ nret = pthread_timedjoin_np(tid_accept, NULL, &ts); - } -+ -+ if (nret != 0) { -+ write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); -+ } -+ } -+ -+ for (i = 0; i < 3; i++) { -+ destroy_io_thread(p, i); - } -+ -+ if (ret == SHIM_ERR_TIMEOUT) { -+ write_message(g_log_fd, INFO_MSG, "Wait %d timeout", p->ctr_pid); -+ exit(SHIM_EXIT_TIMEOUT); -+ } -+ return status; -+ - } -diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h -index 66820f68..7e3259e8 100644 ---- a/src/cmd/isulad-shim/process.h -+++ b/src/cmd/isulad-shim/process.h -@@ -97,7 +97,7 @@ process_t* new_process(char *id, char *bundle, char *runtime); - int open_io(process_t *p, pthread_t *tid_accept); - int process_io_init(process_t *p); - int create_process(process_t *p); --int process_signal_handle_routine(process_t *p, const pthread_t tid_accept); -+int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout); - - #ifdef __cplusplus - } -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index e974964a..5a01b8c6 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -54,6 +54,7 @@ - #define SHIM_LOG_SIZE ((BUFSIZ - 100) / 2) - #define RESIZE_DATA_SIZE 100 - #define PID_WAIT_TIME 120 -+#define SHIM_EXIT_TIMEOUT 2 - - // file name formats of cgroup resources json - #define RESOURCE_FNAME_FORMATS "%s/resources.json" -@@ -692,27 +693,6 @@ static int status_to_exit_code(int status) - return exit_code; - } - --static int try_wait_pid(pid_t pid) --{ -- if (waitpid(pid, NULL, WNOHANG) == pid) { -- return 0; -- } -- -- return 1; --} -- --static void kill_and_show_err(pid_t pid) --{ -- int nret = 0; -- kill(pid, SIGKILL); -- // wait atmost 0.5 seconds -- DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); -- if (nret != 0) { -- WARN("Fail to wait isulad-shim"); -- } -- isulad_set_error_message("Exec container error;exec timeout"); --} -- - static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, - int *exit_code, const int64_t timeout) - { -@@ -731,7 +711,14 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - params[i++] = bundle; - params[i++] = runtime_cmd; - params[i++] = "info"; -- params[i++] = "2m0s"; -+ // execSync timeout -+ if (timeout > 0) { -+ params[i] = util_int_to_string(timeout); -+ if (params[i] == NULL) { -+ ERROR("Failed to convert execSync timeout %ld to string", timeout); -+ return -1; -+ } -+ } - runtime_exec_param_dump(params); - - if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { -@@ -805,7 +792,7 @@ realexec: - goto out; - } - -- status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); -+ status = util_wait_for_pid_status(pid); - if (status < 0) { - ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); - ret = -1; -@@ -1204,6 +1191,13 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - goto errlog_out; - } - -+ if (*exit_code == SHIM_EXIT_TIMEOUT) { -+ ret = -1; -+ isulad_set_error_message("Exec container error;exec timeout"); -+ ERROR("isulad-shim %d exit for execing timeout", pid); -+ goto errlog_out; -+ } -+ - pid = get_container_process_pid(workdir); - if (pid < 0) { - ERROR("%s: failed get exec process id", workdir); --- -2.25.1 - diff --git a/0038-support-to-config-selinux-label-in-cri.patch b/0038-support-to-config-selinux-label-in-cri.patch deleted file mode 100644 index 5d6591e..0000000 --- a/0038-support-to-config-selinux-label-in-cri.patch +++ /dev/null @@ -1,346 +0,0 @@ -From d231a8e5f8ea1c8a5584fdc8baa579b18d1b6a19 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Wed, 29 Mar 2023 03:26:45 +0800 -Subject: [PATCH 38/46] support to config selinux label in cri - -Signed-off-by: zhongtao ---- - .../cri/cri_container_manager_service_impl.cc | 47 +++++---- - .../cri/cri_container_manager_service_impl.h | 1 + - src/daemon/entry/cri/cri_helpers.cc | 95 ++++++++++++++++++- - src/daemon/entry/cri/cri_helpers.h | 17 +++- - .../cri_pod_sandbox_manager_service_impl.cc | 47 ++++----- - 5 files changed, 149 insertions(+), 58 deletions(-) - -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index a64d222c..6278512f 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -98,30 +98,21 @@ auto ContainerManagerServiceImpl::PackCreateContainerHostConfigSecurityContext( - // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) - // New version '=' , old version ':', iSulad cri is based on v18.09, so iSulad cri use new version separator - const char securityOptSep { '=' }; -- std::vector securityOpts = CRIHelpers::GetSecurityOpts( -- containerConfig.linux().security_context().seccomp_profile_path(), securityOptSep, error); -+ const ::runtime::v1alpha2::LinuxContainerSecurityContext &context = containerConfig.linux().security_context(); -+ CRIHelpers::commonSecurityContext commonContext = { -+ .hasSELinuxOption = context.has_selinux_options(), -+ .selinuxOption = context.selinux_options(), -+ .seccompProfile = context.seccomp_profile_path(), -+ }; -+ std::vector securityOpts = CRIHelpers::GetSecurityOpts(commonContext, securityOptSep, error); - if (error.NotEmpty()) { -- error.Errorf("failed to generate security options for container %s", containerConfig.metadata().name().c_str()); -+ error.Errorf("Failed to generate security options for container %s", containerConfig.metadata().name().c_str()); - return -1; - } -- if (!securityOpts.empty()) { -- char **tmp_security_opt = nullptr; -- if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { -- error.Errorf("Out of memory"); -- return -1; -- } -- size_t newSize = (hostconfig->security_opt_len + securityOpts.size()) * sizeof(char *); -- size_t oldSize = hostconfig->security_opt_len * sizeof(char *); -- int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); -- if (ret != 0) { -- error.Errorf("Out of memory"); -- return -1; -- } -- hostconfig->security_opt = tmp_security_opt; -- for (const auto &securityOpt : securityOpts) { -- hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); -- hostconfig->security_opt_len++; -- } -+ CRIHelpers::AddSecurityOptsToHostConfig(securityOpts, hostconfig, error); -+ if (error.NotEmpty()) { -+ error.Errorf("Failed to add securityOpts to hostconfig for container %s", containerConfig.metadata().name().c_str()); -+ return -1; - } - return 0; - } -@@ -148,7 +139,7 @@ auto ContainerManagerServiceImpl::DoUsePodLevelSELinuxConfig(const runtime::v1al - } - - tmp_str = std::string(inspect->process_label); -- selinuxLabelOpts = CRIHelpers::GetSELinuxLabelOpts(tmp_str, error); -+ selinuxLabelOpts = CRIHelpers::GetPodSELinuxLabelOpts(tmp_str, error); - if (error.NotEmpty()) { - ERROR("Failed to get SELinuxLabelOpts for container %s", containerConfig.metadata().name().c_str()); - goto cleanup; -@@ -181,7 +172,14 @@ cleanup: - return ret; - } - -- -+auto ContainerManagerServiceImpl::IsSELinuxLabelEmpty(const ::runtime::v1alpha2::SELinuxOption &selinuxOption) -> bool -+{ -+ if (selinuxOption.user().length() == 0 && selinuxOption.role().length() == 0 && selinuxOption.type().length() == 0 && -+ selinuxOption.level().length() == 0) { -+ return true; -+ } -+ return false; -+} - - auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( - const runtime::v1alpha2::ContainerConfig &containerConfig, -@@ -219,7 +217,8 @@ auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( - - // If selinux label is not specified in container config, use pod level SELinux config - if (!containerConfig.linux().has_security_context() || -- !containerConfig.linux().security_context().has_selinux_options()) { -+ !containerConfig.linux().security_context().has_selinux_options() || -+ IsSELinuxLabelEmpty(containerConfig.linux().security_context().selinux_options())) { - if (DoUsePodLevelSELinuxConfig(containerConfig, hostconfig, realPodSandboxID, error) != 0) { - error.SetError("Failed to security context to host config"); - goto cleanup; -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.h b/src/daemon/entry/cri/cri_container_manager_service_impl.h -index dcb2cd44..ad53a65f 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.h -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.h -@@ -84,6 +84,7 @@ private: - Errors &error) -> container_create_request *; - auto GenerateCreateContainerHostConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, - const std::string &realPodSandboxID, Errors &error) -> host_config *; -+ auto IsSELinuxLabelEmpty(const ::runtime::v1alpha2::SELinuxOption &selinuxOption) -> bool; - auto GenerateCreateContainerCustomConfig(const std::string &containerName, const std::string &realPodSandboxID, - const runtime::v1alpha2::ContainerConfig &containerConfig, - const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, -diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc -index 2bc6bed7..ec14d0b2 100644 ---- a/src/daemon/entry/cri/cri_helpers.cc -+++ b/src/daemon/entry/cri/cri_helpers.cc -@@ -68,6 +68,8 @@ const std::string Constants::CNI_MUTL_NET_EXTENSION_KEY { "extension.network.kub - const std::string Constants::CNI_MUTL_NET_EXTENSION_ARGS_KEY { "CNI_MUTLINET_EXTENSION" }; - const std::string Constants::CNI_ARGS_EXTENSION_PREFIX_KEY { "extension.network.kubernetes.io/cniargs/" }; - const std::string Constants::IMAGE_NAME_ANNOTATION_KEY { "io.kubernetes.cri.image-name" }; -+// Usually, the format of level is "s0:c60,c525" or "s0-s0:c40.c23" -+const std::string Constants::SELINUX_LABEL_LEVEL_PATTERN { "^s[0-9](-s[0-9])?(:c[0-9]{1,4}(\\.c[0-9]{1,4})?(,c[0-9]{1,4}(\\.c[0-9]{1,4})?)*)?$" }; - - const char *InternalLabelKeys[] = { CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY.c_str(), - CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str(), -@@ -598,6 +600,34 @@ auto GetSeccompiSuladOpts(const std::string &seccompProfile, Errors &error) -> s - return ret; - } - -+ -+auto GetSelinuxiSuladOpts(const ::runtime::v1alpha2::SELinuxOption &selinux, Errors &error)-> std::vector -+{ -+ std::vector selinuxOpts { }; -+ // LabeSep is consistent with the separator used when parsing labels -+ const char labeSep { ':' }; -+ -+ if (selinux.level().length() != 0 && -+ util_reg_match(CRIHelpers::Constants::SELINUX_LABEL_LEVEL_PATTERN.c_str(), selinux.level().c_str()) != 0) { -+ error.Errorf("The format of 'level' %s is not correct", selinux.level().c_str()); -+ return selinuxOpts; -+ } -+ -+ if (selinux.user().length() > 0) { -+ selinuxOpts.push_back({ "label", std::string("user") + std::string(1, labeSep) + selinux.user(), "" }); -+ } -+ if (selinux.role().length() > 0) { -+ selinuxOpts.push_back({ "label", std::string("role") + std::string(1, labeSep) + selinux.role(), "" }); -+ } -+ if (selinux.type().length() > 0) { -+ selinuxOpts.push_back({ "label", std::string("type") + std::string(1, labeSep) + selinux.type(), "" }); -+ } -+ if (selinux.level().length() > 0) { -+ selinuxOpts.push_back({ "label", std::string("level") + std::string(1, labeSep) + selinux.level(), "" }); -+ } -+ return selinuxOpts; -+} -+ - auto GetSeccompSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) - -> std::vector - { -@@ -609,17 +639,44 @@ auto GetSeccompSecurityOpts(const std::string &seccompProfile, const char &separ - return fmtiSuladOpts(seccompOpts, separator); - } - --auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) -+auto GetSELinuxLabelOpts(const bool hasSELinuxOption, const ::runtime::v1alpha2::SELinuxOption &selinux, -+ const char &separator, Errors &error) - -> std::vector - { -- std::vector seccompSecurityOpts = GetSeccompSecurityOpts(seccompProfile, separator, error); -+ if (!hasSELinuxOption) { -+ return std::vector(); -+ } -+ -+ std::vector selinuxOpts = GetSelinuxiSuladOpts(selinux, error); - if (error.NotEmpty()) { -- error.Errorf("failed to generate seccomp security options for container: %s", error.GetMessage().c_str()); -+ return std::vector(); - } -- return seccompSecurityOpts; -+ -+ return fmtiSuladOpts(selinuxOpts, separator); - } - --auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -+auto GetSecurityOpts(const commonSecurityContext &context, const char &separator, Errors &error) -+-> std::vector -+{ -+ std::vector securityOpts; -+ std::vector seccompSecurityOpts = GetSeccompSecurityOpts(context.seccompProfile, separator, error); -+ if (error.NotEmpty()) { -+ error.Errorf("Failed to generate seccomp security options for container: %s", error.GetMessage().c_str()); -+ return securityOpts; -+ } -+ -+ std::vector selinuxOpts = CRIHelpers::GetSELinuxLabelOpts(context.hasSELinuxOption, -+ context.selinuxOption, separator, error); -+ if (error.NotEmpty()) { -+ error.Errorf("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); -+ return securityOpts; -+ } -+ securityOpts.insert(securityOpts.end(), seccompSecurityOpts.begin(), seccompSecurityOpts.end()); -+ securityOpts.insert(securityOpts.end(), selinuxOpts.begin(), selinuxOpts.end()); -+ return securityOpts; -+} -+ -+auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) - -> std::vector - { - // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) -@@ -650,6 +707,34 @@ auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) - return fmtiSuladOpts(selinuxOpts, securityOptSep); - } - -+void AddSecurityOptsToHostConfig(std::vector &securityOpts, host_config *hostconfig, Errors &error) -+{ -+ if (securityOpts.empty()) { -+ return; -+ } -+ -+ char **tmp_security_opt = nullptr; -+ if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { -+ error.Errorf("Too many securityOpts"); -+ ERROR("Too many securityOpts"); -+ return; -+ } -+ size_t newSize = (hostconfig->security_opt_len + securityOpts.size()) * sizeof(char *); -+ size_t oldSize = hostconfig->security_opt_len * sizeof(char *); -+ int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); -+ if (ret != 0) { -+ error.Errorf("Out of memory"); -+ ERROR("Out of memory"); -+ return; -+ } -+ hostconfig->security_opt = tmp_security_opt; -+ for (const auto &securityOpt : securityOpts) { -+ hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); -+ hostconfig->security_opt_len++; -+ } -+ -+} -+ - auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string - { - cri_checkpoint *criCheckpoint { nullptr }; -diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h -index 4ef227d7..a5d6cf61 100644 ---- a/src/daemon/entry/cri/cri_helpers.h -+++ b/src/daemon/entry/cri/cri_helpers.h -@@ -67,10 +67,17 @@ public: - static const std::string CNI_MUTL_NET_EXTENSION_KEY; - static const std::string CNI_MUTL_NET_EXTENSION_ARGS_KEY; - static const std::string CNI_ARGS_EXTENSION_PREFIX_KEY; -+ static const std::string SELINUX_LABEL_LEVEL_PATTERN; - - static const std::string IMAGE_NAME_ANNOTATION_KEY; - }; - -+struct commonSecurityContext { -+ const bool hasSELinuxOption; -+ const ::runtime::v1alpha2::SELinuxOption selinuxOption; -+ const std::string seccompProfile; -+}; -+ - auto GetDefaultSandboxImage(Errors &err) -> std::string; - - auto MakeLabels(const google::protobuf::Map &mapLabels, Errors &error) -@@ -124,12 +131,18 @@ auto ValidateCheckpointKey(const std::string &key, Errors &error) -> bool; - - auto ToIsuladContainerStatus(const runtime::v1alpha2::ContainerStateValue &state) -> std::string; - --auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) -+auto GetSELinuxLabelOpts(const bool hasSELinuxOption, const ::runtime::v1alpha2::SELinuxOption &selinux, -+ const char &separator, Errors &error) - -> std::vector; - --auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -+auto GetSecurityOpts(const commonSecurityContext &context, const char &separator, Errors &error) - -> std::vector; - -+auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -+-> std::vector; -+ -+void AddSecurityOptsToHostConfig(std::vector &securityOpts, host_config *hostconfig, Errors &error); -+ - auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string; - - void GetCheckpoint(const std::string &jsonCheckPoint, CRI::PodSandboxCheckpoint &checkpoint, Errors &error); -diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -index 7ff545db..1cb3254d 100644 ---- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc -@@ -197,34 +197,27 @@ void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alph - const char securityOptSep = '='; - - // Security Opts -- if (c.linux().has_security_context()) { -- std::vector securityOpts = -- CRIHelpers::GetSecurityOpts(c.linux().security_context().seccomp_profile_path(), securityOptSep, error); -- if (error.NotEmpty()) { -- error.Errorf("failed to generate security options for sandbox %s: %s", -- c.metadata().name().c_str(), error.GetMessage().c_str()); -- return; -- } -- if (!securityOpts.empty()) { -- char **tmp_security_opt = nullptr; -+ if (!c.linux().has_security_context()) { -+ return; -+ } - -- if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hc->security_opt_len) { -- error.Errorf("Out of memory"); -- return; -- } -- size_t newSize = (hc->security_opt_len + securityOpts.size()) * sizeof(char *); -- size_t oldSize = hc->security_opt_len * sizeof(char *); -- int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hc->security_opt, oldSize); -- if (ret != 0) { -- error.Errorf("Out of memory"); -- return; -- } -- hc->security_opt = tmp_security_opt; -- for (const auto &securityOpt : securityOpts) { -- hc->security_opt[hc->security_opt_len] = util_strdup_s(securityOpt.c_str()); -- hc->security_opt_len++; -- } -- } -+ CRIHelpers::commonSecurityContext commonContext = { -+ .hasSELinuxOption = c.linux().security_context().has_selinux_options(), -+ .selinuxOption = c.linux().security_context().selinux_options(), -+ .seccompProfile = c.linux().security_context().seccomp_profile_path(), -+ }; -+ -+ std::vector securityOpts = CRIHelpers::GetSecurityOpts(commonContext, securityOptSep, error); -+ if (error.NotEmpty()) { -+ error.Errorf("Failed to generate security options for sandbox %s: %s", -+ c.metadata().name().c_str(), error.GetMessage().c_str()); -+ return; -+ } -+ CRIHelpers::AddSecurityOptsToHostConfig(securityOpts, hc, error); -+ if (error.NotEmpty()) { -+ error.Errorf("Failed to add securityOpts to hostconfig for sandbox %s: %s", c.metadata().name().c_str(), -+ error.GetMessage().c_str()); -+ return; - } - } - --- -2.25.1 - diff --git a/0039-add-files_limit-to-oci-spec.patch b/0039-add-files_limit-to-oci-spec.patch deleted file mode 100644 index db23d49..0000000 --- a/0039-add-files_limit-to-oci-spec.patch +++ /dev/null @@ -1,94 +0,0 @@ -From 6b5a0a345ae767cc245d1008377b8489acf2c0c7 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sat, 8 Apr 2023 14:50:20 +0800 -Subject: [PATCH 39/46] add files_limit to oci spec - -Signed-off-by: zhongtao ---- - src/daemon/modules/spec/specs.c | 49 +++++++++++++++++++++++++++++++++ - 1 file changed, 49 insertions(+) - -diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c -index f32ff911..ad6d01d2 100644 ---- a/src/daemon/modules/spec/specs.c -+++ b/src/daemon/modules/spec/specs.c -@@ -890,6 +890,27 @@ static int make_sure_oci_spec_linux_resources_pids(oci_runtime_spec *oci_spec) - return 0; - } - -+static int make_sure_oci_spec_linux_resources_files(oci_runtime_spec *oci_spec) -+{ -+ int ret = 0; -+ -+ ret = make_sure_oci_spec_linux_resources(oci_spec); -+ if (ret < 0) { -+ return -1; -+ } -+ -+ if (oci_spec->linux->resources->files != NULL) { -+ return 0; -+ } -+ -+ oci_spec->linux->resources->files = util_common_calloc_s(sizeof(defs_resources_files)); -+ if (oci_spec->linux->resources->files == NULL) { -+ ERROR("Out of memory"); -+ return -1; -+ } -+ return 0; -+} -+ - static int merge_pids_limit(oci_runtime_spec *oci_spec, int64_t pids_limit) - { - int ret = 0; -@@ -905,6 +926,20 @@ out: - return ret; - } - -+static int merge_files_limit(oci_runtime_spec *oci_spec, int64_t files_limit) -+{ -+ int ret = 0; -+ -+ ret = make_sure_oci_spec_linux_resources_files(oci_spec); -+ if (ret < 0) { -+ ERROR("Failed to merge files limit"); -+ return ret; -+ } -+ -+ oci_spec->linux->resources->files->limit = files_limit; -+ return ret; -+} -+ - static int merge_hostname(oci_runtime_spec *oci_spec, const host_config *host_spec, container_config *container_spec) - { - free(oci_spec->hostname); -@@ -1251,6 +1286,15 @@ out: - return ret; - } - -+static int merge_conf_files_limit(oci_runtime_spec *oci_spec, const host_config *host_spec) -+{ -+ if (host_spec->files_limit == 0) { -+ return 0; -+ } -+ -+ return merge_files_limit(oci_spec, host_spec->files_limit); -+} -+ - int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec) - { - int ret = 0; -@@ -1290,6 +1334,11 @@ int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec) - goto out; - } - -+ ret = merge_conf_files_limit(oci_spec, host_spec); -+ if (ret != 0) { -+ goto out; -+ } -+ - out: - return ret; - } --- -2.25.1 - diff --git a/0040-support-setting-pod-to-privilege.patch b/0040-support-setting-pod-to-privilege.patch deleted file mode 100644 index b0ef0ee..0000000 --- a/0040-support-setting-pod-to-privilege.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 28ee9b5b9e09e0b9e4eaf87356af900d0e380ff0 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Thu, 30 Mar 2023 06:15:56 +0800 -Subject: [PATCH 40/46] support setting pod to privilege - -Signed-off-by: zhongtao ---- - src/daemon/entry/cri/cri_security_context.cc | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/daemon/entry/cri/cri_security_context.cc b/src/daemon/entry/cri/cri_security_context.cc -index 1cd9287d..b2843626 100644 ---- a/src/daemon/entry/cri/cri_security_context.cc -+++ b/src/daemon/entry/cri/cri_security_context.cc -@@ -209,6 +209,7 @@ void ApplySandboxSecurityContext(const runtime::v1alpha2::LinuxPodSandboxConfig - } - if (lc.has_security_context()) { - const runtime::v1alpha2::LinuxSandboxSecurityContext &old = lc.security_context(); -+ sc->set_privileged(old.privileged()); - if (old.has_run_as_user()) { - *sc->mutable_run_as_user() = old.run_as_user(); - } --- -2.25.1 - diff --git a/0041-add-hugepage_limit.patch b/0041-add-hugepage_limit.patch deleted file mode 100644 index d110157..0000000 --- a/0041-add-hugepage_limit.patch +++ /dev/null @@ -1,76 +0,0 @@ -From 1a4e5174a9abcc83c9ace0cf7cabbdaf03697ae3 Mon Sep 17 00:00:00 2001 -From: shijiaqi1 -Date: Wed, 8 Feb 2023 13:31:36 +0800 -Subject: [PATCH 41/46] add hugepage_limit - ---- - .../cri/cri_container_manager_service_impl.cc | 19 +++++++++++++++++ - src/daemon/entry/cri/cri_helpers.cc | 21 ++++++++++++++++++- - 2 files changed, 39 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -index 6278512f..5398c088 100644 ---- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc -+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc -@@ -1226,6 +1226,25 @@ void ContainerManagerServiceImpl::UpdateContainerResources(const std::string &co - if (!resources.cpuset_mems().empty()) { - hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); - } -+ if (resources.hugepage_limits_size() != 0) { -+ hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( -+ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); -+ if (hostconfig->hugetlbs == nullptr) { -+ error.SetError("Out of memory"); -+ return; -+ } -+ for (int i = 0; i < resources.hugepage_limits_size(); i++) { -+ hostconfig->hugetlbs[i] = -+ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); -+ if (hostconfig->hugetlbs[i] == nullptr) { -+ error.SetError("Out of memory"); -+ goto cleanup; -+ } -+ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); -+ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); -+ hostconfig->hugetlbs_len++; -+ } -+ } - - request->host_config = host_config_generate_json(hostconfig, &ctx, &perror); - if (request->host_config == nullptr) { -diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc -index ec14d0b2..e588b6c4 100644 ---- a/src/daemon/entry/cri/cri_helpers.cc -+++ b/src/daemon/entry/cri/cri_helpers.cc -@@ -447,8 +447,27 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, - hc->cpuset_mems = util_strdup_s(rOpts.cpuset_mems().c_str()); - } - hc->oom_score_adj = rOpts.oom_score_adj(); -- } - -+ if (rOpts.hugepage_limits_size() != 0) { -+ hc->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(sizeof(host_config_hugetlbs_element *), -+ rOpts.hugepage_limits_size()); -+ if (hc->hugetlbs == nullptr) { -+ error.SetError("Out of memory"); -+ return; -+ } -+ for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { -+ hc->hugetlbs[i] = -+ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); -+ if (hc->hugetlbs[i] == nullptr) { -+ error.SetError("Out of memory"); -+ return; -+ } -+ hc->hugetlbs[i]->page_size = util_strdup_s(rOpts.hugepage_limits(i).page_size().c_str()); -+ hc->hugetlbs[i]->limit = rOpts.hugepage_limits(i).limit(); -+ hc->hugetlbs_len++; -+ } -+ } -+ } - createConfig->open_stdin = config.stdin(); - createConfig->tty = config.tty(); - } --- -2.25.1 - diff --git a/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch b/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch deleted file mode 100644 index ef6db49..0000000 --- a/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch +++ /dev/null @@ -1,232 +0,0 @@ -From 8e1bd51183eeb2ca2713b85afd52b12e584bc7a8 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Wed, 12 Apr 2023 21:19:56 +0800 -Subject: [PATCH 42/46] add effective and permitted type of cap to oci spec - -Signed-off-by: zhongtao ---- - src/contrib/config/config.json | 28 ++++++ - .../config/systemcontainer_config.json | 28 ++++++ - src/daemon/modules/spec/specs_security.c | 88 ++++++++++++++----- - 3 files changed, 120 insertions(+), 24 deletions(-) - -diff --git a/src/contrib/config/config.json b/src/contrib/config/config.json -index f84f3394..9070a893 100644 ---- a/src/contrib/config/config.json -+++ b/src/contrib/config/config.json -@@ -36,10 +36,38 @@ - "CAP_AUDIT_WRITE" - ], - "effective": [ -+ "CAP_CHOWN", -+ "CAP_DAC_OVERRIDE", -+ "CAP_FSETID", -+ "CAP_FOWNER", -+ "CAP_MKNOD", -+ "CAP_NET_RAW", -+ "CAP_SETGID", -+ "CAP_SETUID", -+ "CAP_SETFCAP", -+ "CAP_SETPCAP", -+ "CAP_NET_BIND_SERVICE", -+ "CAP_SYS_CHROOT", -+ "CAP_KILL", -+ "CAP_AUDIT_WRITE" - ], - "inheritable": [ - ], - "permitted": [ -+ "CAP_CHOWN", -+ "CAP_DAC_OVERRIDE", -+ "CAP_FSETID", -+ "CAP_FOWNER", -+ "CAP_MKNOD", -+ "CAP_NET_RAW", -+ "CAP_SETGID", -+ "CAP_SETUID", -+ "CAP_SETFCAP", -+ "CAP_SETPCAP", -+ "CAP_NET_BIND_SERVICE", -+ "CAP_SYS_CHROOT", -+ "CAP_KILL", -+ "CAP_AUDIT_WRITE" - ], - "ambient": [ - ] -diff --git a/src/contrib/config/systemcontainer_config.json b/src/contrib/config/systemcontainer_config.json -index 8ebce8c6..9169956e 100644 ---- a/src/contrib/config/systemcontainer_config.json -+++ b/src/contrib/config/systemcontainer_config.json -@@ -36,10 +36,38 @@ - "CAP_AUDIT_WRITE" - ], - "effective": [ -+ "CAP_CHOWN", -+ "CAP_DAC_OVERRIDE", -+ "CAP_FSETID", -+ "CAP_FOWNER", -+ "CAP_MKNOD", -+ "CAP_NET_RAW", -+ "CAP_SETGID", -+ "CAP_SETUID", -+ "CAP_SETFCAP", -+ "CAP_SETPCAP", -+ "CAP_NET_BIND_SERVICE", -+ "CAP_SYS_CHROOT", -+ "CAP_KILL", -+ "CAP_AUDIT_WRITE" - ], - "inheritable": [ - ], - "permitted": [ -+ "CAP_CHOWN", -+ "CAP_DAC_OVERRIDE", -+ "CAP_FSETID", -+ "CAP_FOWNER", -+ "CAP_MKNOD", -+ "CAP_NET_RAW", -+ "CAP_SETGID", -+ "CAP_SETUID", -+ "CAP_SETFCAP", -+ "CAP_SETPCAP", -+ "CAP_NET_BIND_SERVICE", -+ "CAP_SYS_CHROOT", -+ "CAP_KILL", -+ "CAP_AUDIT_WRITE" - ], - "ambient": [ - ] -diff --git a/src/daemon/modules/spec/specs_security.c b/src/daemon/modules/spec/specs_security.c -index 62f67082..d4884097 100644 ---- a/src/daemon/modules/spec/specs_security.c -+++ b/src/daemon/modules/spec/specs_security.c -@@ -261,36 +261,80 @@ free_out: - return ret; - } - -+// tweak_all_type_capabilities can tweak all support type capabilities by adding or dropping capabilities -+// based on the basic capabilities. -+static int tweak_all_type_capabilities(defs_process_capabilities **caps, const char **adds, size_t adds_len, -+ const char **drops, size_t drops_len) -+{ -+ int ret = 0; -+ ret = tweak_capabilities(&((*caps)->bounding), &((*caps)->bounding_len), adds, adds_len, drops, drops_len); -+ if (ret != 0) { -+ ERROR("Failed to tweak bounding capabilities"); -+ return -1; -+ } -+ -+ ret = tweak_capabilities(&((*caps)->permitted), &((*caps)->permitted_len), adds, adds_len, drops, drops_len); -+ if (ret != 0) { -+ ERROR("Failed to tweak permitted capabilities"); -+ return -1; -+ } -+ -+ ret = tweak_capabilities(&((*caps)->effective), &((*caps)->effective_len), adds, adds_len, drops, drops_len); -+ if (ret != 0) { -+ ERROR("Failed to tweak effective capabilities"); -+ return -1; -+ } -+ return ret; -+} -+ -+static void clear_caps(defs_process_capabilities **caps) -+{ -+ util_free_array_by_len((*caps)->bounding, (*caps)->bounding_len); -+ (*caps)->bounding_len = 0; -+ (*caps)->bounding = NULL; -+ -+ util_free_array_by_len((*caps)->permitted, (*caps)->permitted_len); -+ (*caps)->permitted_len = 0; -+ (*caps)->permitted = NULL; -+ -+ util_free_array_by_len((*caps)->effective, (*caps)->effective_len); -+ (*caps)->effective_len = 0; -+ (*caps)->effective = NULL; -+} -+ - int refill_oci_process_capabilities(defs_process_capabilities **caps, const char **src_caps, size_t src_caps_len) - { - int ret = 0; -- size_t i = 0; - - if (*caps == NULL) { - *caps = util_common_calloc_s(sizeof(defs_process_capabilities)); - if (*caps == NULL) { -- ret = -1; -- goto out; -+ return -1; - } - } - -- if ((*caps)->bounding != NULL) { -- // free current capabilities -- for (i = 0; i < ((*caps)->bounding_len); i++) { -- free((*caps)->bounding[i]); -- (*caps)->bounding[i] = NULL; -- } -- free((*caps)->bounding); -- (*caps)->bounding = NULL; -- } -- (*caps)->bounding_len = 0; -+ // clear current capabilities -+ clear_caps(caps); - - // copy capabilities - ret = copy_capabilities(&((*caps)->bounding), &((*caps)->bounding_len), src_caps, src_caps_len); - if (ret != 0) { -- ERROR("Failed to copy all capabilities"); -+ ERROR("Failed to copy all bounding capabilities"); -+ return -1; - } --out: -+ -+ ret = copy_capabilities(&((*caps)->permitted), &((*caps)->permitted_len), src_caps, src_caps_len); -+ if (ret != 0) { -+ ERROR("Failed to copy all permitted capabilities"); -+ return -1; -+ } -+ -+ ret = copy_capabilities(&((*caps)->effective), &((*caps)->effective_len), src_caps, src_caps_len); -+ if (ret != 0) { -+ ERROR("Failed to copy all effective capabilities"); -+ return -1; -+ } -+ - return ret; - } - -@@ -823,25 +867,21 @@ int merge_caps(oci_runtime_spec *oci_spec, const char **adds, size_t adds_len, c - - ret = make_sure_oci_spec_process_capabilities(oci_spec); - if (ret < 0) { -- goto out; -+ return ret; - } - - if (adds_len > LIST_SIZE_MAX || drops_len > LIST_SIZE_MAX) { - ERROR("Too many capabilities to add or drop, the limit is %lld", LIST_SIZE_MAX); - isulad_set_error_message("Too many capabilities to add or drop, the limit is %d", LIST_SIZE_MAX); -- ret = -1; -- goto out; -+ return -1; - } - -- ret = tweak_capabilities(&oci_spec->process->capabilities->bounding, &oci_spec->process->capabilities->bounding_len, -- adds, adds_len, drops, drops_len); -+ ret = tweak_all_type_capabilities(&oci_spec->process->capabilities, adds, adds_len, drops, drops_len); - if (ret != 0) { -- ERROR("Failed to tweak capabilities"); -- ret = -1; -- goto out; -+ ERROR("Failed to tweak all type capabilities"); -+ return -1; - } - --out: - return ret; - } - --- -2.25.1 - diff --git a/0043-isulad-shim-fix-log-loss-bug.patch b/0043-isulad-shim-fix-log-loss-bug.patch deleted file mode 100644 index 28fdf44..0000000 --- a/0043-isulad-shim-fix-log-loss-bug.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 795f9f56bd33ed382ae1e68300247ef651d77864 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sun, 16 Apr 2023 19:57:48 +1400 -Subject: [PATCH 43/46] isulad-shim: fix log loss bug - -Signed-off-by: zhongtao ---- - src/cmd/isulad-shim/process.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index 02609911..a676e7ce 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -1348,6 +1348,12 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons - destroy_io_thread(p, i); - } - -+ if (!p->state->exec) { -+ // if log did not contain "/n", print remaind container log when exit isulad-shim -+ shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0); -+ shim_write_container_log_file(p->terminal, STDID_ERR, NULL, 0); -+ } -+ - if (ret == SHIM_ERR_TIMEOUT) { - write_message(g_log_fd, INFO_MSG, "Wait %d timeout", p->ctr_pid); - exit(SHIM_EXIT_TIMEOUT); --- -2.25.1 - diff --git a/0044-remove-unused-func.patch b/0044-remove-unused-func.patch deleted file mode 100644 index b9a8693..0000000 --- a/0044-remove-unused-func.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 2b3a35ec1e0e3afd090618f2120fd8e756c4c0c6 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Wed, 19 Apr 2023 07:22:42 +0800 -Subject: [PATCH 44/46] remove unused func - -Signed-off-by: zhongtao ---- - .../modules/runtime/isula/isula_rt_ops.c | 31 ++----------------- - 1 file changed, 2 insertions(+), 29 deletions(-) - -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index 5a01b8c6..3a6269a1 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -1078,28 +1078,6 @@ int rt_isula_rm(const char *id, const char *runtime, const rt_rm_params_t *param - return 0; - } - --static char *try_generate_exec_id() --{ -- char *id = NULL; -- -- id = util_common_calloc_s(sizeof(char) * (CONTAINER_EXEC_ID_MAX_LEN + 1)); -- if (id == NULL) { -- ERROR("Out of memory"); -- return NULL; -- } -- -- if (util_generate_random_str(id, (size_t)CONTAINER_EXEC_ID_MAX_LEN) != 0) { -- ERROR("Generate id failed"); -- goto err_out; -- } -- -- return id; -- --err_out: -- free(id); -- return NULL; --} -- - static bool fg_exec(const rt_exec_params_t *params) - { - if (params->console_fifos[0] != NULL || params->console_fifos[1] != NULL || params->console_fifos[2] != NULL) { -@@ -1122,7 +1100,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - int pid = 0; - shim_client_process_state p = { 0 }; - -- if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL) { -+ if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL || params->suffix == NULL) { - ERROR("nullptr arguments not allowed"); - return -1; - } -@@ -1135,12 +1113,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - return -1; - } - -- if (params->suffix != NULL) { -- exec_id = util_strdup_s(params->suffix); -- } else { -- exec_id = try_generate_exec_id(); -- } -- -+ exec_id = util_strdup_s(params->suffix); - if (exec_id == NULL) { - ERROR("Out of memory or generate exec id failed"); - return -1; --- -2.25.1 - diff --git a/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch b/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch deleted file mode 100644 index 3470175..0000000 --- a/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 26c4a702ad9c919d8413b64f498d58b13375bbda Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Tue, 18 Apr 2023 17:30:03 +0800 -Subject: [PATCH 45/46] if the exit code in the response of execSync is not - zero, an error will be returned in cri - -Signed-off-by: zhongtao ---- - src/daemon/entry/connect/grpc/runtime_runtime_service.cc | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -index 451eeeef..4cc5a4ee 100644 ---- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -+++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc -@@ -289,6 +289,13 @@ grpc::Status RuntimeRuntimeServiceImpl::ExecSync(grpc::ServerContext *context, - return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); - } - -+ if (reply->exit_code() != 0) { -+ ERROR("Object: CRI, Type: Sync exec in container: %s with exit code: %d", request->container_id().c_str(), -+ reply->exit_code()); -+ error.SetError(reply->stderr()); -+ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); -+ } -+ - WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); - - return grpc::Status::OK; --- -2.25.1 - diff --git a/0046-free-timeout-when-shim_create-finished.patch b/0046-free-timeout-when-shim_create-finished.patch deleted file mode 100644 index 3cabcb5..0000000 --- a/0046-free-timeout-when-shim_create-finished.patch +++ /dev/null @@ -1,89 +0,0 @@ -From 4dfc94f2beb816eb2e26ede07e803a230405b193 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Thu, 20 Apr 2023 13:28:26 +0800 -Subject: [PATCH 46/46] free timeout when shim_create finished - -Signed-off-by: zhongtao ---- - .../modules/runtime/isula/isula_rt_ops.c | 30 +++++++++++++------ - 1 file changed, 21 insertions(+), 9 deletions(-) - -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index 3a6269a1..ceaf464e 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -694,7 +694,7 @@ static int status_to_exit_code(int status) - } - - static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, -- int *exit_code, const int64_t timeout) -+ int *exit_code, const char* timeout) - { - pid_t pid = 0; - int exec_fd[2] = { -1, -1 }; -@@ -712,12 +712,8 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - params[i++] = runtime_cmd; - params[i++] = "info"; - // execSync timeout -- if (timeout > 0) { -- params[i] = util_int_to_string(timeout); -- if (params[i] == NULL) { -- ERROR("Failed to convert execSync timeout %ld to string", timeout); -- return -1; -- } -+ if (timeout != NULL) { -+ params[i++] = timeout; - } - runtime_exec_param_dump(params); - -@@ -917,7 +913,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ - } - - get_runtime_cmd(runtime, &cmd); -- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); -+ ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, NULL); - if (ret != 0) { - runtime_call_delete_force(workdir, runtime, id); - ERROR("%s: failed create shim process", id); -@@ -1099,6 +1095,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - char bundle[PATH_MAX] = { 0 }; - int pid = 0; - shim_client_process_state p = { 0 }; -+ char *timeout = NULL; - - if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL || params->suffix == NULL) { - ERROR("nullptr arguments not allowed"); -@@ -1158,7 +1155,18 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - } - - get_runtime_cmd(runtime, &cmd); -- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); -+ -+ // execSync timeout -+ if (params->timeout > 0) { -+ timeout = util_int_to_string(params->timeout); -+ if (timeout == NULL) { -+ ERROR("Failed to convert execSync timeout %ld to string", params->timeout); -+ ret = -1; -+ goto del_out; -+ } -+ } -+ -+ ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, timeout); - if (ret != 0) { - ERROR("%s: failed create shim process for exec %s", id, exec_id); - goto errlog_out; -@@ -1183,6 +1191,10 @@ errlog_out: - show_shim_runtime_errlog(workdir); - } - -+ if (timeout != NULL) { -+ free(timeout); -+ } -+ - del_out: - if (util_recursive_rmdir(workdir, 0)) { - ERROR("rmdir %s failed", workdir); --- -2.25.1 - diff --git a/0047-clean-isulad-shim-compile-relies.patch b/0047-clean-isulad-shim-compile-relies.patch deleted file mode 100644 index 56ad8be..0000000 --- a/0047-clean-isulad-shim-compile-relies.patch +++ /dev/null @@ -1,347 +0,0 @@ -From fde1c406a837b849f1182d8943f1f942088b608d Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sun, 23 Apr 2023 15:01:58 +0800 -Subject: [PATCH 47/56] clean isulad shim compile relies - -Signed-off-by: zhongtao ---- - src/CMakeLists.txt | 12 +- - src/cmd/isulad-shim/common.c | 179 ++++++++++++++++++++++++++++ - src/cmd/isulad-shim/common.h | 49 ++++++++ - src/cmd/isulad-shim/process.c | 3 - - test/cmd/isulad-shim/CMakeLists.txt | 13 -- - 5 files changed, 238 insertions(+), 18 deletions(-) - -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index f3dd3c19..02d7b13f 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -113,8 +113,16 @@ endif() - add_executable(isulad-shim - ${ISULAD_SHIM_SRCS} - ) --target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS}) --target_link_libraries(isulad-shim libisulad_tools) -+target_include_directories(isulad-shim PUBLIC -+ ${ISULAD_SHIM_INCS} -+ ${CMAKE_CURRENT_SOURCE_DIR} -+ ${COMMON_INCS} -+ ${CMAKE_BINARY_DIR}/conf -+ ${CHECKED_INCLUDE_DIRS} -+ ${SHARED_INCS} -+ ${ISULA_LIBUTILS_INCLUDE_DIR} -+ ) -+target_link_libraries(isulad-shim ${ISULA_LIBUTILS_LIBRARY}) - if (ANDROID OR MUSL) - target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY}) - else() -diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c -index f188da1e..e1ca96e1 100644 ---- a/src/cmd/isulad-shim/common.c -+++ b/src/cmd/isulad-shim/common.c -@@ -27,6 +27,7 @@ - #include - #include - #include -+#include - - int set_fd_no_inherited(int fd) - { -@@ -346,3 +347,181 @@ int shim_util_safe_uint64(const char *numstr, uint64_t *converted) - return 0; - } - -+void util_usleep_nointerupt(unsigned long usec) -+{ -+#define SECOND_TO_USECOND_MUTIPLE 1000000 -+ int ret = 0; -+ struct timespec request = { 0 }; -+ struct timespec remain = { 0 }; -+ if (usec == 0) { -+ return; -+ } -+ -+ request.tv_sec = (time_t)(usec / SECOND_TO_USECOND_MUTIPLE); -+ request.tv_nsec = (long)((usec % SECOND_TO_USECOND_MUTIPLE) * 1000); -+ -+ do { -+ ret = nanosleep(&request, &remain); -+ request = remain; -+ } while (ret == -1 && errno == EINTR); -+} -+ -+void *util_smart_calloc_s(size_t unit_size, size_t count) -+{ -+ if (unit_size == 0) { -+ return NULL; -+ } -+ -+ if (count > (MAX_MEMORY_SIZE / unit_size)) { -+ return NULL; -+ } -+ -+ return calloc(count, unit_size); -+} -+ -+size_t util_array_len(const char **array) -+{ -+ const char **pos; -+ size_t len = 0; -+ -+ for (pos = array; pos != NULL && *pos != NULL; pos++) { -+ len++; -+ } -+ -+ return len; -+} -+ -+void util_free_array(char **array) -+{ -+ char **p; -+ -+ for (p = array; p != NULL && *p != NULL; p++) { -+ UTIL_FREE_AND_SET_NULL(*p); -+ } -+ free(array); -+} -+ -+int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment) -+{ -+ size_t add_capacity; -+ char **add_array = NULL; -+ -+ if (orig_array == NULL || orig_capacity == NULL || increment == 0) { -+ return -1; -+ } -+ -+ if (((*orig_array) == NULL) || ((*orig_capacity) == 0)) { -+ UTIL_FREE_AND_SET_NULL(*orig_array); -+ *orig_capacity = 0; -+ } -+ -+ add_capacity = *orig_capacity; -+ while (size + 1 > add_capacity) { -+ add_capacity += increment; -+ } -+ if (add_capacity != *orig_capacity) { -+ add_array = util_smart_calloc_s(sizeof(void *), add_capacity); -+ if (add_array == NULL) { -+ return -1; -+ } -+ if (*orig_array != NULL) { -+ (void)memcpy(add_array, *orig_array, *orig_capacity * sizeof(void *)); -+ UTIL_FREE_AND_SET_NULL(*orig_array); -+ } -+ -+ *orig_array = add_array; -+ *orig_capacity = add_capacity; -+ } -+ -+ return 0; -+} -+ -+char *util_strdup_s(const char *src) -+{ -+ char *dst = NULL; -+ -+ if (src == NULL) { -+ return NULL; -+ } -+ -+ dst = strdup(src); -+ if (dst == NULL) { -+ abort(); -+ } -+ -+ return dst; -+} -+ -+static char **make_empty_array() -+{ -+ char **res_array = NULL; -+ -+ res_array = calloc(2, sizeof(char *)); -+ if (res_array == NULL) { -+ return NULL; -+ } -+ res_array[0] = util_strdup_s(""); -+ return res_array; -+} -+ -+static char **util_shrink_array(char **orig_array, size_t new_size) -+{ -+ char **new_array = NULL; -+ size_t i = 0; -+ -+ if (new_size == 0) { -+ return orig_array; -+ } -+ new_array = util_smart_calloc_s(sizeof(char *), new_size); -+ if (new_array == NULL) { -+ return orig_array; -+ } -+ -+ for (i = 0; i < new_size; i++) { -+ new_array[i] = orig_array[i]; -+ } -+ free(orig_array); -+ return new_array; -+} -+ -+char **util_string_split_multi(const char *src_str, char delim) -+{ -+ int ret, tmp_errno; -+ char *token = NULL; -+ char *cur = NULL; -+ char **res_array = NULL; -+ char deli[2] = { delim, '\0' }; -+ size_t count = 0; -+ size_t capacity = 0; -+ char *tmpstr = NULL; -+ -+ if (src_str == NULL) { -+ return NULL; -+ } -+ -+ if (src_str[0] == '\0') { -+ return make_empty_array(); -+ } -+ -+ tmpstr = util_strdup_s(src_str); -+ cur = tmpstr; -+ token = strsep(&cur, deli); -+ while (token != NULL) { -+ ret = util_grow_array(&res_array, &capacity, count + 1, 16); -+ if (ret < 0) { -+ goto err_out; -+ } -+ res_array[count] = util_strdup_s(token); -+ count++; -+ token = strsep(&cur, deli); -+ } -+ free(tmpstr); -+ return util_shrink_array(res_array, count + 1); -+ -+err_out: -+ tmp_errno = errno; -+ free(tmpstr); -+ util_free_array(res_array); -+ errno = tmp_errno; -+ return NULL; -+} -\ No newline at end of file -diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h -index 91808295..3de16ace 100644 ---- a/src/cmd/isulad-shim/common.h -+++ b/src/cmd/isulad-shim/common.h -@@ -58,6 +58,43 @@ extern "C" { - #define CONTAINER_ACTION_REBOOT 129 - #define CONTAINER_ACTION_SHUTDOWN 130 - -+ -+void util_usleep_nointerupt(unsigned long usec); -+/** -+ * retry_cnt: max count of call cb; -+ * interval_us: how many us to sleep, after call cb; -+ * cb: retry call function; -+ * return: -+ * 0 is cb successful at least once; -+ * 1 is all cb are failure; -+*/ -+#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ -+ size_t i = 0; \ -+ for(; i < retry_cnt; i++) { \ -+ ret = cb(##__VA_ARGS__); \ -+ if (ret == 0) { \ -+ break; \ -+ } \ -+ util_usleep_nointerupt(interval_us); \ -+ } \ -+ } while(0) -+ -+#define UTIL_FREE_AND_SET_NULL(p) \ -+ do { \ -+ if ((p) != NULL) { \ -+ free((void *)(p)); \ -+ (p) = NULL; \ -+ } \ -+ } while (0) -+ -+#if __WORDSIZE == 64 -+// current max user memory for 64-machine is 2^47 B -+#define MAX_MEMORY_SIZE ((size_t)1 << 47) -+#else -+// current max user memory for 32-machine is 2^31 B -+#define MAX_MEMORY_SIZE ((size_t)1 << 31) -+#endif -+ - ssize_t read_nointr(int fd, void *buf, size_t count); - ssize_t write_nointr(int fd, const void *buf, size_t count); - -@@ -79,6 +116,18 @@ int open_no_inherit(const char *path, int flag, mode_t mode); - - int shim_util_safe_uint64(const char *numstr, uint64_t *converted); - -+void *util_smart_calloc_s(size_t unit_size, size_t count); -+ -+size_t util_array_len(const char **array); -+ -+void util_free_array(char **array); -+ -+int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment); -+ -+char *util_strdup_s(const char *src); -+ -+char **util_string_split_multi(const char *src_str, char delim); -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index a676e7ce..a5e0bd39 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -37,9 +37,6 @@ - - #include "common.h" - #include "terminal.h" --#include "utils_array.h" --#include "utils_string.h" --#include "utils.h" - - #define MAX_EVENTS 100 - #define DEFAULT_IO_COPY_BUF (16 * 1024) -diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt -index dc293f6d..e5c1cd6e 100644 ---- a/test/cmd/isulad-shim/CMakeLists.txt -+++ b/test/cmd/isulad-shim/CMakeLists.txt -@@ -6,26 +6,13 @@ add_executable(${EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c - isulad-shim_ut.cc) - - target_include_directories(${EXE} PUBLIC - ${GTEST_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256 -- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils - ${CMAKE_BINARY_DIR}/conf - ) - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) --- -2.25.1 - diff --git a/0048-remote-layer-store-demo.patch b/0048-remote-layer-store-demo.patch deleted file mode 100644 index c5c93c5..0000000 --- a/0048-remote-layer-store-demo.patch +++ /dev/null @@ -1,2290 +0,0 @@ -From fdd49ec5b0cea74acfb05482045042280c2f2eaa Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Mon, 9 Jan 2023 17:12:53 -0800 -Subject: [PATCH 48/56] remote layer store demo - -Signed-off-by: Neil ---- - cmake/options.cmake | 6 + - src/daemon/config/isulad_config.c | 1 + - src/daemon/modules/image/oci/oci_image.c | 4 + - .../modules/image/oci/storage/CMakeLists.txt | 6 + - .../storage/image_store/image_remote_impl.c | 173 +++++++++ - .../oci/storage/image_store/image_store.c | 76 +++- - .../oci/storage/image_store/image_store.h | 11 + - .../storage/layer_store/graphdriver/driver.c | 4 +- - .../storage/layer_store/graphdriver/driver.h | 3 + - .../graphdriver/overlay2/driver_overlay2.c | 158 +++++++- - .../graphdriver/overlay2/driver_overlay2.h | 8 + - .../overlay2/overlay_remote_impl.c | 282 ++++++++++++++ - .../storage/layer_store/layer_remote_impl.c | 219 +++++++++++ - .../oci/storage/layer_store/layer_store.c | 200 +++++++++- - .../oci/storage/layer_store/layer_store.h | 11 + - .../remote_layer_support/CMakeLists.txt | 12 + - .../remote_layer_support/remote_support.c | 122 ++++++ - .../remote_layer_support/remote_support.h | 58 +++ - .../ro_symlink_maintain.c | 347 ++++++++++++++++++ - .../ro_symlink_maintain.h | 52 +++ - .../modules/image/oci/storage/storage.c | 11 +- - .../modules/image/oci/storage/storage.h | 3 + - 22 files changed, 1749 insertions(+), 18 deletions(-) - create mode 100644 src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c - create mode 100644 src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c - create mode 100644 src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c - create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt - create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c - create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h - create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h - -diff --git a/cmake/options.cmake b/cmake/options.cmake -index a6905c17..701082dd 100644 ---- a/cmake/options.cmake -+++ b/cmake/options.cmake -@@ -121,6 +121,12 @@ if (DISABLE_CLEANUP STREQUAL "ON") - message("${Green}-- Disable cleanup module") - endif() - -+option(ENABLE_REMOTE_LAYER_STORE "enable remote layer store" OFF) -+if (ENABLE_REMOTE_LAYER_STORE STREQUAL "ON") -+ add_definitions(-DENABLE_REMOTE_LAYER_STORE) -+ message("${Green}-- Enable remote layer store") -+endif() -+ - option(MUSL "available for musl" OFF) - if (MUSL) - add_definitions(-D__MUSL__) -diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c -index ad82aeff..996917c4 100644 ---- a/src/daemon/config/isulad_config.c -+++ b/src/daemon/config/isulad_config.c -@@ -1515,6 +1515,7 @@ static int merge_authorization_conf_into_global(struct service_arguments *args, - static int merge_storage_conf_into_global(struct service_arguments *args, isulad_daemon_configs *tmp_json_confs) - { - override_string_value(&args->json_confs->storage_driver, &tmp_json_confs->storage_driver); -+ args->json_confs->storage_enable_remote_layer = tmp_json_confs->storage_enable_remote_layer; - - if (string_array_append(tmp_json_confs->storage_opts, tmp_json_confs->storage_opts_len, - &(args->json_confs->storage_opts_len), &(args->json_confs->storage_opts)) != 0) { -diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c -index 0fc0b7fe..e4442a79 100644 ---- a/src/daemon/modules/image/oci/oci_image.c -+++ b/src/daemon/modules/image/oci/oci_image.c -@@ -214,6 +214,10 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) - goto out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ storage_opts->enable_remote_layer = args->storage_enable_remote_layer; -+#endif -+ - if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, - &storage_opts->driver_opts_len) != 0) { - ERROR("Failed to get storage storage opts"); -diff --git a/src/daemon/modules/image/oci/storage/CMakeLists.txt b/src/daemon/modules/image/oci/storage/CMakeLists.txt -index 06c2f378..24b71450 100644 ---- a/src/daemon/modules/image/oci/storage/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/CMakeLists.txt -@@ -3,12 +3,17 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_storage_srcs) - add_subdirectory(image_store) - add_subdirectory(layer_store) - add_subdirectory(rootfs_store) -+IF (ENABLE_REMOTE_LAYER_STORE) -+add_subdirectory(remote_layer_support) -+ENDIF() -+ - - set(STORAGE_SRCS - ${local_storage_srcs} - ${IMAGE_STORE_SRCS} - ${LAYER_STORE_SRCS} - ${ROOTFS_STORE_SRCS} -+ ${REMOTE_LAYER_SUPPORT_SRCS} - PARENT_SCOPE - ) - -@@ -17,5 +22,6 @@ set(STORAGE_INCS - ${IMAGE_STORE_INCS} - ${LAYER_STORE_INCS} - ${ROOTFS_STORE_INCS} -+ ${REMOTE_LAYER_SUPPORT_INCS} - PARENT_SCOPE - ) -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c -new file mode 100644 -index 00000000..20da8116 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c -@@ -0,0 +1,173 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-03 -+ * Description: provide remote image store functions -+ ******************************************************************************/ -+#define _GNU_SOURCE -+#include "image_store.h" -+ -+#include -+#include -+ -+#include "remote_support.h" -+#include "ro_symlink_maintain.h" -+#include "map.h" -+#include "utils_file.h" -+#include "utils.h" -+#include "layer_store.h" -+#include "utils_array.h" -+ -+struct remote_image_data { -+ const char *image_home; -+}; -+ -+static map_t *image_byid_old = NULL; -+static map_t *image_byid_new = NULL; -+ -+static void *remote_support_create(const char *remote_home, const char *remote_ro) -+{ -+ struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); -+ if (data == NULL) { -+ ERROR("Out of memory"); -+ return NULL; -+ } -+ data->image_home = remote_home; -+ image_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ image_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ return data; -+} -+ -+static void remote_support_destroy(void *data) -+{ -+ if (data == NULL) { -+ return; -+ } -+ -+ map_free(image_byid_old); -+ map_free(image_byid_new); -+ -+ free(data); -+ return; -+} -+ -+static int remote_support_scan(void *data) -+{ -+ int ret = 0; -+ int nret; -+ char **image_dirs = NULL; -+ size_t image_dirs_num = 0; -+ size_t i; -+ char *id_patten = "^[a-f0-9]{64}$"; -+ char image_path[PATH_MAX] = { 0x00 }; -+ bool exist = true; -+ struct remote_image_data *img_data = (struct remote_image_data *)data; -+ -+ ret = util_list_all_subdir(img_data->image_home, &image_dirs); -+ if (ret != 0) { -+ ERROR("Failed to get images directory"); -+ goto out; -+ } -+ image_dirs_num = util_array_len((const char **)image_dirs); -+ -+ for (i = 0; i < image_dirs_num; i++) { -+ bool valid_v1_image = false; -+ -+ if (util_reg_match(id_patten, image_dirs[i]) != 0) { -+ DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", -+ image_dirs[i]); -+ continue; -+ } -+ -+ nret = snprintf(image_path, sizeof(image_path), "%s/%s", img_data->image_home, image_dirs[i]); -+ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { -+ ERROR("Failed to get image path"); -+ continue; -+ } -+ -+ if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { -+ ERROR("Failed to validate manifest schema version 1 format"); -+ continue; -+ } -+ -+ if (!valid_v1_image) { -+ map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); -+ } -+ } -+ -+out: -+ util_free_array(image_dirs); -+ return ret; -+} -+ -+static int remote_support_add(void *data) -+{ -+ char **array_added = NULL; -+ char **array_deleted = NULL; -+ char *top_layer = NULL; -+ map_t *tmp_map = NULL; -+ int i = 0; -+ int ret = 0; -+ -+ if (data == NULL) { -+ return -1; -+ } -+ -+ array_added = added_layers(image_byid_old, image_byid_new); -+ array_deleted = deleted_layers(image_byid_old, image_byid_new); -+ -+ for (i = 0; i < util_array_len((const char **)array_added); i++) { -+ top_layer = get_top_layer_from_json(array_added[i]); -+ if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { -+ ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); -+ map_remove(image_byid_new, (void *)array_added[i]); -+ continue; -+ } -+ -+ if (append_image_by_directory_with_lock(array_added[i]) != 0) { -+ ERROR("Failed to load image into memrory: %s", array_added[i]); -+ ret = -1; -+ } -+ } -+ -+ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { -+ if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { -+ ERROR("Failed to remove remote memory store"); -+ ret = -1; -+ } -+ } -+ -+ tmp_map = image_byid_old; -+ image_byid_old = image_byid_new; -+ image_byid_new = tmp_map; -+ empty_map(image_byid_new); -+ -+ util_free_array(array_added); -+ util_free_array(array_deleted); -+ free(top_layer); -+ -+ return ret; -+} -+ -+remote_support *image_store_impl_remote_support(void) -+{ -+ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -+ if (rs == NULL) { -+ return NULL; -+ } -+ -+ rs->create = remote_support_create; -+ rs->destroy = remote_support_destroy; -+ rs->scan_remote_dir = remote_support_scan; -+ rs->load_item = remote_support_add; -+ -+ return rs; -+} -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c -index 7e1a5373..c6836e46 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.c -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c -@@ -50,6 +50,9 @@ - #include "image_type.h" - #include "linked_list.h" - #include "utils_verify.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "ro_symlink_maintain.h" -+#endif - - // the name of the big data item whose contents we consider useful for computing a "digest" of the - // image, by which we can locate the image later. -@@ -3099,7 +3102,7 @@ out: - return ret; - } - --static int validate_manifest_schema_version_1(const char *path, bool *valid) -+int validate_manifest_schema_version_1(const char *path, bool *valid) - { - int ret = 0; - int nret; -@@ -3643,6 +3646,10 @@ int image_store_init(struct storage_module_init_options *opts) - goto out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ remote_image_init(g_image_store->dir); -+#endif -+ - out: - if (ret != 0) { - free_image_store(g_image_store); -@@ -3651,3 +3658,70 @@ out: - free(root_dir); - return ret; - } -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+int append_image_by_directory_with_lock(const char *id) -+{ -+ int ret = 0; -+ int nret = 0; -+ char image_path[PATH_MAX] = { 0x00 }; -+ -+ if (!image_store_lock(EXCLUSIVE)) { -+ ERROR("Failed to lock remote image store when handle: %s", id); -+ return -1; -+ } -+ -+ nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); -+ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { -+ ERROR("Failed to get image path"); -+ return -1; -+ } -+ -+ ret = append_image_by_directory(image_path); -+ image_store_unlock(); -+ -+ return ret; -+} -+ -+int remove_image_from_memory_with_lock(const char *id) -+{ -+ int ret = 0; -+ -+ if (!image_store_lock(EXCLUSIVE)) { -+ ERROR("Failed to lock remote image store when handle: %s", id); -+ return -1; -+ } -+ -+ ret = remove_image_from_memory(id); -+ image_store_unlock(); -+ -+ return ret; -+} -+ -+char *get_top_layer_from_json(const char *img_id) -+{ -+ -+ char *ret = NULL; -+ int nret = 0; -+ char image_path[PATH_MAX] = { 0x00 }; -+ storage_image *im = NULL; -+ parser_error err = NULL; -+ -+ nret = snprintf(image_path, sizeof(image_path), "%s/%s/%s", g_image_store->dir, img_id, IMAGE_JSON); -+ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { -+ ERROR("Failed to get image path"); -+ return NULL; -+ } -+ -+ im = storage_image_parse_file(image_path, NULL, &err); -+ if (im == NULL) { -+ ERROR("Failed to parse images path: %s", err); -+ return NULL; -+ } -+ -+ ret = util_strdup_s(im->layer); -+ free_storage_image(im); -+ -+ return ret; -+} -+#endif -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h -index edd4fa2c..c3cb50e3 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.h -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h -@@ -28,6 +28,9 @@ - #include "isula_libutils/imagetool_image.h" - #include "isula_libutils/imagetool_images_list.h" - #include "isula_libutils/imagetool_image_summary.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "remote_support.h" -+#endif - - struct storage_module_init_options; - -@@ -108,6 +111,14 @@ void image_store_free(); - - imagetool_image_summary *image_store_get_image_summary(const char *id); - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+remote_support *image_store_impl_remote_support(); -+int validate_manifest_schema_version_1(const char *path, bool *valid); -+int append_image_by_directory_with_lock(const char *image_dir); -+int remove_image_from_memory_with_lock(const char *id); -+char *get_top_layer_from_json(const char *img_id); /* return top layer id */ -+#endif -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -index d3b5209a..b83c63b1 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c -@@ -141,7 +141,9 @@ int graphdriver_init(const struct storage_module_init_options *opts) - ret = -1; - goto out; - } -- -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ g_drivers[i].enable_remote_layer = opts->enable_remote_layer; -+#endif - if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts, - opts->driver_opts_len) != 0) { - ret = -1; -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h -index 7faf70c8..acd847cc 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h -@@ -90,6 +90,9 @@ struct graphdriver { - bool support_dtype; - - bool support_quota; -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ bool enable_remote_layer; -+#endif - struct pquota_control *quota_ctrl; - - // options for overlay2 -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c -index 330c230a..eedbeef2 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c -@@ -12,6 +12,7 @@ - * Create: 2020-04-02 - * Description: provide overlay2 function definition - ******************************************************************************/ -+#define _GNU_SOURCE - #include "driver_overlay2.h" - - #include -@@ -44,6 +45,9 @@ - #include "utils_timestamp.h" - #include "selinux_label.h" - #include "err_msg.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "ro_symlink_maintain.h" -+#endif - - struct io_read_wrapper; - -@@ -343,6 +347,13 @@ int overlay2_init(struct graphdriver *driver, const char *driver_home, const cha - return -1; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (driver->enable_remote_layer && remote_overlay_init(driver_home) != 0) { -+ ERROR("Failed to init overlay remote"); -+ return -1; -+ } -+#endif -+ - driver->home = util_strdup_s(driver_home); - - root_dir = util_path_dir(driver_home); -@@ -423,7 +434,7 @@ static int mk_diff_directory(const char *layer_dir) - int ret = 0; - char *diff_dir = NULL; - #ifdef ENABLE_USERNS_REMAP -- char* userns_remap = conf_get_isulad_userns_remap(); -+ char *userns_remap = conf_get_isulad_userns_remap(); - #endif - - diff_dir = util_path_join(layer_dir, OVERLAY_LAYER_DIFF); -@@ -538,7 +549,7 @@ static int mk_work_directory(const char *layer_dir) - int ret = 0; - char *work_dir = NULL; - #ifdef ENABLE_USERNS_REMAP -- char* userns_remap = conf_get_isulad_userns_remap(); -+ char *userns_remap = conf_get_isulad_userns_remap(); - #endif - - work_dir = util_path_join(layer_dir, OVERLAY_LAYER_WORK); -@@ -575,7 +586,7 @@ static int mk_merged_directory(const char *layer_dir) - int ret = 0; - char *merged_dir = NULL; - #ifdef ENABLE_USERNS_REMAP -- char* userns_remap = conf_get_isulad_userns_remap(); -+ char *userns_remap = conf_get_isulad_userns_remap(); - #endif - - merged_dir = util_path_join(layer_dir, OVERLAY_LAYER_MERGED); -@@ -852,13 +863,115 @@ out: - return ret; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+static int do_create_remote_ro(const char *id, const char *parent, const struct graphdriver *driver, -+ const struct driver_create_opts *create_opts) -+{ -+ int ret = 0; -+ int get_err = 0; -+ char *ro_symlink = NULL; -+ char *ro_home = NULL; -+ char *layer_dir = NULL; -+#ifdef ENABLE_USERNS_REMAP -+ char *userns_remap = conf_get_isulad_userns_remap(); -+#endif -+ -+ ro_home = util_path_join(driver->home, OVERLAY_RO_DIR); -+ if (ro_home == NULL) { -+ ERROR("Failed to join ro_home"); -+ ret = -1; -+ goto out; -+ } -+ -+ layer_dir = util_path_join(ro_home, id); -+ if (layer_dir == NULL) { -+ ERROR("Failed to join layer_dir"); -+ ret = -1; -+ goto out; -+ } -+ -+ ro_symlink = util_path_join(driver->home, id); -+ if (ro_symlink == NULL) { -+ ERROR("Failed to join ro_symlink"); -+ ret = -1; -+ goto out; -+ } -+ -+ if (layer_dir == NULL) { -+ ERROR("Failed to join layer dir:%s", id); -+ ret = -1; -+ goto out; -+ } -+ -+ if (check_parent_valid(parent, driver) != 0) { -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_mkdir_p(layer_dir, 0700) != 0) { -+ ERROR("Unable to create layer directory %s.", layer_dir); -+ ret = -1; -+ goto out; -+ } -+ -+ // mk symbol link -+ if (symlink(layer_dir, ro_symlink) != 0) { -+ SYSERROR("Unable to create symbol link to layer directory %s", layer_dir); -+ ret = -1; -+ goto err_out; -+ } -+ -+#ifdef ENABLE_USERNS_REMAP -+ if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { -+ ERROR("Unable to change directory %s owner for user remap.", layer_dir); -+ ret = -1; -+ goto out; -+ } -+#endif -+ -+ if (create_opts->storage_opt != NULL && create_opts->storage_opt->len != 0) { -+ if (set_layer_quota(layer_dir, create_opts->storage_opt, driver) != 0) { -+ ERROR("Unable to set layer quota %s", layer_dir); -+ ret = -1; -+ goto err_out; -+ } -+ } -+ -+ if (mk_sub_directories(id, parent, layer_dir, driver->home) != 0) { -+ ret = -1; -+ goto err_out; -+ } -+ -+ goto out; -+ -+err_out: -+ if (util_recursive_rmdir(layer_dir, 0)) { -+ ERROR("Failed to delete layer path: %s", layer_dir); -+ } -+ -+ // to remove a file -+ if (util_fileself_exists(ro_symlink) && !util_force_remove_file(ro_symlink, &get_err)) { -+ ERROR("Failed to remove symbol link %s", ro_symlink); -+ } -+ -+out: -+ free(layer_dir); -+ free(ro_home); -+ free(ro_symlink); -+#ifdef ENABLE_USERNS_REMAP -+ free(userns_remap); -+#endif -+ return ret; -+} -+#endif -+ - static int do_create(const char *id, const char *parent, const struct graphdriver *driver, - const struct driver_create_opts *create_opts) - { - int ret = 0; - char *layer_dir = NULL; - #ifdef ENABLE_USERNS_REMAP -- char* userns_remap = conf_get_isulad_userns_remap(); -+ char *userns_remap = conf_get_isulad_userns_remap(); - #endif - - layer_dir = util_path_join(driver->home, id); -@@ -1002,7 +1115,15 @@ int overlay2_create_ro(const char *id, const char *parent, const struct graphdri - return -1; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (driver->enable_remote_layer) { -+ return do_create_remote_ro(id, parent, driver, create_opts); -+ } else { -+ return do_create(id, parent, driver, create_opts); -+ } -+#else - return do_create(id, parent, driver, create_opts); -+#endif - } - - static char *read_layer_link_file(const char *layer_dir) -@@ -1047,6 +1168,9 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) - char *link_id = NULL; - char link_path[PATH_MAX] = { 0 }; - char clean_path[PATH_MAX] = { 0 }; -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ struct stat stat_buf; -+#endif - - if (id == NULL || driver == NULL) { - ERROR("Invalid input arguments"); -@@ -1079,11 +1203,34 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) - } - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (lstat(layer_dir, &stat_buf) < 0) { -+ SYSERROR("Failed to lstat path: %s", layer_dir); -+ ret = -1; -+ goto out; -+ } -+ -+ if (driver->enable_remote_layer && S_ISLNK(stat_buf.st_mode)) { -+ // jusdge if the dir is symlink? -+ if (remote_overlay_remove_ro_dir(id) != 0) { -+ ERROR("Failed to delete symlink to layer dir: %s", layer_dir); -+ ret = -1; -+ goto out; -+ } -+ } else { -+ if (util_recursive_rmdir(layer_dir, 0) != 0) { -+ SYSERROR("Failed to remove layer directory %s", layer_dir); -+ ret = -1; -+ goto out; -+ } -+ } -+#else - if (util_recursive_rmdir(layer_dir, 0) != 0) { - SYSERROR("Failed to remove layer directory %s", layer_dir); - ret = -1; - goto out; - } -+#endif - - out: - free(layer_dir); -@@ -1747,7 +1894,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const - int ret = 0; - #ifdef ENABLE_USERNS_REMAP - unsigned int size = 0; -- char* userns_remap = conf_get_isulad_userns_remap(); -+ char *userns_remap = conf_get_isulad_userns_remap(); - #endif - char *layer_dir = NULL; - char *layer_diff = NULL; -@@ -2166,3 +2313,4 @@ out: - free(layer_diff); - return ret; - } -+ -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -index e14271b1..5c1d93fb 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -@@ -22,6 +22,9 @@ - #include - - #include "driver.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "remote_support.h" -+#endif - - struct driver_create_opts; - struct driver_mount_opts; -@@ -68,6 +71,11 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap - - int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+remote_support *overlay_driver_impl_remote_support(void); -+bool overlay_remote_layer_valid(const char *layer_id); -+#endif -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c -new file mode 100644 -index 00000000..a674a00f ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c -@@ -0,0 +1,282 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-02-27 -+ * Description: provide remote implementation for driver overlay -+ ******************************************************************************/ -+#define _GNU_SOURCE -+#include "driver_overlay2.h" -+ -+#include -+ -+#include "map.h" -+#include "remote_support.h" -+#include "ro_symlink_maintain.h" -+#include "isula_libutils/log.h" -+#include "utils.h" -+#include "utils_array.h" -+#include "utils_file.h" -+#include "path.h" -+ -+#define OVERLAY_LINK_DIR "l" -+#define OVERLAY_LAYER_LINK "link" -+ -+struct remote_overlay_data { -+ const char *overlay_home; -+ const char *overlay_ro; -+}; -+ -+static map_t *overlay_byid_old = NULL; -+static map_t *overlay_byid_new = NULL; -+ -+static void *remote_support_create(const char *remote_home, const char *remote_ro) -+{ -+ struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); -+ if (data == NULL) { -+ ERROR("Out of memory"); -+ return NULL; -+ } -+ data->overlay_home = remote_home; -+ data->overlay_ro = remote_ro; -+ overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ -+ return data; -+} -+ -+static void remote_support_destroy(void *data) -+{ -+ if (data == NULL) { -+ return; -+ } -+ -+ map_free(overlay_byid_old); -+ map_free(overlay_byid_new); -+ free(data); -+} -+ -+static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -+{ -+ bool exist = true; -+ if (!map_insert(overlay_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { -+ ERROR("can't insert remote layer into map"); -+ return false; -+ } -+ -+ return true; -+} -+ -+static int remote_support_scan(void *data) -+{ -+ struct remote_overlay_data *remote_data = data; -+ return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); -+} -+ -+static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) -+{ -+ int ret = 0; -+ int nret = 0; -+ char target_path[PATH_MAX] = { 0 }; -+ char link_path[PATH_MAX] = { 0 }; -+ char clean_path[PATH_MAX] = { 0 }; -+ -+ nret = snprintf(target_path, PATH_MAX, "../%s/diff", id); -+ if (nret < 0 || nret >= PATH_MAX) { -+ ERROR("Failed to get target path %s", id); -+ ret = -1; -+ goto out; -+ } -+ -+ nret = snprintf(link_path, PATH_MAX, "%s/%s/%s", driver_home, OVERLAY_LINK_DIR, link_id); -+ if (nret < 0 || nret >= PATH_MAX) { -+ ERROR("Failed to get link path %s", link_id); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { -+ ERROR("failed to get clean path %s", link_path); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_fileself_exists(clean_path) && util_path_remove(clean_path) != 0) { -+ ERROR("failed to remove old symbol link"); -+ ret = -1; -+ goto out; -+ } -+ -+ nret = symlink(target_path, clean_path); -+ if (nret < 0) { -+ SYSERROR("Failed to create symlink from \"%s\" to \"%s\"", clean_path, target_path); -+ ret = -1; -+ goto out; -+ } -+ -+out: -+ return ret; -+} -+ -+static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) -+{ -+ char *ro_symlink = NULL; -+ char clean_path[PATH_MAX] = { 0 }; -+ int nret = 0; -+ int ret = 0; -+ -+ nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Create layer symbol link path failed"); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { -+ ERROR("Failed to clean path: %s", ro_symlink); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_path_remove(clean_path) != 0) { -+ SYSERROR("Failed to remove link path %s", clean_path); -+ } -+ -+out: -+ free(ro_symlink); -+ return ret; -+} -+ -+static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) -+{ -+ char *ro_symlink = NULL; -+ char *layer_dir = NULL; -+ char *link_file = NULL; -+ char *diff_symlink = NULL; -+ int ret = 0; -+ -+ ro_symlink = util_path_join(data->overlay_home, overlay_id); -+ if (ro_symlink == NULL) { -+ ERROR("Failed to join ro symlink path: %s", overlay_id); -+ ret = -1; -+ goto free_out; -+ } -+ -+ layer_dir = util_path_join(data->overlay_ro, overlay_id); -+ if (layer_dir == NULL) { -+ ERROR("Failed to join ro layer dir: %s", overlay_id); -+ ret = -1; -+ goto free_out; -+ } -+ -+ // add RO symbol link first -+ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { -+ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); -+ ret = -1; -+ goto free_out; -+ } -+ -+ // maintain link -+ // try read link file in layer_dir -+ // mk symlink between ro_symlink -+ link_file = util_path_join(layer_dir, OVERLAY_LAYER_LINK); -+ if (link_file == NULL) { -+ ERROR("Failed to get layer link file %s", layer_dir); -+ ret = -1; -+ goto free_out; -+ } -+ -+ if (!util_fileself_exists(link_file)) { -+ ERROR("link file for layer %s not exist", layer_dir); -+ ret = -1; -+ goto free_out; -+ } -+ -+ diff_symlink = util_read_content_from_file(link_file); -+ if (link_file == NULL) { -+ ERROR("Failed to read content from link file of layer %s", layer_dir); -+ ret = -1; -+ goto free_out; -+ } -+ -+ if (do_diff_symlink(overlay_id, diff_symlink, data->overlay_home) != 0) { -+ ERROR("Failed to add diff link for layer %s", overlay_id); -+ ret = -1; -+ } -+ -+free_out: -+ free(ro_symlink); -+ free(layer_dir); -+ free(link_file); -+ free(diff_symlink); -+ -+ return ret; -+} -+ -+static int remote_support_add(void *data) -+{ -+ int ret = 0; -+ char **array_added = NULL; -+ char **array_deleted = NULL; -+ map_t *tmp_map = NULL; -+ int i = 0; -+ -+ if (data == NULL) { -+ return -1; -+ } -+ -+ array_added = added_layers(overlay_byid_old, overlay_byid_new); -+ array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); -+ -+ for (i = 0; i < util_array_len((const char **)array_added); i++) { -+ if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { -+ ERROR("Failed to add remote overlay layer: %s", array_added[i]); -+ ret = -1; -+ } -+ } -+ -+ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { -+ if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { -+ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); -+ ret = -1; -+ } -+ } -+ -+ tmp_map = overlay_byid_old; -+ overlay_byid_old = overlay_byid_new; -+ overlay_byid_new = tmp_map; -+ empty_map(overlay_byid_new); -+ -+ util_free_array(array_added); -+ util_free_array(array_deleted); -+ -+ return ret; -+} -+ -+remote_support *overlay_driver_impl_remote_support(void) -+{ -+ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -+ if (rs == NULL) { -+ ERROR("Failed to calloc overlay supporter"); -+ return NULL; -+ } -+ -+ rs->create = remote_support_create; -+ rs->destroy = remote_support_destroy; -+ rs->scan_remote_dir = remote_support_scan; -+ rs->load_item = remote_support_add; -+ -+ return rs; -+} -+ -+bool overlay_remote_layer_valid(const char *layer_id) -+{ -+ return map_search(overlay_byid_old, (void *)layer_id) != NULL; -+} -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -new file mode 100644 -index 00000000..d03fc20b ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -@@ -0,0 +1,219 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-02-27 -+ * Description: remote layer store implementation -+ ******************************************************************************/ -+#define _GNU_SOURCE -+#include "layer_store.h" -+ -+#include -+#include -+#include -+ -+#include "map.h" -+#include "utils.h" -+#include "remote_support.h" -+#include "ro_symlink_maintain.h" -+#include "path.h" -+#include "driver_overlay2.h" -+ -+struct remote_layer_data { -+ const char *layer_home; -+ const char *layer_ro; -+}; -+ -+static map_t *layer_byid_old = NULL; -+static map_t *layer_byid_new = NULL; -+ -+static void *remote_support_create(const char *layer_home, const char *layer_ro) -+{ -+ struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); -+ if (data == NULL) { -+ ERROR("Out of memory"); -+ return NULL; -+ } -+ data->layer_home = util_strdup_s(layer_home); -+ data->layer_ro = util_strdup_s(layer_ro); -+ layer_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ layer_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ -+ return data; -+}; -+ -+static void remote_support_destroy(void *data) -+{ -+ if (data == NULL) { -+ return; -+ } -+ -+ map_free(layer_byid_old); -+ map_free(layer_byid_new); -+ free(data); -+} -+ -+static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) -+{ -+ bool exist = true; -+ -+ if (!map_insert(layer_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { -+ ERROR("can't insert remote layer into map"); -+ return false; -+ } -+ -+ return true; -+} -+ -+static int remote_support_scan(void *data) -+{ -+ struct remote_layer_data *remote_data = data; -+ return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); -+} -+ -+static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) -+{ -+ char *ro_symlink = NULL; -+ char clean_path[PATH_MAX] = { 0 }; -+ int nret = 0; -+ int ret = 0; -+ -+ nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Create layer symbol link path failed"); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { -+ ERROR("Failed to clean path: %s", ro_symlink); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_path_remove(clean_path) != 0) { -+ SYSERROR("Failed to remove link path %s", clean_path); -+ } -+ -+ if (remove_memory_stores_with_lock(layer_id) != 0) { -+ ERROR("Failed to remove remote layer store memory"); -+ ret = -1; -+ } -+ -+out: -+ free(ro_symlink); -+ return ret; -+ -+} -+ -+static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) -+{ -+ char *ro_symlink = NULL; -+ char *layer_dir = NULL; -+ int ret = 0; -+ -+ ro_symlink = util_path_join(data->layer_home, layer_id); -+ layer_dir = util_path_join(data->layer_ro, layer_id); -+ -+ if (ro_symlink == NULL) { -+ ERROR("Failed to join ro symlink path: %s", layer_id); -+ ret = -1; -+ goto free_out; -+ } -+ -+ if (layer_dir == NULL) { -+ ERROR("Failed to join ro layer dir: %s", layer_id); -+ ret = -1; -+ goto free_out; -+ } -+ // add symbol link first -+ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { -+ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); -+ ret = -1; -+ goto free_out; -+ } -+ // insert layer into memory -+ if (load_one_layer(layer_id) != 0) { -+ ERROR("Failed to load new layer: %s into memory", layer_id); -+ ret = -1; -+ } -+ -+free_out: -+ free(ro_symlink); -+ free(layer_dir); -+ -+ return ret; -+} -+ -+static int remote_support_add(void *data) -+{ -+ int ret = 0; -+ char **array_added = NULL; -+ char **array_deleted = NULL; -+ map_t *tmp_map = NULL; -+ int i = 0; -+ -+ if (data == NULL) { -+ return -1; -+ } -+ -+ array_added = added_layers(layer_byid_old, layer_byid_new); -+ array_deleted = deleted_layers(layer_byid_old, layer_byid_new); -+ -+ for (i = 0; i < util_array_len((const char **)array_added); i++) { -+ if (!overlay_remote_layer_valid(array_added[i]) != 0) { -+ map_remove(layer_byid_new, (void *)array_added[i]); -+ ERROR("remote overlay layer current not valid: %s", array_added[i]); -+ continue; -+ } -+ -+ if (add_one_remote_layer(data, array_added[i]) != 0) { -+ ERROR("Failed to add remote overlay layer: %s", array_added[i]); -+ ret = -1; -+ } -+ } -+ -+ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { -+ if (remove_one_remote_layer(data, array_deleted[i]) != 0) { -+ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); -+ ret = -1; -+ } -+ } -+ -+ tmp_map = layer_byid_old; -+ layer_byid_old = layer_byid_new; -+ layer_byid_new = tmp_map; -+ empty_map(layer_byid_new); -+ -+ util_free_array(array_added); -+ util_free_array(array_deleted); -+ -+ return ret; -+} -+ -+remote_support *layer_store_impl_remote_support() -+{ -+ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -+ if (rs == NULL) { -+ return NULL; -+ } -+ -+ rs->create = remote_support_create; -+ rs->destroy = remote_support_destroy; -+ rs->scan_remote_dir = remote_support_scan; -+ rs->load_item = remote_support_add; -+ -+ return rs; -+} -+ -+bool layer_remote_layer_valid(const char *layer_id) -+{ -+ return map_search(layer_byid_old, (void *)layer_id) != NULL; -+} -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -index 60aaff22..8b8f5f1e 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -@@ -31,8 +31,6 @@ - #include - #include - #include --#include -- - #include - #include - -@@ -50,6 +48,10 @@ - #include "http.h" - #include "utils_base64.h" - #include "constants.h" -+#include "path.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "ro_symlink_maintain.h" -+#endif - - #define PAYLOAD_CRC_LEN 12 - -@@ -78,6 +80,9 @@ typedef struct { - static layer_store_metadata g_metadata; - static char *g_root_dir; - static char *g_run_dir; -+#ifdef ENABLE_REMOTE_LAYER_STORE -+static bool g_enable_remote_layer; -+#endif - - static inline char *tar_split_path(const char *id); - static inline char *mountpoint_json_path(const char *id); -@@ -131,7 +136,7 @@ void layer_store_cleanup() - map_free(g_metadata.by_uncompress_digest); - g_metadata.by_uncompress_digest = NULL; - -- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { -+ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { - linked_list_del(item); - layer_ref_dec((layer_t *)item->elem); - free(item); -@@ -162,7 +167,7 @@ static void free_digest_layer_t(digest_layer_t *ptr) - return; - } - -- linked_list_for_each_safe(item, &(ptr->layer_list), next) { -+ linked_list_for_each_safe (item, &(ptr->layer_list), next) { - linked_list_del(item); - free(item->elem); - item->elem = NULL; -@@ -276,6 +281,10 @@ static bool init_from_conf(const struct storage_module_init_options *conf) - g_root_dir = tmp_path; - tmp_path = NULL; - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ g_enable_remote_layer = conf->enable_remote_layer; -+#endif -+ - return true; - free_out: - free(g_run_dir); -@@ -613,7 +622,7 @@ static int delete_digest_from_map(map_t *by_digest, const char *digest, const ch - return 0; - } - -- linked_list_for_each_safe(item, &(old_list->layer_list), next) { -+ linked_list_for_each_safe (item, &(old_list->layer_list), next) { - char *t_id = (char *)item->elem; - if (strcmp(t_id, id) == 0) { - linked_list_del(item); -@@ -726,7 +735,7 @@ static int remove_memory_stores(const char *id) - } - } - -- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { -+ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { - layer_t *tl = (layer_t *)item->elem; - if (strcmp(tl->slayer->id, id) != 0) { - continue; -@@ -1119,10 +1128,25 @@ static int new_layer_by_opts(const char *id, const struct layer_opts *opts) - ret = -1; - goto out; - } -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote_layer && !opts->writable) { -+ if (remote_layer_build_ro_dir(id) != 0) { -+ ret = -1; -+ goto out; -+ } -+ } else { -+ if (!build_layer_dir(id)) { -+ ret = -1; -+ goto out; -+ } -+ } -+#else - if (!build_layer_dir(id)) { - ret = -1; - goto out; - } -+#endif - - ret = update_layer_datas(id, opts, l); - if (ret != 0) { -@@ -1304,7 +1328,15 @@ clear_memory: - driver_remove: - if (ret != 0) { - (void)graphdriver_rm_layer(lid); -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote_layer) { -+ (void)remote_layer_remove_ro_dir(lid); -+ } else { -+ (void)layer_store_remove_layer(lid); -+ } -+#else - (void)layer_store_remove_layer(lid); -+#endif - } - free_out: - layer_store_unlock(); -@@ -1379,7 +1411,15 @@ static int do_delete_layer(const char *id) - goto free_out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (l->slayer->writable) { -+ ret = layer_store_remove_layer(l->slayer->id); -+ } else { -+ ret = remote_layer_remove_ro_dir(l->slayer->id); -+ } -+#else - ret = layer_store_remove_layer(l->slayer->id); -+#endif - - free_out: - free(tspath); -@@ -1461,7 +1501,7 @@ int layer_store_list(struct layer_list *resp) - goto unlock; - } - -- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { -+ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { - layer_t *l = (layer_t *)item->elem; - resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); - if (resp->layers[i] == NULL) { -@@ -1504,7 +1544,7 @@ static int layers_by_digest_map(map_t *m, const char *digest, struct layer_list - goto free_out; - } - -- linked_list_for_each_safe(item, &(id_list->layer_list), next) { -+ linked_list_for_each_safe (item, &(id_list->layer_list), next) { - layer_t *l = NULL; - resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); - if (resp->layers[i] == NULL) { -@@ -1748,6 +1788,114 @@ out: - return ret; - } - -+static layer_t *load_one_layer_from_json(const char *id) -+{ -+ int nret = 0; -+ char *mount_point_path = NULL; -+ char tmpdir[PATH_MAX] = { 0 }; -+ char *rpath = NULL; -+ layer_t *l = NULL; -+ bool layer_valid = false; -+ -+ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); -+ if (nret < 0 || nret >= PATH_MAX) { -+ ERROR("Sprintf: %s failed", id); -+ goto free_out; -+ } -+ -+ mount_point_path = mountpoint_json_path(id); -+ if (mount_point_path == NULL) { -+ ERROR("Out of Memory"); -+ goto free_out; -+ } -+ -+ rpath = layer_json_path(id); -+ if (rpath == NULL) { -+ ERROR("%s is invalid layer", id); -+ goto free_out; -+ } -+ -+ l = load_layer(rpath, mount_point_path); -+ if (l == NULL) { -+ ERROR("load layer: %s failed, remove it", id); -+ goto free_out; -+ } -+ -+ if (do_validate_image_layer(tmpdir, l) != 0) { -+ ERROR("%s is invalid image layer", id); -+ goto free_out; -+ } -+ -+ if (do_validate_rootfs_layer(l) != 0) { -+ ERROR("%s is invalid rootfs layer", id); -+ goto free_out; -+ } -+ -+ layer_valid = true; -+ -+free_out: -+ free(rpath); -+ free(mount_point_path); -+ if (!layer_valid) { -+ free_layer_t(l); -+ l = NULL; -+ } -+ // always return true; -+ // if load layer failed, just remove it -+ return l; -+} -+ -+int load_one_layer(const char *id) -+{ -+ int ret = 0; -+ layer_t *tl = NULL; -+ int i = 0; -+ -+ if (!layer_store_lock(true)) { -+ return -1; -+ } -+ -+ tl = load_one_layer_from_json(id); -+ if (tl == NULL) { -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { -+ ERROR("Insert id: %s for layer failed", tl->slayer->id); -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ for (; i < tl->slayer->names_len; i++) { -+ // this should be done by master isulad -+ // if (remove_name(tl->slayer->names[i])) { -+ // should_save = true; -+ // } -+ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { -+ ret = -1; -+ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); -+ goto unlock_out; -+ } -+ } -+ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); -+ if (ret != 0) { -+ ERROR("update layer: %s compress failed", tl->slayer->id); -+ goto unlock_out; -+ } -+ -+ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); -+ if (ret != 0) { -+ ERROR("update layer: %s uncompress failed", tl->slayer->id); -+ goto unlock_out; -+ } -+ -+ ret = 0; -+unlock_out: -+ layer_store_unlock(); -+ return ret; -+} -+ - static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) - { - #define LAYER_NAME_LEN 64 -@@ -1764,6 +1912,14 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d - goto free_out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ // skip RO dir -+ // otherwise, RO dir will be treat as invalid layer dir -+ if (strcmp(sub_dir->d_name, REMOTE_RO_LAYER_DIR) == 0) { -+ goto free_out; -+ } -+#endif -+ - if (!util_dir_exists(tmpdir)) { - // ignore non-dir - DEBUG("%s is not directory", sub_dir->d_name); -@@ -1850,7 +2006,7 @@ static int load_layers_from_json_files() - goto unlock_out; - } - -- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { -+ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { - layer_t *tl = (layer_t *)item->elem; - size_t i = 0; - -@@ -1957,6 +2113,13 @@ int layer_store_init(const struct storage_module_init_options *conf) - goto free_out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote_layer && remote_layer_init(g_root_dir) != 0) { -+ ERROR("Failed to init layer remote"); -+ goto free_out; -+ } -+#endif -+ - if (load_layers_from_json_files() != 0) { - goto free_out; - } -@@ -2129,7 +2292,7 @@ static tar_split *new_tar_split(layer_t *l, const char *tspath) - int ret = 0; - int nret = 0; - tar_split *ts = NULL; -- char path[PATH_MAX] = {0}; -+ char path[PATH_MAX] = { 0 }; - - ts = util_common_calloc_s(sizeof(tar_split)); - if (ts == NULL) { -@@ -2312,3 +2475,20 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char - { - return graphdriver_get_metadata(id); - } -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+int remove_memory_stores_with_lock(const char *id) -+{ -+ int ret = 0; -+ -+ if (!layer_store_lock(true)) { -+ ERROR("Failed to lock layer store when handle: %s", id); -+ return -1; -+ } -+ -+ ret = remove_memory_stores(id); -+ layer_store_unlock(); -+ -+ return ret; -+} -+#endif -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -index 94d4bf04..44bd297e 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -@@ -23,6 +23,10 @@ - - #include "storage.h" - #include "io_wrapper.h" -+#include "map.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "remote_support.h" -+#endif - - struct io_read_wrapper; - struct layer_list; -@@ -79,6 +83,13 @@ int layer_store_check(const char *id); - - container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+remote_support *layer_store_impl_remote_support(); -+bool layer_remote_layer_valid(const char *layer_id); -+int load_one_layer(const char *id); -+int remove_memory_stores_with_lock(const char *id); -+#endif -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt -new file mode 100644 -index 00000000..06c78678 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt -@@ -0,0 +1,12 @@ -+# get current directory sources files -+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_remote_layer_support_srcs) -+ -+set(REMOTE_LAYER_SUPPORT_SRCS -+ ${local_remote_layer_support_srcs} -+ PARENT_SCOPE -+ ) -+ -+set(REMOTE_LAYER_SUPPORT_INCS -+ ${CMAKE_CURRENT_SOURCE_DIR} -+ PARENT_SCOPE -+) -\ No newline at end of file -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -new file mode 100644 -index 00000000..9dc096f7 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -@@ -0,0 +1,122 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-03 -+ * Description: provide image store functions -+ ******************************************************************************/ -+ -+#include "remote_support.h" -+ -+#include "layer_store.h" -+#include "image_store.h" -+#include "isula_libutils/log.h" -+#include "driver_overlay2.h" -+#include "utils.h" -+ -+remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) -+{ -+ remote_support *handlers = layer_store_impl_remote_support(); -+ if (handlers == NULL || handlers->create == NULL) { -+ return NULL; -+ } -+ -+ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -+ if (supporter == NULL) { -+ goto err_out; -+ } -+ -+ supporter->handlers = handlers; -+ supporter->data = handlers->create(remote_home, remote_ro); -+ -+ return supporter; -+ -+err_out: -+ free(handlers); -+ free(supporter); -+ return NULL; -+} -+ -+remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) -+{ -+ remote_support *handlers = image_store_impl_remote_support(); -+ if (handlers == NULL || handlers->create == NULL) { -+ return NULL; -+ } -+ -+ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -+ if (supporter == NULL) { -+ goto err_out; -+ } -+ -+ supporter->handlers = handlers; -+ supporter->data = handlers->create(remote_home, remote_ro); -+ -+ return supporter; -+ -+err_out: -+ free(handlers); -+ free(supporter); -+ return NULL; -+} -+ -+remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) -+{ -+ remote_support *handlers = overlay_driver_impl_remote_support(); -+ if (handlers == NULL || handlers->create == NULL) { -+ return NULL; -+ } -+ -+ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -+ if (supporter == NULL) { -+ goto err_out; -+ } -+ -+ supporter->handlers = handlers; -+ supporter->data = handlers->create(remote_home, remote_ro); -+ -+ return supporter; -+ -+err_out: -+ free(handlers); -+ free(supporter); -+ return NULL; -+ -+} -+ -+void destroy_suppoter(remote_supporter *supporter) -+{ -+ if (supporter->handlers->destroy == NULL) { -+ ERROR("destroy_supporter operation not supported"); -+ return; -+ } -+ -+ supporter->handlers->destroy(supporter->data); -+ free(supporter->handlers); -+ free(supporter); -+} -+ -+int scan_remote_dir(remote_supporter *supporter) -+{ -+ if (supporter->handlers->scan_remote_dir == NULL) { -+ ERROR("scan_remote_dir operation not supported"); -+ return -1; -+ } -+ return supporter->handlers->scan_remote_dir(supporter->data); -+} -+ -+int load_item(remote_supporter *supporter) -+{ -+ if (supporter->handlers->scan_remote_dir == NULL) { -+ ERROR("load_item operation not supported"); -+ return -1; -+ } -+ return supporter->handlers->load_item(supporter->data); -+} -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -new file mode 100644 -index 00000000..d1f7af35 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -@@ -0,0 +1,58 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-03 -+ * Description: provide remote support functions -+ ******************************************************************************/ -+ -+#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H -+#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H -+ -+#include "linked_list.h" -+#define REMOTE_RO_LAYER_DIR "RO" -+#define OVERLAY_RO_DIR "RO" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+typedef struct { -+ void *(*create)(const char *remote_home, const char *remote_ro); -+ void (*destroy)(void *data); -+ // populate the list contains all dirs -+ int (*scan_remote_dir)(void *data); -+ // consume the list contains all dirs -+ int (*load_item)(void *data); -+} remote_support; -+ -+typedef struct { -+ void *data; -+ remote_support *handlers; -+} remote_supporter; -+ -+// RemoteSupport *impl_remote_support(); -+remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); -+ -+remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); -+ -+remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); -+ -+void destroy_suppoter(remote_supporter *supporter); -+ -+int scan_remote_dir(remote_supporter *supporter); -+ -+int load_item(remote_supporter *supporter); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -new file mode 100644 -index 00000000..7df7a221 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -@@ -0,0 +1,347 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-01-12 -+ * Description: provide remote symlink maintain functions -+ ******************************************************************************/ -+#define _GNU_SOURCE -+#include "ro_symlink_maintain.h" -+ -+#include -+#include -+#include -+#include -+#include "map.h" -+#include "path.h" -+#include "linked_list.h" -+#include "layer_store.h" -+#include "layer.h" -+#include "isula_libutils/log.h" -+#include "image_store.h" -+#include "remote_support.h" -+#include "utils.h" -+#include "utils_file.h" -+#include "stdlib.h" -+ -+#define REMOTE_RO_LAYER_DIR "RO" -+ -+// overlay-layers and overlay-layers/RO -+static char *image_home; -+ -+static char *layer_ro_dir; -+static char *layer_home; -+ -+// overlay and overlay/RO -+static char *overlay_ro_dir; -+static char *overlay_home; -+ -+struct supporters { -+ remote_supporter *image_supporter; -+ remote_supporter *layer_supporter; -+ remote_supporter *overlay_supporter; -+}; -+ -+static struct supporters supporters; -+ -+int remote_image_init(const char *root_dir) -+{ -+ if (root_dir == NULL) { -+ goto out; -+ } -+ -+ image_home = util_strdup_s(root_dir); -+ if (image_home == NULL) { -+ ERROR("Failed create path for remote image home"); -+ goto out; -+ } -+ return 0; -+ -+out: -+ remote_maintain_cleanup(); -+ return -1; -+} -+ -+int remote_layer_init(const char *root_dir) -+{ -+ if (root_dir == NULL) { -+ goto out; -+ } -+ -+ layer_home = util_strdup_s(root_dir); -+ layer_ro_dir = util_path_join(root_dir, REMOTE_RO_LAYER_DIR); -+ if (layer_ro_dir == NULL) { -+ ERROR("Failed join path when init remote layer maintainer"); -+ goto out; -+ } -+ if (!util_file_exists(layer_ro_dir) && util_mkdir_p(layer_ro_dir, 0700) != 0) { -+ ERROR("Failed to create RO dir under overlay"); -+ goto out; -+ } -+ -+ return 0; -+ -+out: -+ remote_maintain_cleanup(); -+ return -1; -+} -+ -+int remote_overlay_init(const char *driver_home) -+{ -+ if (driver_home == NULL) { -+ goto out; -+ } -+ -+ overlay_home = util_strdup_s(driver_home); -+ overlay_ro_dir = util_path_join(driver_home, REMOTE_RO_LAYER_DIR); -+ if (overlay_ro_dir == NULL) { -+ ERROR("Failed to join path when init remote maintainer"); -+ goto out; -+ } -+ // build RO dir if not exist -+ if (!util_file_exists(overlay_ro_dir) && util_mkdir_p(overlay_ro_dir, 0700) != 0) { -+ ERROR("Failed to create RO dir under overlay"); -+ goto out; -+ } -+ -+ return 0; -+ -+out: -+ remote_maintain_cleanup(); -+ return -1; -+} -+ -+void remote_maintain_cleanup(void) -+{ -+ free(image_home); -+ image_home = NULL; -+ -+ free(layer_home); -+ layer_home = NULL; -+ free(layer_ro_dir); -+ layer_ro_dir = NULL; -+ free(overlay_home); -+ -+ overlay_home = NULL; -+ free(overlay_ro_dir); -+ overlay_ro_dir = NULL; -+} -+ -+// to maintain the symbol links, add new symbol link and delete invalid symbol link -+// arg is const char *driver_home -+// scanning driver->home/RO/ directory, build symlink in driver->home -+static void *remote_refresh_ro_symbol_link(void *arg) -+{ -+ struct supporters *supporters = (struct supporters *)arg; -+ prctl(PR_SET_NAME, "RoLayerRefresh"); -+ -+ while (true) { -+ util_usleep_nointerupt(5 * 1000 * 1000); -+ DEBUG("remote refresh start\n"); -+ scan_remote_dir(supporters->overlay_supporter); -+ load_item(supporters->overlay_supporter); -+ scan_remote_dir(supporters->layer_supporter); -+ load_item(supporters->layer_supporter); -+ scan_remote_dir(supporters->image_supporter); -+ load_item(supporters->image_supporter); -+ DEBUG("remote refresh end\n"); -+ } -+ return NULL; -+} -+ -+int start_refresh_thread(void) -+{ -+ int res = 0; -+ pthread_t a_thread; -+ -+ supporters.image_supporter = create_image_supporter(image_home, NULL); -+ if (supporters.image_supporter == NULL) { -+ goto free_out; -+ } -+ -+ supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); -+ if (supporters.layer_supporter == NULL) { -+ goto free_out; -+ } -+ -+ supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); -+ if (supporters.overlay_supporter == NULL) { -+ goto free_out; -+ } -+ -+ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); -+ if (res != 0) { -+ CRIT("Thread creation failed"); -+ return -1; -+ } -+ -+ if (pthread_detach(a_thread) != 0) { -+ SYSERROR("Failed to detach 0x%lx", a_thread); -+ return -1; -+ } -+ -+ return 0; -+ -+free_out: -+ destroy_suppoter(supporters.image_supporter); -+ destroy_suppoter(supporters.layer_supporter); -+ destroy_suppoter(supporters.overlay_supporter); -+ -+ return -1; -+} -+ -+static int do_build_ro_dir(const char *home, const char *id) -+{ -+ char *ro_symlink = NULL; -+ char *ro_layer_dir = NULL; -+ int nret = 0; -+ // bool ret = true; -+ int ret = 0; -+ -+ nret = asprintf(&ro_symlink, "%s/%s", home, id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Failed create ro layer dir sym link path"); -+ return -1; -+ } -+ -+ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Failed to create ro layer dir path"); -+ return -1; -+ } -+ -+ if (util_mkdir_p(ro_layer_dir, IMAGE_STORE_PATH_MODE) != 0) { -+ ret = -1; -+ ERROR("Failed to create layer direcotry %s", ro_layer_dir); -+ goto out; -+ } -+ -+ if (symlink(ro_layer_dir, ro_symlink) != 0) { -+ ret = -1; -+ SYSERROR("Failed to create symlink to layer dir %s", ro_layer_dir); -+ goto err_out; -+ } -+ -+ goto out; -+ -+err_out: -+ if (util_recursive_rmdir(ro_layer_dir, 0)) { -+ ERROR("Failed to delete layer path: %s", ro_layer_dir); -+ } -+ -+out: -+ free(ro_layer_dir); -+ free(ro_symlink); -+ return ret; -+} -+ -+int remote_overlay_build_ro_dir(const char *id) -+{ -+ return do_build_ro_dir(overlay_home, id); -+} -+ -+int remote_layer_build_ro_dir(const char *id) -+{ -+ return do_build_ro_dir(layer_home, id); -+} -+ -+int do_remove_ro_dir(const char *home, const char *id) -+{ -+ char *ro_layer_dir = NULL; -+ char *ro_symlink = NULL; -+ char clean_path[PATH_MAX] = { 0 }; -+ int ret = 0; -+ int nret = 0; -+ -+ if (id == NULL) { -+ return 0; -+ } -+ -+ nret = asprintf(&ro_symlink, "%s/%s", home, id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Create layer sym link path failed"); -+ return -1; -+ } -+ -+ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { -+ ERROR("Failed to clean path: %s", ro_symlink); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_path_remove(clean_path) != 0) { -+ SYSERROR("Failed to remove link path %s", clean_path); -+ } -+ -+ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Create layer json path failed"); -+ ret = -1; -+ goto out; -+ } -+ -+ ret = util_recursive_rmdir(ro_layer_dir, 0); -+ -+out: -+ free(ro_layer_dir); -+ free(ro_symlink); -+ return ret; -+} -+ -+int remote_layer_remove_ro_dir(const char *id) -+{ -+ return do_remove_ro_dir(layer_home, id); -+} -+ -+int remote_overlay_remove_ro_dir(const char *id) -+{ -+ return do_remove_ro_dir(overlay_home, id); -+} -+ -+static char **map_diff(map_t *map_a, map_t *map_b) -+{ -+ char **array = NULL; -+ map_itor *itor = map_itor_new(map_a); -+ bool *found = NULL; -+ -+ // iter new_map, every item not in old, append them to new_layers -+ for (; map_itor_valid(itor); map_itor_next(itor)) { -+ char *id = map_itor_key(itor); -+ found = map_search(map_b, id); -+ if (found == NULL) { -+ util_array_append(&array, util_strdup_s(id)); -+ } -+ } -+ -+ map_itor_free(itor); -+ -+ return array; -+} -+ -+char **deleted_layers(map_t *old, map_t *new) -+{ -+ return map_diff(old, new); -+} -+ -+char **added_layers(map_t *old, map_t *new) -+{ -+ return map_diff(new, old); -+} -+ -+int empty_map(map_t *mp) -+{ -+ if (mp == NULL) { -+ return -1; -+ } -+ -+ map_clear(mp); -+ mp->store->root = mp->store->nil; -+ return 0; -+} -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h -new file mode 100644 -index 00000000..25712d40 ---- /dev/null -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h -@@ -0,0 +1,52 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-01-12 -+ * Description: provide remote symlink maintain functions -+ ******************************************************************************/ -+#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H -+#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H -+ -+#include "map.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+int remote_image_init(const char *root_dir); -+ -+int remote_layer_init(const char *root_dir); -+ -+int remote_overlay_init(const char *driver_home); -+ -+void remote_maintain_cleanup(void); -+ -+int start_refresh_thread(void); -+ -+int remote_layer_build_ro_dir(const char *id); -+ -+int remote_overlay_build_ro_dir(const char *id); -+ -+int remote_layer_remove_ro_dir(const char *id); -+ -+int remote_overlay_remove_ro_dir(const char *id); -+ -+char **deleted_layers(map_t *old, map_t *new); -+ -+char **added_layers(map_t *old, map_t *new); -+ -+int empty_map(map_t *mp); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c -index 2f4bdf5f..31812a22 100644 ---- a/src/daemon/modules/image/oci/storage/storage.c -+++ b/src/daemon/modules/image/oci/storage/storage.c -@@ -42,6 +42,9 @@ - #include "utils_string.h" - #include "utils_verify.h" - #include "sha256.h" -+#ifdef ENABLE_REMOTE_LAYER_STORE -+#include "ro_symlink_maintain.h" -+#endif - - static pthread_rwlock_t g_storage_rwlock; - static char *g_storage_run_root; -@@ -1870,6 +1873,12 @@ int storage_module_init(struct storage_module_init_options *opts) - goto out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (opts->enable_remote_layer && start_refresh_thread() != 0) { -+ ERROR("Failed to start remote refresh thread"); -+ } -+#endif -+ - if (restore_images_size() != 0) { - ERROR("Failed to recal image size"); - ret = -1; -@@ -1906,4 +1915,4 @@ out: - char *storage_rootfs_get_dir() - { - return rootfs_store_get_data_dir(); --} -\ No newline at end of file -+} -diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h -index 3ec47959..7404ee54 100644 ---- a/src/daemon/modules/image/oci/storage/storage.h -+++ b/src/daemon/modules/image/oci/storage/storage.h -@@ -70,6 +70,9 @@ struct storage_module_init_options { - char **driver_opts; - size_t driver_opts_len; - bool integration_check; -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ bool enable_remote_layer; -+#endif - }; - - struct storage_img_create_options { --- -2.25.1 - diff --git a/0049-add-ci-for-remote-ro.patch b/0049-add-ci-for-remote-ro.patch deleted file mode 100644 index 9e6ece1..0000000 --- a/0049-add-ci-for-remote-ro.patch +++ /dev/null @@ -1,119 +0,0 @@ -From 1b1900c183886e13edd86c2e2cb6e5c42cfebd3d Mon Sep 17 00:00:00 2001 -From: Neil -Date: Sun, 5 Mar 2023 12:23:29 +0000 -Subject: [PATCH 49/56] add ci for remote ro - -Signed-off-by: Neil ---- - CI/make-and-install.sh | 4 +- - .../container_cases/test_data/daemon.json | 1 + - CI/test_cases/image_cases/ro_separate.sh | 69 +++++++++++++++++++ - 3 files changed, 72 insertions(+), 2 deletions(-) - create mode 100644 CI/test_cases/image_cases/ro_separate.sh - -diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh -index 81022d75..e714d206 100755 ---- a/CI/make-and-install.sh -+++ b/CI/make-and-install.sh -@@ -106,9 +106,9 @@ rm -rf build - mkdir build - cd build - if [[ ${enable_gcov} -ne 0 ]]; then -- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON .. -+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. - else -- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON .. -+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. - fi - make -j $(nproc) - make install -diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json -index aa88c9da..2664c6b2 100644 ---- a/CI/test_cases/container_cases/test_data/daemon.json -+++ b/CI/test_cases/container_cases/test_data/daemon.json -@@ -19,6 +19,7 @@ - "hook-spec": "/etc/default/isulad/hooks/default.json", - "start-timeout": "2m", - "storage-driver": "overlay2", -+ "storage-enable-remote-layer": false, - "storage-opts": [ - "overlay2.override_kernel_check=true" - ], -diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh -new file mode 100644 -index 00000000..47e04abb ---- /dev/null -+++ b/CI/test_cases/image_cases/ro_separate.sh -@@ -0,0 +1,69 @@ -+#!/bin/bash -+# -+# attributes: isulad basic image -+# concurrent: NA -+# spend time: 22 -+ -+####################################################################### -+##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+# - iSulad licensed under the Mulan PSL v2. -+# - You can use this software according to the terms and conditions of the Mulan PSL v2. -+# - You may obtain a copy of Mulan PSL v2 at: -+# - http://license.coscl.org.cn/MulanPSL2 -+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# - PURPOSE. -+# - See the Mulan PSL v2 for more details. -+##- @Description:CI -+##- @Author: wangrunze -+##- @Create: 2023-03-03 -+####################################################################### -+ -+declare -r curr_path=$(dirname $(readlink -f "$0")) -+source ../helpers.sh -+single_image="${curr_path}/busybox.tar" -+ -+function test_separate_ro() -+{ -+ local ret=0 -+ local test="isula separate ro test => (${FUNCNAME[@]})" -+ -+ msg_info "${test} starting..." -+ -+ sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json -+ start_isulad_with_valgrind -+ wait_isulad_running -+ -+ isula rmi busybox -+ -+ isula pull busybox -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++)) -+ -+ isula run -tid --name test_separate busybox /bin/sh -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) -+ -+ isula stop test_separate -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) -+ -+ isula rmi busybox -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) -+ -+ isula load -i $single_image -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++)) -+ -+ check_valgrind_log -+ [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++)) -+ -+ sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json -+ start_isulad_with_valgrind -+ wait_isulad_running -+ -+ msg_info "${test} finished with return ${ret}..." -+ return ${ret} -+} -+ -+declare -i ans=0 -+ -+test_separate_ro || ((ans++)) -+ -+show_result ${ans} "${curr_path}/${0}" --- -2.25.1 - diff --git a/0050-fix-compile-error-when-not-enable-remote-ro.patch b/0050-fix-compile-error-when-not-enable-remote-ro.patch deleted file mode 100644 index baba9ea..0000000 --- a/0050-fix-compile-error-when-not-enable-remote-ro.patch +++ /dev/null @@ -1,62 +0,0 @@ -From e4f309f61b169529c263b7a83a0eda16ebe132f5 Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Sun, 5 Mar 2023 18:55:40 -0800 -Subject: [PATCH 50/56] fix compile error when not enable remote ro - -Signed-off-by: Neil.wrz ---- - .../modules/image/oci/storage/image_store/CMakeLists.txt | 3 +++ - .../modules/image/oci/storage/layer_store/CMakeLists.txt | 4 ++++ - .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt | 3 +++ - 3 files changed, 10 insertions(+) - -diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -index ecf21caa..7d4fb77c 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -@@ -1,5 +1,8 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) -+IF (NOT ENABLE_REMOTE_LAYER_STORE) -+list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") -+ENDIF() - - set(IMAGE_STORE_SRCS - ${local_image_store_srcs} -diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -index f964f709..e04b4ad7 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -@@ -1,5 +1,8 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) -+IF (NOT ENABLE_REMOTE_LAYER_STORE) -+list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") -+ENDIF() - add_subdirectory(graphdriver) - - set(LAYER_STORE_SRCS -@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS - ${GRAPHDRIVER_SRCS} - PARENT_SCOPE - ) -+ - set(LAYER_STORE_INCS - ${CMAKE_CURRENT_SOURCE_DIR} - ${GRAPHDRIVER_INCS} -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -index ceed16b7..dd4e82aa 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -@@ -1,5 +1,8 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) -+IF (NOT ENABLE_REMOTE_LAYER_STORE) -+list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") -+ENDIF() - - set(OVERLAY2_SRCS - ${local_overlay2_srcs} --- -2.25.1 - diff --git a/0051-CI-not-enable-remote-ro-for-ut.patch b/0051-CI-not-enable-remote-ro-for-ut.patch deleted file mode 100644 index 46a3eb6..0000000 --- a/0051-CI-not-enable-remote-ro-for-ut.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 1d5aa99d97df3f4ddec4ae436cb0ccbbba3e863a Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Mon, 6 Mar 2023 18:59:43 -0800 -Subject: [PATCH 51/56] CI not enable remote ro for ut - -Signed-off-by: Neil.wrz ---- - CI/make-and-install.sh | 2 +- - CI/test_cases/image_cases/ro_separate.sh | 3 +++ - 2 files changed, 4 insertions(+), 1 deletion(-) - -diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh -index e714d206..7c194826 100755 ---- a/CI/make-and-install.sh -+++ b/CI/make-and-install.sh -@@ -106,7 +106,7 @@ rm -rf build - mkdir build - cd build - if [[ ${enable_gcov} -ne 0 ]]; then -- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. -+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. - else - cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. - fi -diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh -index 47e04abb..df45e120 100644 ---- a/CI/test_cases/image_cases/ro_separate.sh -+++ b/CI/test_cases/image_cases/ro_separate.sh -@@ -45,6 +45,9 @@ function test_separate_ro() - isula stop test_separate - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) - -+ isula rm test_separate -+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++)) -+ - isula rmi busybox - [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) - --- -2.25.1 - diff --git a/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch b/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch deleted file mode 100644 index adab3d1..0000000 --- a/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch +++ /dev/null @@ -1,99 +0,0 @@ -From 569e2d07cc153f2918868ba58bc7da9a626e4db0 Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Tue, 7 Mar 2023 23:59:56 -0800 -Subject: [PATCH 52/56] bugfix remote ro try add or remove image/layer twice - -Signed-off-by: Neil.wrz ---- - .../image/oci/storage/image_store/image_store.c | 14 ++++++++++++++ - .../oci/storage/layer_store/layer_remote_impl.c | 2 +- - .../image/oci/storage/layer_store/layer_store.c | 11 +++++++++++ - 3 files changed, 26 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c -index c6836e46..a44bf362 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.c -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c -@@ -3671,6 +3671,11 @@ int append_image_by_directory_with_lock(const char *id) - return -1; - } - -+ if (map_search(g_image_store->byid, (void *)id) != NULL ) { -+ DEBUG("remote image already exist, not added: %s", id); -+ goto out; -+ } -+ - nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); - if (nret < 0 || (size_t)nret >= sizeof(image_path)) { - ERROR("Failed to get image path"); -@@ -3678,6 +3683,8 @@ int append_image_by_directory_with_lock(const char *id) - } - - ret = append_image_by_directory(image_path); -+ -+out: - image_store_unlock(); - - return ret; -@@ -3692,7 +3699,14 @@ int remove_image_from_memory_with_lock(const char *id) - return -1; - } - -+ if (map_search(g_image_store->byid, (void *)id) == NULL) { -+ DEBUG("remote image already remvoed, don't delete twice: %s", id); -+ goto out; -+ } -+ - ret = remove_image_from_memory(id); -+ -+out: - image_store_unlock(); - - return ret; -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -index d03fc20b..d676458c 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -@@ -175,7 +175,7 @@ static int remote_support_add(void *data) - } - - if (add_one_remote_layer(data, array_added[i]) != 0) { -- ERROR("Failed to add remote overlay layer: %s", array_added[i]); -+ ERROR("Failed to add remote layer: %s", array_added[i]); - ret = -1; - } - } -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -index 8b8f5f1e..29ead711 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -@@ -1855,6 +1855,11 @@ int load_one_layer(const char *id) - return -1; - } - -+ if (map_search(g_metadata.by_id, (void *)id) != NULL) { -+ DEBUG("remote layer already exist, not added: %s", id); -+ goto unlock_out; -+ } -+ - tl = load_one_layer_from_json(id); - if (tl == NULL) { - ret = -1; -@@ -2485,8 +2490,14 @@ int remove_memory_stores_with_lock(const char *id) - ERROR("Failed to lock layer store when handle: %s", id); - return -1; - } -+ if (map_search(g_metadata.by_id, (void *)id) == NULL) { -+ DEBUG("remote layer already removed, don't delete: %s", id); -+ goto unlock_out; -+ } - - ret = remove_memory_stores(id); -+ -+unlock_out: - layer_store_unlock(); - - return ret; --- -2.25.1 - diff --git a/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch b/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch deleted file mode 100644 index 94801b9..0000000 --- a/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch +++ /dev/null @@ -1,34 +0,0 @@ -From bbcc4c58d67aeceee55fcc1126deedafe0e43c20 Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Tue, 14 Mar 2023 20:32:23 -0700 -Subject: [PATCH 53/56] bugfix can't delete layers under dir overlay-layers - -Signed-off-by: Neil.wrz ---- - .../modules/image/oci/storage/layer_store/layer_store.c | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -index 29ead711..680b35a2 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -@@ -1412,10 +1412,14 @@ static int do_delete_layer(const char *id) - } - - #ifdef ENABLE_REMOTE_LAYER_STORE -- if (l->slayer->writable) { -+ if (!g_enable_remote_layer) { - ret = layer_store_remove_layer(l->slayer->id); - } else { -- ret = remote_layer_remove_ro_dir(l->slayer->id); -+ if (l->slayer->writable) { -+ ret = layer_store_remove_layer(l->slayer->id); -+ } else { -+ ret = remote_layer_remove_ro_dir(l->slayer->id); -+ } - } - #else - ret = layer_store_remove_layer(l->slayer->id); --- -2.25.1 - diff --git a/0054-refactor-remote-ro-code.patch b/0054-refactor-remote-ro-code.patch deleted file mode 100644 index 86196e8..0000000 --- a/0054-refactor-remote-ro-code.patch +++ /dev/null @@ -1,2089 +0,0 @@ -From 2bfce1ac385eabd9614e4d8d6f50c02efb9653fc Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Mon, 6 Mar 2023 17:43:17 -0800 -Subject: [PATCH 54/56] refactor remote ro code - -Signed-off-by: Neil.wrz ---- - CI/make-and-install.sh | 2 +- - .../oci/storage/image_store/CMakeLists.txt | 3 - - .../oci/storage/image_store/image_store.c | 37 +-- - .../oci/storage/image_store/image_store.h | 13 +- - .../oci/storage/layer_store/CMakeLists.txt | 3 - - .../graphdriver/overlay2/CMakeLists.txt | 3 - - .../graphdriver/overlay2/driver_overlay2.h | 8 - - .../oci/storage/layer_store/layer_store.c | 230 +++++++++--------- - .../oci/storage/layer_store/layer_store.h | 9 +- - .../image_remote_impl.c | 61 +++-- - .../layer_remote_impl.c | 71 +++--- - .../overlay_remote_impl.c | 103 +++++--- - .../remote_layer_support/remote_support.c | 141 ++++++----- - .../remote_layer_support/remote_support.h | 62 +++-- - .../ro_symlink_maintain.c | 124 +--------- - .../ro_symlink_maintain.h | 19 +- - .../modules/image/oci/storage/storage.c | 4 +- - src/utils/cutils/map/rb_tree.c | 1 + - test/image/oci/registry/CMakeLists.txt | 2 + - test/image/oci/storage/CMakeLists.txt | 3 + - test/image/oci/storage/images/CMakeLists.txt | 2 + - test/image/oci/storage/layers/CMakeLists.txt | 4 + - .../remote_layer_support/CMakeLists.txt | 44 ++++ - .../remote_layer_support/remote_layer_ut.cc | 93 +++++++ - test/image/oci/storage/rootfs/CMakeLists.txt | 2 + - test/mocks/remote_store_mock.cc | 68 ++++++ - test/mocks/remote_store_mock.h | 40 +++ - 27 files changed, 683 insertions(+), 469 deletions(-) - rename src/daemon/modules/image/oci/storage/{image_store => remote_layer_support}/image_remote_impl.c (71%) - rename src/daemon/modules/image/oci/storage/{layer_store => remote_layer_support}/layer_remote_impl.c (76%) - rename src/daemon/modules/image/oci/storage/{layer_store/graphdriver/overlay2 => remote_layer_support}/overlay_remote_impl.c (72%) - create mode 100644 test/image/oci/storage/remote_layer_support/CMakeLists.txt - create mode 100644 test/image/oci/storage/remote_layer_support/remote_layer_ut.cc - create mode 100644 test/mocks/remote_store_mock.cc - create mode 100644 test/mocks/remote_store_mock.h - -diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh -index 7c194826..e714d206 100755 ---- a/CI/make-and-install.sh -+++ b/CI/make-and-install.sh -@@ -106,7 +106,7 @@ rm -rf build - mkdir build - cd build - if [[ ${enable_gcov} -ne 0 ]]; then -- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. -+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. - else - cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. - fi -diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -index 7d4fb77c..ecf21caa 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt -@@ -1,8 +1,5 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) --IF (NOT ENABLE_REMOTE_LAYER_STORE) --list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") --ENDIF() - - set(IMAGE_STORE_SRCS - ${local_image_store_srcs} -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c -index a44bf362..7837f9db 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.c -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c -@@ -133,7 +133,7 @@ static void free_image_store(image_store_t *store) - (void)map_free(store->bydigest); - store->bydigest = NULL; - -- linked_list_for_each_safe(item, &(store->images_list), next) { -+ linked_list_for_each_safe (item, &(store->images_list), next) { - linked_list_del(item); - image_ref_dec((image_t *)item->elem); - free(item); -@@ -165,7 +165,7 @@ static void image_store_digest_field_kvfree(void *key, void *value) - - free(key); - if (val != NULL) { -- linked_list_for_each_safe(item, &(val->images_list), next) { -+ linked_list_for_each_safe (item, &(val->images_list), next) { - linked_list_del(item); - free(item); - item = NULL; -@@ -501,7 +501,7 @@ static void digest_image_slice_without_value(digest_image_t *digest_filter_image - return; - } - -- linked_list_for_each_safe(item, &(digest_filter_images->images_list), next) { -+ linked_list_for_each_safe (item, &(digest_filter_images->images_list), next) { - tmp = (image_t *)item->elem; - if (strcmp(tmp->simage->id, img->simage->id) == 0) { - linked_list_del(item); -@@ -582,7 +582,7 @@ static int remove_image_from_memory(const char *id) - goto out; - } - -- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { -+ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { - image_t *tmp = (image_t *)item->elem; - if (strcmp(tmp->simage->id, id) != 0) { - continue; -@@ -681,7 +681,7 @@ static void free_digest_image(digest_image_t *ptr) - return; - } - -- linked_list_for_each_safe(item, &(ptr->images_list), next) { -+ linked_list_for_each_safe (item, &(ptr->images_list), next) { - linked_list_del(item); - free(item); - item = NULL; -@@ -2679,7 +2679,7 @@ int image_store_get_all_images(imagetool_images_list *images_list) - goto unlock; - } - -- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { -+ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { - imagetool_image_summary *imginfo = NULL; - image_t *img = (image_t *)item->elem; - imginfo = get_image_summary(img); -@@ -3102,7 +3102,7 @@ out: - return ret; - } - --int validate_manifest_schema_version_1(const char *path, bool *valid) -+int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) - { - int ret = 0; - int nret; -@@ -3509,7 +3509,7 @@ static int get_images_from_json() - continue; - } - -- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { -+ if (image_store_validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { - ERROR("Failed to validate manifest schema version 1 format"); - continue; - } -@@ -3546,7 +3546,7 @@ static void image_store_check_all_images() - return; - } - -- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { -+ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { - image_t *img = (image_t *)item->elem; - if (img->spec == NULL) { - ERROR("Failed to check spec info of image: %s, try to delete", img->simage->id); -@@ -3660,18 +3660,23 @@ out: - } - - #ifdef ENABLE_REMOTE_LAYER_STORE --int append_image_by_directory_with_lock(const char *id) -+int remote_append_image_by_directory_with_lock(const char *id) - { - int ret = 0; - int nret = 0; - char image_path[PATH_MAX] = { 0x00 }; - -+ if (id == NULL) { -+ ERROR("can't add NULL remote image"); -+ return -1; -+ } -+ - if (!image_store_lock(EXCLUSIVE)) { - ERROR("Failed to lock remote image store when handle: %s", id); - return -1; - } - -- if (map_search(g_image_store->byid, (void *)id) != NULL ) { -+ if (map_search(g_image_store->byid, (void *)id) != NULL) { - DEBUG("remote image already exist, not added: %s", id); - goto out; - } -@@ -3690,10 +3695,15 @@ out: - return ret; - } - --int remove_image_from_memory_with_lock(const char *id) -+int remote_remove_image_from_memory_with_lock(const char *id) - { - int ret = 0; - -+ if (id == NULL) { -+ ERROR("can't remove NULL remote image"); -+ return -1; -+ } -+ - if (!image_store_lock(EXCLUSIVE)) { - ERROR("Failed to lock remote image store when handle: %s", id); - return -1; -@@ -3712,9 +3722,8 @@ out: - return ret; - } - --char *get_top_layer_from_json(const char *img_id) -+char *remote_image_get_top_layer_from_json(const char *img_id) - { -- - char *ret = NULL; - int nret = 0; - char image_path[PATH_MAX] = { 0x00 }; -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h -index c3cb50e3..5164cc73 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_store.h -+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h -@@ -28,9 +28,6 @@ - #include "isula_libutils/imagetool_image.h" - #include "isula_libutils/imagetool_images_list.h" - #include "isula_libutils/imagetool_image_summary.h" --#ifdef ENABLE_REMOTE_LAYER_STORE --#include "remote_support.h" --#endif - - struct storage_module_init_options; - -@@ -112,11 +109,11 @@ void image_store_free(); - imagetool_image_summary *image_store_get_image_summary(const char *id); - - #ifdef ENABLE_REMOTE_LAYER_STORE --remote_support *image_store_impl_remote_support(); --int validate_manifest_schema_version_1(const char *path, bool *valid); --int append_image_by_directory_with_lock(const char *image_dir); --int remove_image_from_memory_with_lock(const char *id); --char *get_top_layer_from_json(const char *img_id); /* return top layer id */ -+int image_store_validate_manifest_schema_version_1(const char *path, bool *valid); -+int remote_append_image_by_directory_with_lock(const char *image_dir); -+int remote_remove_image_from_memory_with_lock(const char *id); -+// return top layer id -+char *remote_image_get_top_layer_from_json(const char *img_id); - #endif - - #ifdef __cplusplus -diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -index e04b4ad7..c218a7c0 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt -@@ -1,8 +1,5 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) --IF (NOT ENABLE_REMOTE_LAYER_STORE) --list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") --ENDIF() - add_subdirectory(graphdriver) - - set(LAYER_STORE_SRCS -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -index dd4e82aa..ceed16b7 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt -@@ -1,8 +1,5 @@ - # get current directory sources files - aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) --IF (NOT ENABLE_REMOTE_LAYER_STORE) --list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") --ENDIF() - - set(OVERLAY2_SRCS - ${local_overlay2_srcs} -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -index 5c1d93fb..e14271b1 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h -@@ -22,9 +22,6 @@ - #include - - #include "driver.h" --#ifdef ENABLE_REMOTE_LAYER_STORE --#include "remote_support.h" --#endif - - struct driver_create_opts; - struct driver_mount_opts; -@@ -71,11 +68,6 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap - - int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); - --#ifdef ENABLE_REMOTE_LAYER_STORE --remote_support *overlay_driver_impl_remote_support(void); --bool overlay_remote_layer_valid(const char *layer_id); --#endif -- - #ifdef __cplusplus - } - #endif -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -index 680b35a2..12fca5ff 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c -@@ -1792,119 +1792,6 @@ out: - return ret; - } - --static layer_t *load_one_layer_from_json(const char *id) --{ -- int nret = 0; -- char *mount_point_path = NULL; -- char tmpdir[PATH_MAX] = { 0 }; -- char *rpath = NULL; -- layer_t *l = NULL; -- bool layer_valid = false; -- -- nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); -- if (nret < 0 || nret >= PATH_MAX) { -- ERROR("Sprintf: %s failed", id); -- goto free_out; -- } -- -- mount_point_path = mountpoint_json_path(id); -- if (mount_point_path == NULL) { -- ERROR("Out of Memory"); -- goto free_out; -- } -- -- rpath = layer_json_path(id); -- if (rpath == NULL) { -- ERROR("%s is invalid layer", id); -- goto free_out; -- } -- -- l = load_layer(rpath, mount_point_path); -- if (l == NULL) { -- ERROR("load layer: %s failed, remove it", id); -- goto free_out; -- } -- -- if (do_validate_image_layer(tmpdir, l) != 0) { -- ERROR("%s is invalid image layer", id); -- goto free_out; -- } -- -- if (do_validate_rootfs_layer(l) != 0) { -- ERROR("%s is invalid rootfs layer", id); -- goto free_out; -- } -- -- layer_valid = true; -- --free_out: -- free(rpath); -- free(mount_point_path); -- if (!layer_valid) { -- free_layer_t(l); -- l = NULL; -- } -- // always return true; -- // if load layer failed, just remove it -- return l; --} -- --int load_one_layer(const char *id) --{ -- int ret = 0; -- layer_t *tl = NULL; -- int i = 0; -- -- if (!layer_store_lock(true)) { -- return -1; -- } -- -- if (map_search(g_metadata.by_id, (void *)id) != NULL) { -- DEBUG("remote layer already exist, not added: %s", id); -- goto unlock_out; -- } -- -- tl = load_one_layer_from_json(id); -- if (tl == NULL) { -- ret = -1; -- goto unlock_out; -- } -- -- if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { -- ERROR("Insert id: %s for layer failed", tl->slayer->id); -- ret = -1; -- goto unlock_out; -- } -- -- for (; i < tl->slayer->names_len; i++) { -- // this should be done by master isulad -- // if (remove_name(tl->slayer->names[i])) { -- // should_save = true; -- // } -- if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { -- ret = -1; -- ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); -- goto unlock_out; -- } -- } -- ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); -- if (ret != 0) { -- ERROR("update layer: %s compress failed", tl->slayer->id); -- goto unlock_out; -- } -- -- ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); -- if (ret != 0) { -- ERROR("update layer: %s uncompress failed", tl->slayer->id); -- goto unlock_out; -- } -- -- ret = 0; --unlock_out: -- layer_store_unlock(); -- return ret; --} -- - static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) - { - #define LAYER_NAME_LEN 64 -@@ -2486,7 +2373,7 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char - } - - #ifdef ENABLE_REMOTE_LAYER_STORE --int remove_memory_stores_with_lock(const char *id) -+int remote_layer_remove_memory_stores_with_lock(const char *id) - { - int ret = 0; - -@@ -2499,6 +2386,11 @@ int remove_memory_stores_with_lock(const char *id) - goto unlock_out; - } - -+ if (map_search(g_metadata.by_id, (void *)id) == NULL) { -+ DEBUG("remote layer already removed, don't delete: %s", id); -+ goto unlock_out; -+ } -+ - ret = remove_memory_stores(id); - - unlock_out: -@@ -2506,4 +2398,114 @@ unlock_out: - - return ret; - } -+ -+static layer_t *load_one_layer_from_json(const char *id) -+{ -+ int nret = 0; -+ char *mount_point_path = NULL; -+ char tmpdir[PATH_MAX] = { 0 }; -+ char *rpath = NULL; -+ layer_t *l = NULL; -+ bool layer_valid = false; -+ -+ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); -+ if (nret < 0 || nret >= PATH_MAX) { -+ ERROR("Sprintf: %s failed", id); -+ goto free_out; -+ } -+ -+ mount_point_path = mountpoint_json_path(id); -+ if (mount_point_path == NULL) { -+ ERROR("Out of Memory"); -+ goto free_out; -+ } -+ -+ rpath = layer_json_path(id); -+ if (rpath == NULL) { -+ ERROR("%s is invalid layer", id); -+ goto free_out; -+ } -+ -+ l = load_layer(rpath, mount_point_path); -+ if (l == NULL) { -+ ERROR("load layer: %s failed, remove it", id); -+ goto free_out; -+ } -+ -+ if (do_validate_image_layer(tmpdir, l) != 0) { -+ ERROR("%s is invalid image layer", id); -+ goto free_out; -+ } -+ -+ if (do_validate_rootfs_layer(l) != 0) { -+ ERROR("%s is invalid rootfs layer", id); -+ goto free_out; -+ } -+ -+ layer_valid = true; -+ -+free_out: -+ free(rpath); -+ free(mount_point_path); -+ if (!layer_valid) { -+ free_layer_t(l); -+ l = NULL; -+ } -+ // always return true; -+ // if load layer failed, just remove it -+ return l; -+} -+ -+int remote_load_one_layer(const char *id) -+{ -+ int ret = 0; -+ layer_t *tl = NULL; -+ int i = 0; -+ -+ if (!layer_store_lock(true)) { -+ return -1; -+ } -+ -+ if (map_search(g_metadata.by_id, (void *)id) != NULL) { -+ DEBUG("remote layer already exist, not added: %s", id); -+ goto unlock_out; -+ } -+ -+ tl = load_one_layer_from_json(id); -+ if (tl == NULL) { -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { -+ ERROR("Insert id: %s for layer failed", tl->slayer->id); -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ for (; i < tl->slayer->names_len; i++) { -+ // this should be done by master isulad -+ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { -+ ret = -1; -+ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); -+ goto unlock_out; -+ } -+ } -+ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); -+ if (ret != 0) { -+ ERROR("update layer: %s compress failed", tl->slayer->id); -+ goto unlock_out; -+ } -+ -+ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); -+ if (ret != 0) { -+ ERROR("update layer: %s uncompress failed", tl->slayer->id); -+ goto unlock_out; -+ } -+ -+ ret = 0; -+unlock_out: -+ layer_store_unlock(); -+ return ret; -+} - #endif -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -index 44bd297e..4677e5ee 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -@@ -24,9 +24,6 @@ - #include "storage.h" - #include "io_wrapper.h" - #include "map.h" --#ifdef ENABLE_REMOTE_LAYER_STORE --#include "remote_support.h" --#endif - - struct io_read_wrapper; - struct layer_list; -@@ -84,10 +81,8 @@ int layer_store_check(const char *id); - container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); - - #ifdef ENABLE_REMOTE_LAYER_STORE --remote_support *layer_store_impl_remote_support(); --bool layer_remote_layer_valid(const char *layer_id); --int load_one_layer(const char *id); --int remove_memory_stores_with_lock(const char *id); -+int remote_load_one_layer(const char *id); -+int remote_layer_remove_memory_stores_with_lock(const char *id); - #endif - - #ifdef __cplusplus -diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c -similarity index 71% -rename from src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c -rename to src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c -index 20da8116..a822ea81 100644 ---- a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c -@@ -13,27 +13,23 @@ - * Description: provide remote image store functions - ******************************************************************************/ - #define _GNU_SOURCE --#include "image_store.h" -+#include "remote_support.h" - - #include - #include - --#include "remote_support.h" - #include "ro_symlink_maintain.h" - #include "map.h" - #include "utils_file.h" - #include "utils.h" - #include "layer_store.h" -+#include "image_store.h" - #include "utils_array.h" - --struct remote_image_data { -- const char *image_home; --}; -- - static map_t *image_byid_old = NULL; - static map_t *image_byid_new = NULL; - --static void *remote_support_create(const char *remote_home, const char *remote_ro) -+struct remote_image_data *remote_image_create(const char *remote_home, const char *remote_ro) - { - struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); - if (data == NULL) { -@@ -46,7 +42,7 @@ static void *remote_support_create(const char *remote_home, const char *remote_r - return data; - } - --static void remote_support_destroy(void *data) -+void remote_image_destroy(struct remote_image_data *data) - { - if (data == NULL) { - return; -@@ -59,7 +55,7 @@ static void remote_support_destroy(void *data) - return; - } - --static int remote_support_scan(void *data) -+static int remote_dir_scan(void *data) - { - int ret = 0; - int nret; -@@ -79,7 +75,7 @@ static int remote_support_scan(void *data) - image_dirs_num = util_array_len((const char **)image_dirs); - - for (i = 0; i < image_dirs_num; i++) { -- bool valid_v1_image = false; -+ bool is_v1_image = false; - - if (util_reg_match(id_patten, image_dirs[i]) != 0) { - DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", -@@ -93,12 +89,14 @@ static int remote_support_scan(void *data) - continue; - } - -- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { -+ if (image_store_validate_manifest_schema_version_1(image_path, &is_v1_image) != 0) { - ERROR("Failed to validate manifest schema version 1 format"); - continue; - } - -- if (!valid_v1_image) { -+ // for refresh, we don't care v1 image, cause image should be handled by master isulad -+ // when master isulad pull images -+ if (!is_v1_image) { - map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); - } - } -@@ -108,12 +106,13 @@ out: - return ret; - } - --static int remote_support_add(void *data) -+static int remote_image_add(void *data) - { - char **array_added = NULL; - char **array_deleted = NULL; - char *top_layer = NULL; - map_t *tmp_map = NULL; -+ bool exist = true; - int i = 0; - int ret = 0; - -@@ -121,26 +120,28 @@ static int remote_support_add(void *data) - return -1; - } - -- array_added = added_layers(image_byid_old, image_byid_new); -- array_deleted = deleted_layers(image_byid_old, image_byid_new); -+ array_added = remote_added_layers(image_byid_old, image_byid_new); -+ array_deleted = remote_deleted_layers(image_byid_old, image_byid_new); - - for (i = 0; i < util_array_len((const char **)array_added); i++) { -- top_layer = get_top_layer_from_json(array_added[i]); -- if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { -- ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); -+ top_layer = remote_image_get_top_layer_from_json(array_added[i]); -+ if (top_layer != NULL && !remote_layer_layer_valid(top_layer)) { -+ WARN("Current not find valid under layer, remoet image:%s not added", array_added[i]); - map_remove(image_byid_new, (void *)array_added[i]); - continue; - } - -- if (append_image_by_directory_with_lock(array_added[i]) != 0) { -+ if (remote_append_image_by_directory_with_lock(array_added[i]) != 0) { - ERROR("Failed to load image into memrory: %s", array_added[i]); -+ map_remove(image_byid_new, (void *)array_added[i]); - ret = -1; - } - } - - for (i = 0; i < util_array_len((const char **)array_deleted); i++) { -- if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { -+ if (remote_remove_image_from_memory_with_lock(array_deleted[i]) != 0) { - ERROR("Failed to remove remote memory store"); -+ map_insert(image_byid_new, array_deleted[i], (void *)&exist); - ret = -1; - } - } -@@ -148,7 +149,7 @@ static int remote_support_add(void *data) - tmp_map = image_byid_old; - image_byid_old = image_byid_new; - image_byid_new = tmp_map; -- empty_map(image_byid_new); -+ map_clear(image_byid_new); - - util_free_array(array_added); - util_free_array(array_deleted); -@@ -157,17 +158,13 @@ static int remote_support_add(void *data) - return ret; - } - --remote_support *image_store_impl_remote_support(void) --{ -- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -- if (rs == NULL) { -- return NULL; -+void remote_image_refresh(struct remote_image_data *data) { -+ if (remote_dir_scan(data) != 0) { -+ ERROR("remote overlay failed to scan dir, skip refresh"); -+ return; - } - -- rs->create = remote_support_create; -- rs->destroy = remote_support_destroy; -- rs->scan_remote_dir = remote_support_scan; -- rs->load_item = remote_support_add; -- -- return rs; -+ if (remote_image_add(data) != 0) { -+ ERROR("refresh overlay failed"); -+ } - } -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c -similarity index 76% -rename from src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -rename to src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c -index d676458c..3e3afff6 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c -@@ -13,7 +13,7 @@ - * Description: remote layer store implementation - ******************************************************************************/ - #define _GNU_SOURCE --#include "layer_store.h" -+#include "remote_support.h" - - #include - #include -@@ -21,20 +21,15 @@ - - #include "map.h" - #include "utils.h" --#include "remote_support.h" - #include "ro_symlink_maintain.h" -+#include "layer_store.h" - #include "path.h" - #include "driver_overlay2.h" - --struct remote_layer_data { -- const char *layer_home; -- const char *layer_ro; --}; -- - static map_t *layer_byid_old = NULL; - static map_t *layer_byid_new = NULL; - --static void *remote_support_create(const char *layer_home, const char *layer_ro) -+struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro) - { - struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); - if (data == NULL) { -@@ -49,7 +44,7 @@ static void *remote_support_create(const char *layer_home, const char *layer_ro) - return data; - }; - --static void remote_support_destroy(void *data) -+void remote_layer_destroy(struct remote_layer_data *data) - { - if (data == NULL) { - return; -@@ -72,10 +67,9 @@ static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_di - return true; - } - --static int remote_support_scan(void *data) -+static int remote_dir_scan(struct remote_layer_data *data) - { -- struct remote_layer_data *remote_data = data; -- return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); -+ return util_scan_subdirs(data->layer_ro, layer_walk_dir_cb, data); - } - - static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) -@@ -85,6 +79,11 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i - int nret = 0; - int ret = 0; - -+ if (layer_id == NULL) { -+ ERROR("can't delete NULL remote layer"); -+ return -1; -+ } -+ - nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); - if (nret < 0 || nret > PATH_MAX) { - SYSERROR("Create layer symbol link path failed"); -@@ -98,11 +97,14 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i - goto out; - } - -+ // return 0 if path already removed - if (util_path_remove(clean_path) != 0) { - SYSERROR("Failed to remove link path %s", clean_path); -+ ret = -1; -+ goto out; - } - -- if (remove_memory_stores_with_lock(layer_id) != 0) { -+ if (remote_layer_remove_memory_stores_with_lock(layer_id) != 0) { - ERROR("Failed to remove remote layer store memory"); - ret = -1; - } -@@ -110,7 +112,6 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i - out: - free(ro_symlink); - return ret; -- - } - - static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) -@@ -119,6 +120,11 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) - char *layer_dir = NULL; - int ret = 0; - -+ if (layer_id == NULL) { -+ ERROR("can't add NULL remote layer"); -+ return -1; -+ } -+ - ro_symlink = util_path_join(data->layer_home, layer_id); - layer_dir = util_path_join(data->layer_ro, layer_id); - -@@ -140,7 +146,7 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) - goto free_out; - } - // insert layer into memory -- if (load_one_layer(layer_id) != 0) { -+ if (remote_load_one_layer(layer_id) != 0) { - ERROR("Failed to load new layer: %s into memory", layer_id); - ret = -1; - } -@@ -152,30 +158,32 @@ free_out: - return ret; - } - --static int remote_support_add(void *data) -+static int remote_layer_add(struct remote_layer_data *data) - { - int ret = 0; - char **array_added = NULL; - char **array_deleted = NULL; - map_t *tmp_map = NULL; -+ bool exist = true; - int i = 0; - - if (data == NULL) { - return -1; - } - -- array_added = added_layers(layer_byid_old, layer_byid_new); -- array_deleted = deleted_layers(layer_byid_old, layer_byid_new); -+ array_added = remote_added_layers(layer_byid_old, layer_byid_new); -+ array_deleted = remote_deleted_layers(layer_byid_old, layer_byid_new); - - for (i = 0; i < util_array_len((const char **)array_added); i++) { -- if (!overlay_remote_layer_valid(array_added[i]) != 0) { -+ if (!remote_overlay_layer_valid(array_added[i]) != 0) { -+ WARN("remote overlay layer current not valid: %s", array_added[i]); - map_remove(layer_byid_new, (void *)array_added[i]); -- ERROR("remote overlay layer current not valid: %s", array_added[i]); - continue; - } - - if (add_one_remote_layer(data, array_added[i]) != 0) { - ERROR("Failed to add remote layer: %s", array_added[i]); -+ map_remove(layer_byid_new, (void *)array_added[i]); - ret = -1; - } - } -@@ -183,6 +191,7 @@ static int remote_support_add(void *data) - for (i = 0; i < util_array_len((const char **)array_deleted); i++) { - if (remove_one_remote_layer(data, array_deleted[i]) != 0) { - ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); -+ map_insert(layer_byid_new, array_deleted[i], (void *)&exist); - ret = -1; - } - } -@@ -190,7 +199,7 @@ static int remote_support_add(void *data) - tmp_map = layer_byid_old; - layer_byid_old = layer_byid_new; - layer_byid_new = tmp_map; -- empty_map(layer_byid_new); -+ map_clear(layer_byid_new); - - util_free_array(array_added); - util_free_array(array_deleted); -@@ -198,22 +207,20 @@ static int remote_support_add(void *data) - return ret; - } - --remote_support *layer_store_impl_remote_support() -+void remote_layer_refresh(struct remote_layer_data *data) - { -- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -- if (rs == NULL) { -- return NULL; -+ if (remote_dir_scan(data) != 0) { -+ ERROR("remote layer failed to scan dir, skip refresh"); -+ return; - } - -- rs->create = remote_support_create; -- rs->destroy = remote_support_destroy; -- rs->scan_remote_dir = remote_support_scan; -- rs->load_item = remote_support_add; -- -- return rs; -+ if (remote_layer_add(data) != 0) { -+ ERROR("refresh overlay failed"); -+ } - } - --bool layer_remote_layer_valid(const char *layer_id) -+ -+bool remote_layer_layer_valid(const char *layer_id) - { - return map_search(layer_byid_old, (void *)layer_id) != NULL; - } -diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c -similarity index 72% -rename from src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c -rename to src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c -index a674a00f..de2e583c 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c -@@ -13,13 +13,13 @@ - * Description: provide remote implementation for driver overlay - ******************************************************************************/ - #define _GNU_SOURCE --#include "driver_overlay2.h" -+#include "remote_support.h" - - #include - - #include "map.h" --#include "remote_support.h" - #include "ro_symlink_maintain.h" -+#include "driver_overlay2.h" - #include "isula_libutils/log.h" - #include "utils.h" - #include "utils_array.h" -@@ -29,15 +29,13 @@ - #define OVERLAY_LINK_DIR "l" - #define OVERLAY_LAYER_LINK "link" - --struct remote_overlay_data { -- const char *overlay_home; -- const char *overlay_ro; --}; -- -+// key: id, value: short id in 'l' dir -+// store short id to delete symbol link in 'l' dir - static map_t *overlay_byid_old = NULL; - static map_t *overlay_byid_new = NULL; -+static map_t *overlay_id_link = NULL; - --static void *remote_support_create(const char *remote_home, const char *remote_ro) -+struct remote_overlay_data *remote_overlay_create(const char *remote_home, const char *remote_ro) - { - struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); - if (data == NULL) { -@@ -48,11 +46,12 @@ static void *remote_support_create(const char *remote_home, const char *remote_r - data->overlay_ro = remote_ro; - overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); - overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ overlay_id_link = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); - - return data; - } - --static void remote_support_destroy(void *data) -+void remote_overlay_destroy(struct remote_overlay_data *data) - { - if (data == NULL) { - return; -@@ -60,6 +59,7 @@ static void remote_support_destroy(void *data) - - map_free(overlay_byid_old); - map_free(overlay_byid_new); -+ map_free(overlay_id_link); - free(data); - } - -@@ -74,10 +74,9 @@ static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_ - return true; - } - --static int remote_support_scan(void *data) -+static int remote_dir_scan(struct remote_overlay_data *data) - { -- struct remote_overlay_data *remote_data = data; -- return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); -+ return util_scan_subdirs(data->overlay_ro, overlay_walk_dir_cb, data); - } - - static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) -@@ -128,10 +127,17 @@ out: - static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) - { - char *ro_symlink = NULL; -+ char *link_path = NULL; -+ char *link_id = NULL; - char clean_path[PATH_MAX] = { 0 }; - int nret = 0; - int ret = 0; - -+ if (overlay_id == NULL) { -+ ERROR("can't remove NULL remote layer"); -+ return -1; -+ } -+ - nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); - if (nret < 0 || nret > PATH_MAX) { - SYSERROR("Create layer symbol link path failed"); -@@ -149,8 +155,40 @@ static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, con - SYSERROR("Failed to remove link path %s", clean_path); - } - -+ link_id = (char *)map_search(overlay_id_link, (void *)overlay_id); -+ -+ if (link_id == NULL) { -+ ERROR("Failed to find link id for overlay layer: %s", overlay_id); -+ ret = -1; -+ goto out; -+ } -+ -+ nret = asprintf(&link_path, "%s/%s/%s", data->overlay_home, OVERLAY_LINK_DIR, link_id); -+ if (nret < 0 || nret > PATH_MAX) { -+ SYSERROR("Create link path failed"); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { -+ ERROR("Failed to clean path: %s", ro_symlink); -+ ret = -1; -+ goto out; -+ } -+ -+ if (util_path_remove(clean_path) != 0) { -+ SYSERROR("Failed to remove link path %s", clean_path); -+ } -+ -+ if (!map_remove(overlay_id_link, (void *)overlay_id)) { -+ ERROR("Failed to remove link id for overlay layers: %s", overlay_id); -+ ret = -1; -+ goto out; -+ } -+ - out: - free(ro_symlink); -+ free(link_path); - return ret; - } - -@@ -162,6 +200,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const - char *diff_symlink = NULL; - int ret = 0; - -+ if (overlay_id == NULL) { -+ ERROR("can't add NULL remote layer"); -+ return -1; -+ } -+ - ro_symlink = util_path_join(data->overlay_home, overlay_id); - if (ro_symlink == NULL) { - ERROR("Failed to join ro symlink path: %s", overlay_id); -@@ -211,6 +254,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const - ret = -1; - } - -+ if (!map_insert(overlay_id_link, util_strdup_s(overlay_id), (void *)diff_symlink)) { -+ ERROR("can't insert remote layer into map"); -+ ret = -1; -+ } -+ - free_out: - free(ro_symlink); - free(layer_dir); -@@ -220,24 +268,26 @@ free_out: - return ret; - } - --static int remote_support_add(void *data) -+static int remote_image_add(struct remote_overlay_data *data) - { - int ret = 0; - char **array_added = NULL; - char **array_deleted = NULL; - map_t *tmp_map = NULL; -+ bool exist = true; - int i = 0; - - if (data == NULL) { - return -1; - } - -- array_added = added_layers(overlay_byid_old, overlay_byid_new); -- array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); -+ array_added = remote_added_layers(overlay_byid_old, overlay_byid_new); -+ array_deleted = remote_deleted_layers(overlay_byid_old, overlay_byid_new); - - for (i = 0; i < util_array_len((const char **)array_added); i++) { - if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { - ERROR("Failed to add remote overlay layer: %s", array_added[i]); -+ map_remove(overlay_byid_new, (void *)array_added[i]); - ret = -1; - } - } -@@ -245,6 +295,7 @@ static int remote_support_add(void *data) - for (i = 0; i < util_array_len((const char **)array_deleted); i++) { - if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { - ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); -+ map_insert(overlay_byid_new, array_deleted[i], (void *)&exist); - ret = -1; - } - } -@@ -252,7 +303,7 @@ static int remote_support_add(void *data) - tmp_map = overlay_byid_old; - overlay_byid_old = overlay_byid_new; - overlay_byid_new = tmp_map; -- empty_map(overlay_byid_new); -+ map_clear(overlay_byid_new); - - util_free_array(array_added); - util_free_array(array_deleted); -@@ -260,23 +311,19 @@ static int remote_support_add(void *data) - return ret; - } - --remote_support *overlay_driver_impl_remote_support(void) -+void remote_overlay_refresh(struct remote_overlay_data *data) - { -- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); -- if (rs == NULL) { -- ERROR("Failed to calloc overlay supporter"); -- return NULL; -+ if (remote_dir_scan(data) != 0) { -+ ERROR("remote overlay failed to scan dir, skip refresh"); -+ return; - } - -- rs->create = remote_support_create; -- rs->destroy = remote_support_destroy; -- rs->scan_remote_dir = remote_support_scan; -- rs->load_item = remote_support_add; -- -- return rs; -+ if (remote_image_add(data) != 0) { -+ ERROR("refresh overlay failed"); -+ } - } - --bool overlay_remote_layer_valid(const char *layer_id) -+bool remote_overlay_layer_valid(const char *layer_id) - { - return map_search(overlay_byid_old, (void *)layer_id) != NULL; - } -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -index 9dc096f7..3c7d0f54 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -@@ -15,108 +15,107 @@ - - #include "remote_support.h" - --#include "layer_store.h" --#include "image_store.h" -+#include -+ - #include "isula_libutils/log.h" --#include "driver_overlay2.h" - #include "utils.h" - --remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) --{ -- remote_support *handlers = layer_store_impl_remote_support(); -- if (handlers == NULL || handlers->create == NULL) { -- return NULL; -- } -+struct supporters { -+ struct remote_image_data *image_data; -+ struct remote_layer_data *layer_data; -+ struct remote_overlay_data *overlay_data; -+}; - -- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -- if (supporter == NULL) { -- goto err_out; -- } -+static struct supporters supporters; -+ -+static void *remote_refresh_ro_symbol_link(void *arg) -+{ -+ struct supporters *refresh_supporters = (struct supporters *)arg; -+ prctl(PR_SET_NAME, "RoLayerRefresh"); - -- supporter->handlers = handlers; -- supporter->data = handlers->create(remote_home, remote_ro); -+ while (true) { -+ util_usleep_nointerupt(5 * 1000 * 1000); -+ DEBUG("remote refresh start\n"); - -- return supporter; -+ remote_overlay_refresh(refresh_supporters->overlay_data); -+ remote_layer_refresh(refresh_supporters->layer_data); -+ remote_image_refresh(refresh_supporters->image_data); - --err_out: -- free(handlers); -- free(supporter); -+ DEBUG("remote refresh end\n"); -+ } - return NULL; - } - --remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) -+int remote_start_refresh_thread(void) - { -- remote_support *handlers = image_store_impl_remote_support(); -- if (handlers == NULL || handlers->create == NULL) { -- return NULL; -- } -+ int res = 0; -+ pthread_t a_thread; -+ maintain_context ctx = get_maintain_context(); - -- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -- if (supporter == NULL) { -- goto err_out; -+ supporters.image_data = remote_image_create(ctx.image_home, NULL); -+ if (supporters.image_data == NULL) { -+ goto free_out; - } - -- supporter->handlers = handlers; -- supporter->data = handlers->create(remote_home, remote_ro); -- -- return supporter; -- --err_out: -- free(handlers); -- free(supporter); -- return NULL; --} -+ supporters.layer_data = remote_layer_create(ctx.layer_home, ctx.layer_ro_dir); -+ if (supporters.layer_data == NULL) { -+ goto free_out; -+ } - --remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) --{ -- remote_support *handlers = overlay_driver_impl_remote_support(); -- if (handlers == NULL || handlers->create == NULL) { -- return NULL; -+ supporters.overlay_data = remote_overlay_create(ctx.overlay_home, ctx.overlay_ro_dir); -+ if (supporters.overlay_data == NULL) { -+ goto free_out; - } - -- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); -- if (supporter == NULL) { -- goto err_out; -+ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); -+ if (res != 0) { -+ CRIT("Thread creation failed"); -+ return -1; - } - -- supporter->handlers = handlers; -- supporter->data = handlers->create(remote_home, remote_ro); -+ if (pthread_detach(a_thread) != 0) { -+ SYSERROR("Failed to detach 0x%lx", a_thread); -+ return -1; -+ } - -- return supporter; -+ return 0; - --err_out: -- free(handlers); -- free(supporter); -- return NULL; -+free_out: -+ remote_image_destroy(supporters.image_data); -+ remote_layer_destroy(supporters.layer_data); -+ remote_overlay_destroy(supporters.overlay_data); - -+ return -1; - } - --void destroy_suppoter(remote_supporter *supporter) -+// this function calculate map_a - map_b => diff_list -+// diff_list contains keys inside map_a but not inside map_b -+static char **map_diff(const map_t *map_a, const map_t *map_b) - { -- if (supporter->handlers->destroy == NULL) { -- ERROR("destroy_supporter operation not supported"); -- return; -+ char **array = NULL; -+ map_itor *itor = map_itor_new(map_a); -+ bool *found = NULL; -+ -+ // iter new_map, every item not in old, append them to new_layers -+ for (; map_itor_valid(itor); map_itor_next(itor)) { -+ char *id = map_itor_key(itor); -+ found = map_search(map_b, id); -+ if (found == NULL) { -+ util_array_append(&array, util_strdup_s(id)); -+ } - } - -- supporter->handlers->destroy(supporter->data); -- free(supporter->handlers); -- free(supporter); -+ map_itor_free(itor); -+ -+ return array; - } - --int scan_remote_dir(remote_supporter *supporter) -+char **remote_deleted_layers(const map_t *old, const map_t *new) - { -- if (supporter->handlers->scan_remote_dir == NULL) { -- ERROR("scan_remote_dir operation not supported"); -- return -1; -- } -- return supporter->handlers->scan_remote_dir(supporter->data); -+ return map_diff(old, new); - } - --int load_item(remote_supporter *supporter) -+char **remote_added_layers(const map_t *old, const map_t *new) - { -- if (supporter->handlers->scan_remote_dir == NULL) { -- ERROR("load_item operation not supported"); -- return -1; -- } -- return supporter->handlers->load_item(supporter->data); -+ return map_diff(new, old); - } -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -index d1f7af35..892a9155 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -@@ -17,39 +17,59 @@ - #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H - - #include "linked_list.h" --#define REMOTE_RO_LAYER_DIR "RO" --#define OVERLAY_RO_DIR "RO" -+#include "map.h" -+#include "ro_symlink_maintain.h" - - #ifdef __cplusplus - extern "C" { - #endif - --typedef struct { -- void *(*create)(const char *remote_home, const char *remote_ro); -- void (*destroy)(void *data); -- // populate the list contains all dirs -- int (*scan_remote_dir)(void *data); -- // consume the list contains all dirs -- int (*load_item)(void *data); --} remote_support; -+struct remote_overlay_data { -+ const char *overlay_home; -+ const char *overlay_ro; -+}; - --typedef struct { -- void *data; -- remote_support *handlers; --} remote_supporter; -+struct remote_layer_data { -+ const char *layer_home; -+ const char *layer_ro; -+}; - --// RemoteSupport *impl_remote_support(); --remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); -+struct remote_image_data { -+ const char *image_home; -+}; - --remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); -+// image impl -+struct remote_image_data *remote_image_create(const char *image_home, const char *image_ro); - --remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); -+void remote_image_destroy(struct remote_image_data *data); - --void destroy_suppoter(remote_supporter *supporter); -+void remote_image_refresh(struct remote_image_data *data); - --int scan_remote_dir(remote_supporter *supporter); -+// layer impl -+struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro); - --int load_item(remote_supporter *supporter); -+void remote_layer_destroy(struct remote_layer_data *data); -+ -+void remote_layer_refresh(struct remote_layer_data *data); -+ -+bool remote_layer_layer_valid(const char *layer_id); -+ -+// overlay impl -+struct remote_overlay_data *remote_overlay_create(const char *overlay_home, const char *overlay_ro); -+ -+void remote_overlay_destroy(struct remote_overlay_data *data); -+ -+void remote_overlay_refresh(struct remote_overlay_data *data); -+ -+bool remote_overlay_layer_valid(const char *layer_id); -+ -+// start refresh remote -+int remote_start_refresh_thread(void); -+ -+// extra map utils -+char **remote_deleted_layers(const map_t *old, const map_t *new_l); -+ -+char **remote_added_layers(const map_t *old, const map_t *new_l); - - #ifdef __cplusplus - } -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -index 7df7a221..a3aa3aa4 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -@@ -17,19 +17,15 @@ - - #include - #include -+#include - #include - #include --#include "map.h" -+ - #include "path.h" - #include "linked_list.h" --#include "layer_store.h" --#include "layer.h" - #include "isula_libutils/log.h" --#include "image_store.h" --#include "remote_support.h" - #include "utils.h" - #include "utils_file.h" --#include "stdlib.h" - - #define REMOTE_RO_LAYER_DIR "RO" - -@@ -43,14 +39,6 @@ static char *layer_home; - static char *overlay_ro_dir; - static char *overlay_home; - --struct supporters { -- remote_supporter *image_supporter; -- remote_supporter *layer_supporter; -- remote_supporter *overlay_supporter; --}; -- --static struct supporters supporters; -- - int remote_image_init(const char *root_dir) - { - if (root_dir == NULL) { -@@ -134,75 +122,11 @@ void remote_maintain_cleanup(void) - overlay_ro_dir = NULL; - } - --// to maintain the symbol links, add new symbol link and delete invalid symbol link --// arg is const char *driver_home --// scanning driver->home/RO/ directory, build symlink in driver->home --static void *remote_refresh_ro_symbol_link(void *arg) --{ -- struct supporters *supporters = (struct supporters *)arg; -- prctl(PR_SET_NAME, "RoLayerRefresh"); -- -- while (true) { -- util_usleep_nointerupt(5 * 1000 * 1000); -- DEBUG("remote refresh start\n"); -- scan_remote_dir(supporters->overlay_supporter); -- load_item(supporters->overlay_supporter); -- scan_remote_dir(supporters->layer_supporter); -- load_item(supporters->layer_supporter); -- scan_remote_dir(supporters->image_supporter); -- load_item(supporters->image_supporter); -- DEBUG("remote refresh end\n"); -- } -- return NULL; --} -- --int start_refresh_thread(void) --{ -- int res = 0; -- pthread_t a_thread; -- -- supporters.image_supporter = create_image_supporter(image_home, NULL); -- if (supporters.image_supporter == NULL) { -- goto free_out; -- } -- -- supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); -- if (supporters.layer_supporter == NULL) { -- goto free_out; -- } -- -- supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); -- if (supporters.overlay_supporter == NULL) { -- goto free_out; -- } -- -- res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); -- if (res != 0) { -- CRIT("Thread creation failed"); -- return -1; -- } -- -- if (pthread_detach(a_thread) != 0) { -- SYSERROR("Failed to detach 0x%lx", a_thread); -- return -1; -- } -- -- return 0; -- --free_out: -- destroy_suppoter(supporters.image_supporter); -- destroy_suppoter(supporters.layer_supporter); -- destroy_suppoter(supporters.overlay_supporter); -- -- return -1; --} -- - static int do_build_ro_dir(const char *home, const char *id) - { - char *ro_symlink = NULL; - char *ro_layer_dir = NULL; - int nret = 0; -- // bool ret = true; - int ret = 0; - - nret = asprintf(&ro_symlink, "%s/%s", home, id); -@@ -305,43 +229,15 @@ int remote_overlay_remove_ro_dir(const char *id) - return do_remove_ro_dir(overlay_home, id); - } - --static char **map_diff(map_t *map_a, map_t *map_b) --{ -- char **array = NULL; -- map_itor *itor = map_itor_new(map_a); -- bool *found = NULL; -- -- // iter new_map, every item not in old, append them to new_layers -- for (; map_itor_valid(itor); map_itor_next(itor)) { -- char *id = map_itor_key(itor); -- found = map_search(map_b, id); -- if (found == NULL) { -- util_array_append(&array, util_strdup_s(id)); -- } -- } -- -- map_itor_free(itor); -- -- return array; --} -- --char **deleted_layers(map_t *old, map_t *new) --{ -- return map_diff(old, new); --} -- --char **added_layers(map_t *old, map_t *new) -+maintain_context get_maintain_context(void) - { -- return map_diff(new, old); --} -+ maintain_context ctx = {0x0}; - --int empty_map(map_t *mp) --{ -- if (mp == NULL) { -- return -1; -- } -+ ctx.image_home = image_home; -+ ctx.layer_ro_dir = layer_ro_dir; -+ ctx.layer_home = layer_home; -+ ctx.overlay_ro_dir = overlay_ro_dir; -+ ctx.overlay_home = overlay_home; - -- map_clear(mp); -- mp->store->root = mp->store->nil; -- return 0; -+ return ctx; - } -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h -index 25712d40..aa2036ea 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h -@@ -15,12 +15,21 @@ - #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H - #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H - --#include "map.h" -+#define REMOTE_RO_LAYER_DIR "RO" -+#define OVERLAY_RO_DIR "RO" - - #ifdef __cplusplus - extern "C" { - #endif - -+typedef struct { -+ const char *image_home; -+ const char *layer_ro_dir; -+ const char *layer_home; -+ const char *overlay_ro_dir; -+ const char *overlay_home; -+} maintain_context; -+ - int remote_image_init(const char *root_dir); - - int remote_layer_init(const char *root_dir); -@@ -29,8 +38,6 @@ int remote_overlay_init(const char *driver_home); - - void remote_maintain_cleanup(void); - --int start_refresh_thread(void); -- - int remote_layer_build_ro_dir(const char *id); - - int remote_overlay_build_ro_dir(const char *id); -@@ -39,11 +46,7 @@ int remote_layer_remove_ro_dir(const char *id); - - int remote_overlay_remove_ro_dir(const char *id); - --char **deleted_layers(map_t *old, map_t *new); -- --char **added_layers(map_t *old, map_t *new); -- --int empty_map(map_t *mp); -+maintain_context get_maintain_context(void); - - #ifdef __cplusplus - } -diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c -index 31812a22..f9830ac3 100644 ---- a/src/daemon/modules/image/oci/storage/storage.c -+++ b/src/daemon/modules/image/oci/storage/storage.c -@@ -43,7 +43,7 @@ - #include "utils_verify.h" - #include "sha256.h" - #ifdef ENABLE_REMOTE_LAYER_STORE --#include "ro_symlink_maintain.h" -+#include "remote_support.h" - #endif - - static pthread_rwlock_t g_storage_rwlock; -@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) - } - - #ifdef ENABLE_REMOTE_LAYER_STORE -- if (opts->enable_remote_layer && start_refresh_thread() != 0) { -+ if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { - ERROR("Failed to start remote refresh thread"); - } - #endif -diff --git a/src/utils/cutils/map/rb_tree.c b/src/utils/cutils/map/rb_tree.c -index e933003a..b85371e3 100644 ---- a/src/utils/cutils/map/rb_tree.c -+++ b/src/utils/cutils/map/rb_tree.c -@@ -134,6 +134,7 @@ void rbtree_clear(rb_tree_t *tree) - return; - } - rbtree_destroy_all(tree, tree->root); -+ tree->root = tree->nil; - } - - void rbtree_free(rb_tree_t *tree) -diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt -index 31240ae0..df184b09 100644 ---- a/test/image/oci/registry/CMakeLists.txt -+++ b/test/image/oci/registry/CMakeLists.txt -@@ -26,6 +26,7 @@ add_executable(${EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv2.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/http_request.c -@@ -56,6 +57,7 @@ target_include_directories(${EXE} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry - ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks - ) -diff --git a/test/image/oci/storage/CMakeLists.txt b/test/image/oci/storage/CMakeLists.txt -index feb81b14..7a9d77fe 100644 ---- a/test/image/oci/storage/CMakeLists.txt -+++ b/test/image/oci/storage/CMakeLists.txt -@@ -3,3 +3,6 @@ project(iSulad_UT) - add_subdirectory(images) - add_subdirectory(rootfs) - add_subdirectory(layers) -+IF (ENABLE_REMOTE_LAYER_STORE) -+add_subdirectory(remote_layer_support) -+ENDIF() -diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt -index 3e6b69a4..8446ebba 100644 ---- a/test/image/oci/storage/images/CMakeLists.txt -+++ b/test/image/oci/storage/images/CMakeLists.txt -@@ -21,6 +21,7 @@ add_executable(${EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry_type.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc - storage_images_ut.cc) -@@ -39,6 +40,7 @@ target_include_directories(${EXE} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks - ) -diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt -index 952e9483..ae0ac9c3 100644 ---- a/test/image/oci/storage/layers/CMakeLists.txt -+++ b/test/image/oci/storage/layers/CMakeLists.txt -@@ -30,6 +30,7 @@ add_executable(${DRIVER_EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc - storage_driver_ut.cc) -@@ -52,6 +53,7 @@ target_include_directories(${DRIVER_EXE} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks - ) -@@ -105,6 +107,7 @@ add_executable(${LAYER_EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc - storage_layers_ut.cc) - -@@ -129,6 +132,7 @@ target_include_directories(${LAYER_EXE} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks - ) - -diff --git a/test/image/oci/storage/remote_layer_support/CMakeLists.txt b/test/image/oci/storage/remote_layer_support/CMakeLists.txt -new file mode 100644 -index 00000000..c36d9049 ---- /dev/null -+++ b/test/image/oci/storage/remote_layer_support/CMakeLists.txt -@@ -0,0 +1,44 @@ -+project(iSulad_UT) -+ -+SET(EXE remote_layer_support_ut) -+ -+add_executable(${EXE} -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/remote_store_mock.cc -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/map.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_file.c -+ remote_layer_ut.cc -+ ) -+ -+target_include_directories(${EXE} PUBLIC -+ ${GTEST_INCLUDE_DIR} -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../include -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks -+ ) -+ -+target_link_libraries(${EXE} -+ ${GTEST_BOTH_LIBRARIES} -+ ${GMOCK_LIBRARY} -+ ${GMOCK_MAIN_LIBRARY} -+ ${CMAKE_THREAD_LIBS_INIT} -+ ${ISULA_LIBUTILS_LIBRARY} -+ -lgtest -lgtest_main libutils_ut -lcrypto -lyajl -lz) -+ -+add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) -+set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -diff --git a/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc -new file mode 100644 -index 00000000..5f5e92fb ---- /dev/null -+++ b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc -@@ -0,0 +1,93 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-16 -+ * Description: provide remote layer support ut -+ ******************************************************************************/ -+#include -+ -+#include "remote_store_mock.h" -+#include "ro_symlink_maintain.h" -+#include "remote_support.h" -+#include "map.h" -+ -+using ::testing::Invoke; -+ -+bool invokeOverlayRemoteLayerValid(const char *id) -+{ -+ return true; /* currently always valid overlay layer */ -+} -+ -+bool invokeLayerRemoteLayerValid(const char *id) -+{ -+ return true; -+} -+ -+int invokeLayerLoadOneLayer(const char *id) -+{ -+ return 0; -+} -+ -+int invokeLayerRemoveOneLayer(const char *id) -+{ -+ return 0; -+} -+ -+int invokeImageAppendOneImage(const char *id) -+{ -+ return 0; -+} -+ -+int invokeImageRemoveOneImage(const char *id) -+{ -+ return 0; -+} -+ -+char *invokeImageGetTopLayer(const char *id) -+{ -+ return NULL; -+} -+ -+int invokeImageValidSchemaV1(const char *path, bool *valid) -+{ -+ return 0; -+} -+ -+void mockCommonAll(MockRemoteStore *mock) -+{ -+ EXPECT_CALL(*mock, LayerLoadOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerLoadOneLayer)); -+ EXPECT_CALL(*mock, LayerRemoveOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerRemoveOneLayer)); -+ -+ EXPECT_CALL(*mock, ImageAppendOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageAppendOneImage)); -+ EXPECT_CALL(*mock, ImageRemoveOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageRemoveOneImage)); -+ EXPECT_CALL(*mock, ImageGetTopLayer(::testing::_)).WillRepeatedly(Invoke(invokeImageGetTopLayer)); -+ EXPECT_CALL(*mock, ImageValidSchemaV1(::testing::_, ::testing::_)).WillRepeatedly(Invoke(invokeImageValidSchemaV1)); -+} -+ -+TEST(remote_Layer_ut, test_map_diff) -+{ -+ // old: a b x -+ // new: x b c -+ map_t *old_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ map_t *new_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); -+ bool exist = true; -+ -+ map_insert(old_one, (void *)"a", (void *)&exist); -+ map_insert(old_one, (void *)"b", (void *)&exist); -+ map_insert(new_one, (void *)"b", (void *)&exist); -+ map_insert(new_one, (void *)"c", (void *)&exist); -+ -+ char **added = remote_added_layers(old_one, new_one); -+ char **deleted = remote_deleted_layers(old_one, new_one); -+ -+ ASSERT_EQ(added[0][0], 'c'); -+ ASSERT_EQ(deleted[0][0], 'a'); -+} -diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt -index 4d7d3533..5383fbac 100644 ---- a/test/image/oci/storage/rootfs/CMakeLists.txt -+++ b/test/image/oci/storage/rootfs/CMakeLists.txt -@@ -22,6 +22,7 @@ add_executable(${EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/utils_images.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc - storage_rootfs_ut.cc) -@@ -41,6 +42,7 @@ target_include_directories(${EXE} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage - ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support - ) - - target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) -diff --git a/test/mocks/remote_store_mock.cc b/test/mocks/remote_store_mock.cc -new file mode 100644 -index 00000000..c6428623 ---- /dev/null -+++ b/test/mocks/remote_store_mock.cc -@@ -0,0 +1,68 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-09 -+ * Description: provide mock for image store, layer store and driver overlay -+ ******************************************************************************/ -+ -+#include "remote_store_mock.h" -+ -+namespace { -+MockRemoteStore *g_remote_store_mock = nullptr; -+} -+ -+int remote_load_one_layer(const char *id) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->LayerLoadOneLayer(id); -+ } -+ return -1; -+} -+ -+int remote_layer_remove_memory_stores_with_lock(const char *id) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->LayerRemoveOneLayer(id); -+ } -+ return -1; -+} -+ -+int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->ImageValidSchemaV1(path, valid); -+ } -+ return -1; -+} -+ -+int remote_append_image_by_directory_with_lock(const char *image_dir) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->ImageAppendOneImage(image_dir); -+ } -+ return -1; -+} -+ -+int remote_remove_image_from_memory_with_lock(const char *id) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->ImageRemoveOneImage(id); -+ } -+ return -1; -+} -+ -+char *remote_image_get_top_layer_from_json(const char *img_id) -+{ -+ if (g_remote_store_mock != nullptr) { -+ return g_remote_store_mock->ImageGetTopLayer(img_id); -+ } -+ return nullptr; -+} -diff --git a/test/mocks/remote_store_mock.h b/test/mocks/remote_store_mock.h -new file mode 100644 -index 00000000..e8d73ef1 ---- /dev/null -+++ b/test/mocks/remote_store_mock.h -@@ -0,0 +1,40 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: wangrunze -+ * Create: 2023-03-09 -+ * Description: provide mock for image store, layer store and driver overlay -+ ******************************************************************************/ -+ -+#ifndef _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H -+#define _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H -+ -+#include -+ -+#include "image_store.h" -+#include "layer_store.h" -+#include "driver_overlay2.h" -+ -+class MockRemoteStore { -+public: -+ virtual ~MockRemoteStore() = default; -+ // MOCK_METHOD1(OverlayRemoteLayerValid, bool(const char *)); -+ -+ // MOCK_METHOD1(LayerRemoteLayerValid, bool(const char *)); -+ MOCK_METHOD1(LayerLoadOneLayer, int(const char *)); -+ MOCK_METHOD1(LayerRemoveOneLayer, int(const char *)); -+ -+ MOCK_METHOD1(ImageAppendOneImage, int(const char *)); -+ MOCK_METHOD1(ImageRemoveOneImage, int(const char *)); -+ MOCK_METHOD1(ImageGetTopLayer, char *(const char *)); -+ MOCK_METHOD2(ImageValidSchemaV1, int(const char *, bool *)); -+}; -+ -+#endif // _ISULAD_TEST_MOCKS_IMAGE_MOCK_H --- -2.25.1 - diff --git a/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch b/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch deleted file mode 100644 index 37a3bae..0000000 --- a/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch +++ /dev/null @@ -1,319 +0,0 @@ -From e15b302f7001507ebbf37d5485143f3124f8f54f Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Mon, 20 Mar 2023 23:47:25 -0700 -Subject: [PATCH 55/56] bugfix when refresh can't load or pull images - -Signed-off-by: Neil.wrz ---- - src/daemon/modules/image/oci/oci_image.c | 105 +++++++++++++++++- - .../remote_layer_support/remote_support.c | 34 +++++- - .../remote_layer_support/remote_support.h | 4 +- - .../modules/image/oci/storage/storage.c | 2 +- - .../modules/image/oci/storage/storage.h | 2 + - 5 files changed, 143 insertions(+), 4 deletions(-) - -diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c -index e4442a79..06a0c6ac 100644 ---- a/src/daemon/modules/image/oci/oci_image.c -+++ b/src/daemon/modules/image/oci/oci_image.c -@@ -41,6 +41,39 @@ - - struct oci_image_module_data g_oci_image_module_data = { 0 }; - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+// intend to make remote refresh and oci ops exlusive -+static bool g_enable_remote; -+static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER; -+ -+static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable) -+{ -+ int nret = 0; -+ -+ if (writable) { -+ nret = pthread_rwlock_wrlock(remote_lock); -+ } else { -+ nret = pthread_rwlock_rdlock(remote_lock); -+ } -+ if (nret != 0) { -+ ERROR("Lock memory store failed: %s", strerror(nret)); -+ return false; -+ } -+ -+ return true; -+} -+ -+static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock) -+{ -+ int nret = 0; -+ -+ nret = pthread_rwlock_unlock(remote_lock); -+ if (nret != 0) { -+ FATAL("Unlock memory store failed: %s", strerror(nret)); -+ } -+} -+#endif -+ - static void free_oci_image_data(void) - { - free(g_oci_image_module_data.root_dir); -@@ -216,6 +249,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) - - #ifdef ENABLE_REMOTE_LAYER_STORE - storage_opts->enable_remote_layer = args->storage_enable_remote_layer; -+ storage_opts->remote_lock = &g_remote_lock; - #endif - - if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, -@@ -299,6 +333,10 @@ int oci_init(const isulad_daemon_configs *args) - goto out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ g_enable_remote = args->storage_enable_remote_layer; -+#endif -+ - if (storage_module_init_helper(args) != 0) { - ret = -1; - goto out; -@@ -317,6 +355,7 @@ void oci_exit() - - int oci_pull_rf(const im_pull_request *request, im_pull_response *response) - { -+ int ret = 0; - if (request == NULL || request->image == NULL || response == NULL) { - ERROR("Invalid NULL param"); - return -1; -@@ -327,8 +366,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response) - isulad_try_set_error_message("Invalid image name: %s", request->image); - return -1; - } -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ // read lock here because pull have exclusive access against remote refresh -+ // pull can work concurrently with other oci operations. -+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { -+ ERROR("Failed to lock oci remote lock when load image"); -+ return -1; -+ } -+#endif -+ -+ ret = oci_do_pull_image(request, response); -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote) { -+ oci_remote_unlock(&g_remote_lock); -+ } -+#endif - -- return oci_do_pull_image(request, response); -+ return ret; - } - - int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs) -@@ -437,6 +492,15 @@ int oci_rmi(const im_rmi_request *request) - return -1; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ // read lock here because load have exclusive access against remote refresh -+ // load can work concurrently with other oci operations. -+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { -+ ERROR("Failed to lock oci remote lock when load image"); -+ return -1; -+ } -+#endif -+ - if (!util_valid_image_name(request->image.image)) { - ERROR("Invalid image name: %s", request->image.image); - isulad_try_set_error_message("Invalid image name: %s", request->image.image); -@@ -498,6 +562,11 @@ int oci_rmi(const im_rmi_request *request) - } - - out: -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote) { -+ oci_remote_unlock(&g_remote_lock); -+ } -+#endif - free(real_image_name); - free(image_ID); - util_free_array_by_len(image_names, image_names_len); -@@ -523,7 +592,24 @@ int oci_import(const im_import_request *request, char **id) - goto err_out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ // read lock here because import have exclusive access against remote refresh -+ // import can work concurrently with other oci operations. -+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { -+ ERROR("Failed to lock oci remote lock when load image"); -+ ret = -1; -+ goto err_out; -+ } -+#endif -+ - ret = oci_do_import(request->file, dest_name, id); -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote) { -+ oci_remote_unlock(&g_remote_lock); -+ } -+#endif -+ - if (ret != 0) { - goto err_out; - } -@@ -673,7 +759,24 @@ int oci_load_image(const im_load_request *request) - goto out; - } - -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ // read lock here because load have exclusive access against remote refresh -+ // load can work concurrently with other oci operations. -+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { -+ ERROR("Failed to lock oci remote lock when load image"); -+ ret = -1; -+ goto out; -+ } -+#endif -+ - ret = oci_do_load(request); -+ -+#ifdef ENABLE_REMOTE_LAYER_STORE -+ if (g_enable_remote) { -+ oci_remote_unlock(&g_remote_lock); -+ } -+#endif -+ - if (ret != 0) { - ERROR("Failed to load image"); - goto out; -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -index 3c7d0f54..7d457755 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c -@@ -24,10 +24,38 @@ struct supporters { - struct remote_image_data *image_data; - struct remote_layer_data *layer_data; - struct remote_overlay_data *overlay_data; -+ pthread_rwlock_t *remote_lock; - }; - - static struct supporters supporters; - -+static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable) -+{ -+ int nret = 0; -+ -+ if (writable) { -+ nret = pthread_rwlock_wrlock(remote_lock); -+ } else { -+ nret = pthread_rwlock_rdlock(remote_lock); -+ } -+ if (nret != 0) { -+ ERROR("Lock memory store failed: %s", strerror(nret)); -+ return false; -+ } -+ -+ return true; -+} -+ -+static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock) -+{ -+ int nret = 0; -+ -+ nret = pthread_rwlock_unlock(remote_lock); -+ if (nret != 0) { -+ FATAL("Unlock memory store failed: %s", strerror(nret)); -+ } -+} -+ - static void *remote_refresh_ro_symbol_link(void *arg) - { - struct supporters *refresh_supporters = (struct supporters *)arg; -@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg) - util_usleep_nointerupt(5 * 1000 * 1000); - DEBUG("remote refresh start\n"); - -+ remote_refresh_lock(supporters.remote_lock, true); - remote_overlay_refresh(refresh_supporters->overlay_data); - remote_layer_refresh(refresh_supporters->layer_data); - remote_image_refresh(refresh_supporters->image_data); -+ remote_refresh_unlock(supporters.remote_lock); - - DEBUG("remote refresh end\n"); - } - return NULL; - } - --int remote_start_refresh_thread(void) -+int remote_start_refresh_thread(pthread_rwlock_t *remote_lock) - { - int res = 0; - pthread_t a_thread; -@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void) - goto free_out; - } - -+ supporters.remote_lock = remote_lock; -+ - res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); - if (res != 0) { - CRIT("Thread creation failed"); -diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -index 892a9155..30e3ebb0 100644 ---- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h -@@ -16,6 +16,8 @@ - #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H - #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H - -+#include -+ - #include "linked_list.h" - #include "map.h" - #include "ro_symlink_maintain.h" -@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data); - bool remote_overlay_layer_valid(const char *layer_id); - - // start refresh remote --int remote_start_refresh_thread(void); -+int remote_start_refresh_thread(pthread_rwlock_t *remote_lock); - - // extra map utils - char **remote_deleted_layers(const map_t *old, const map_t *new_l); -diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c -index f9830ac3..836ccf4d 100644 ---- a/src/daemon/modules/image/oci/storage/storage.c -+++ b/src/daemon/modules/image/oci/storage/storage.c -@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) - } - - #ifdef ENABLE_REMOTE_LAYER_STORE -- if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { -+ if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) { - ERROR("Failed to start remote refresh thread"); - } - #endif -diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h -index 7404ee54..df9fd761 100644 ---- a/src/daemon/modules/image/oci/storage/storage.h -+++ b/src/daemon/modules/image/oci/storage/storage.h -@@ -18,6 +18,7 @@ - #include - #include - #include -+#include - #include - #include - -@@ -72,6 +73,7 @@ struct storage_module_init_options { - bool integration_check; - #ifdef ENABLE_REMOTE_LAYER_STORE - bool enable_remote_layer; -+ pthread_rwlock_t *remote_lock; - #endif - }; - --- -2.25.1 - diff --git a/0056-remove-unused-headers.patch b/0056-remove-unused-headers.patch deleted file mode 100644 index b37b9f4..0000000 --- a/0056-remove-unused-headers.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 2b798cf4053298dc44304319073cda1a00a466f4 Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Thu, 27 Apr 2023 00:26:15 -0700 -Subject: [PATCH 56/56] remove unused headers - -Signed-off-by: Neil.wrz ---- - src/daemon/modules/image/oci/storage/layer_store/layer_store.h | 1 - - src/daemon/modules/image/oci/storage/storage.h | 2 ++ - 2 files changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -index 4677e5ee..be8c52dc 100644 ---- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h -@@ -23,7 +23,6 @@ - - #include "storage.h" - #include "io_wrapper.h" --#include "map.h" - - struct io_read_wrapper; - struct layer_list; -diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h -index df9fd761..a761938c 100644 ---- a/src/daemon/modules/image/oci/storage/storage.h -+++ b/src/daemon/modules/image/oci/storage/storage.h -@@ -18,7 +18,9 @@ - #include - #include - #include -+#ifdef ENABLE_REMOTE_LAYER_STORE - #include -+#endif - #include - #include - --- -2.25.1 - diff --git a/0057-change-isulad-shim-epoll-struct.patch b/0057-change-isulad-shim-epoll-struct.patch deleted file mode 100644 index a270d9d..0000000 --- a/0057-change-isulad-shim-epoll-struct.patch +++ /dev/null @@ -1,1793 +0,0 @@ -From 15275529aed5b4f2e317902dfd3390ee114b310f Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Sun, 16 Apr 2023 13:56:18 +0800 -Subject: [PATCH 57/57] change isulad-shim epoll struct - -Signed-off-by: zhongtao ---- - src/cmd/isulad-shim/common.c | 9 + - src/cmd/isulad-shim/common.h | 2 + - src/cmd/isulad-shim/linked_list.h | 134 ++++ - src/cmd/isulad-shim/main.c | 13 +- - src/cmd/isulad-shim/mainloop.c | 168 +++++ - src/cmd/isulad-shim/mainloop.h | 53 ++ - src/cmd/isulad-shim/process.c | 941 ++++++++++++------------- - src/cmd/isulad-shim/process.h | 47 +- - test/cmd/isulad-shim/CMakeLists.txt | 1 + - test/cmd/isulad-shim/isulad-shim_ut.cc | 1 + - 10 files changed, 825 insertions(+), 544 deletions(-) - create mode 100644 src/cmd/isulad-shim/linked_list.h - create mode 100644 src/cmd/isulad-shim/mainloop.c - create mode 100644 src/cmd/isulad-shim/mainloop.h - -diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c -index e1ca96e1..18443385 100644 ---- a/src/cmd/isulad-shim/common.c -+++ b/src/cmd/isulad-shim/common.c -@@ -524,4 +524,13 @@ err_out: - util_free_array(res_array); - errno = tmp_errno; - return NULL; -+} -+ -+void *util_common_calloc_s(size_t size) -+{ -+ if (size == 0 || size > MAX_MEMORY_SIZE) { -+ return NULL; -+ } -+ -+ return calloc((size_t)1, size); - } -\ No newline at end of file -diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h -index 3de16ace..d733823b 100644 ---- a/src/cmd/isulad-shim/common.h -+++ b/src/cmd/isulad-shim/common.h -@@ -128,6 +128,8 @@ char *util_strdup_s(const char *src); - - char **util_string_split_multi(const char *src_str, char delim); - -+void *util_common_calloc_s(size_t size); -+ - #ifdef __cplusplus - } - #endif -diff --git a/src/cmd/isulad-shim/linked_list.h b/src/cmd/isulad-shim/linked_list.h -new file mode 100644 -index 00000000..3a87990a ---- /dev/null -+++ b/src/cmd/isulad-shim/linked_list.h -@@ -0,0 +1,134 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: tanyifeng -+ * Create: 2018-11-08 -+ * Description: provide container linked list function definition -+ ******************************************************************************/ -+#ifndef __LINKED_LIST_H -+#define __LINKED_LIST_H -+ -+#include -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+struct linked_list { -+ void *elem; -+ struct linked_list *next; -+ struct linked_list *prev; -+}; -+ -+/* Iterate through an linked list. */ -+#define linked_list_for_each(__iterator, __list) \ -+ for ((__iterator) = (__list)->next; \ -+ (__iterator) != (__list); \ -+ (__iterator) = (__iterator)->next) -+ -+/* Iterate safely through an linked list. */ -+#define linked_list_for_each_safe(__iterator, __list, __next) \ -+ for ((__iterator) = (__list)->next, (__next) = (__iterator)->next; \ -+ (__iterator) != (__list); \ -+ (__iterator) = (__next), (__next) = (__next)->next) -+ -+/* Initialize list. */ -+static inline void linked_list_init(struct linked_list *list) -+{ -+ list->elem = NULL; -+ list->next = list->prev = list; -+} -+ -+/* Add an element to a list. See linked_list_add() and linked_list_add_tail() for an -+ * idiom. */ -+static inline void linked_list_add_elem(struct linked_list *list, void *elem) -+{ -+ list->elem = elem; -+} -+ -+/* Retrieve first element of list. */ -+static inline void *linked_list_first_elem(struct linked_list *list) -+{ -+ return list->next->elem; -+} -+ -+/* Retrieve last element of list. */ -+static inline void *linked_list_last_elem(struct linked_list *list) -+{ -+ return list->prev->elem; -+} -+ -+/* Retrieve first node of list. */ -+static inline void *linked_list_first_node(struct linked_list *list) -+{ -+ return list->next; -+} -+ -+/* Determine if list is empty. */ -+static inline int linked_list_empty(struct linked_list *list) -+{ -+ return list == list->next; -+} -+ -+/* Workhorse to be called from linked_list_add() and linked_list_add_tail(). */ -+static inline void __linked_list_add(struct linked_list *newlist, -+ struct linked_list *prev, -+ struct linked_list *next) -+{ -+ next->prev = newlist; -+ newlist->next = next; -+ newlist->prev = prev; -+ prev->next = newlist; -+} -+ -+/* Idiom to add an element to the beginning of an linked list */ -+static inline void linked_list_add(struct linked_list *head, -+ struct linked_list *list) -+{ -+ __linked_list_add(list, head, head->next); -+} -+ -+/* Idiom to add an element to the end of an linked list */ -+static inline void linked_list_add_tail(struct linked_list *head, -+ struct linked_list *list) -+{ -+ __linked_list_add(list, head->prev, head); -+} -+ -+/* Idiom to free an linked list */ -+static inline void linked_list_del(const struct linked_list *list) -+{ -+ struct linked_list *next = NULL; -+ struct linked_list *prev = NULL; -+ -+ next = list->next; -+ prev = list->prev; -+ next->prev = prev; -+ prev->next = next; -+} -+ -+/* Return length of the list. */ -+static inline size_t linked_list_len(struct linked_list *list) -+{ -+ size_t i = 0; -+ struct linked_list *iter = NULL; -+ linked_list_for_each(iter, list) { -+ i++; -+ } -+ -+ return i; -+} -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -+ -diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c -index 68e99e53..0cccdf06 100644 ---- a/src/cmd/isulad-shim/main.c -+++ b/src/cmd/isulad-shim/main.c -@@ -105,9 +105,9 @@ int main(int argc, char **argv) - int ret = SHIM_ERR; - int efd = -1; - process_t *p = NULL; -- pthread_t tid_accept; - // execSync timeout - uint64_t timeout = 0; -+ pthread_t tid_epoll; - - g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); - if (g_log_fd < 0) { -@@ -154,18 +154,13 @@ int main(int argc, char **argv) - } - } - -- /* create main loop and start epoll for io copy */ -- ret = process_io_init(p); -+ /* start epoll for io copy */ -+ ret = process_io_start(p, &tid_epoll); - if (ret != SHIM_OK) { - write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret); - exit(EXIT_FAILURE); - } - -- ret = open_io(p, &tid_accept); -- if (ret != SHIM_OK) { -- exit(EXIT_FAILURE); -- } -- - ret = create_process(p); - if (ret != SHIM_OK) { - if (p->console_sock_path != NULL) { -@@ -176,5 +171,5 @@ int main(int argc, char **argv) - - released_timeout_exit(); - -- return process_signal_handle_routine(p, tid_accept, timeout); -+ return process_signal_handle_routine(p, tid_epoll, timeout); - } -diff --git a/src/cmd/isulad-shim/mainloop.c b/src/cmd/isulad-shim/mainloop.c -new file mode 100644 -index 00000000..53c4d856 ---- /dev/null -+++ b/src/cmd/isulad-shim/mainloop.c -@@ -0,0 +1,168 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: tanyifeng -+ * Create: 2018-11-08 -+ * Description: provide container mainloop functions -+ ******************************************************************************/ -+#include "mainloop.h" -+#include -+#include -+#include -+#include -+#include -+ -+#include "common.h" -+ -+struct epoll_loop_handler { -+ epoll_loop_callback_t cb; -+ int cbfd; -+ void *cbdata; -+}; -+ -+#define MAX_EVENTS 100 -+ -+/* epoll loop */ -+int epoll_loop(struct epoll_descr *descr, int t) -+{ -+ int i; -+ int ret = 0; -+ struct epoll_loop_handler *epoll_handler = NULL; -+ struct epoll_event evs[MAX_EVENTS]; -+ -+ while (1) { -+ int ep_fds = epoll_wait(descr->fd, evs, MAX_EVENTS, t); -+ if (ep_fds < 0) { -+ if (errno == EINTR) { -+ continue; -+ } -+ ret = -1; -+ goto out; -+ } -+ -+ for (i = 0; i < ep_fds; i++) { -+ epoll_handler = (struct epoll_loop_handler *)(evs[i].data.ptr); -+ if (epoll_handler->cb(epoll_handler->cbfd, evs[i].events, epoll_handler->cbdata, descr) != -+ EPOLL_LOOP_HANDLE_CONTINUE) { -+ goto out; -+ } -+ } -+ -+ if (ep_fds == 0 && t != 0) { -+ if (descr->timeout_cb != NULL) { -+ descr->timeout_cb(descr->timeout_cbdata); -+ } -+ goto out; -+ } -+ -+ if (linked_list_empty(&descr->handler_list)) { -+ goto out; -+ } -+ } -+out: -+ return ret; -+} -+ -+/* epoll loop add handler */ -+int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data) -+{ -+ struct epoll_event ev = { 0 }; -+ struct epoll_loop_handler *epoll_handler = NULL; -+ struct linked_list *node = NULL; -+ -+ // if fd == -1; dose not add handler for it. -+ if (fd < 0) { -+ return 0; -+ } -+ -+ epoll_handler = util_common_calloc_s(sizeof(*epoll_handler)); -+ if (epoll_handler == NULL) { -+ goto fail_out; -+ } -+ -+ epoll_handler->cbfd = fd; -+ epoll_handler->cb = callback; -+ epoll_handler->cbdata = data; -+ -+ ev.events = EPOLLIN; -+ ev.data.ptr = epoll_handler; -+ -+ if (epoll_ctl(descr->fd, EPOLL_CTL_ADD, fd, &ev) < 0) { -+ goto fail_out; -+ } -+ -+ node = util_common_calloc_s(sizeof(struct linked_list)); -+ if (node == NULL) { -+ goto fail_out; -+ } -+ -+ node->elem = epoll_handler; -+ linked_list_add(&descr->handler_list, node); -+ return 0; -+ -+fail_out: -+ (void)epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, &ev); -+ free(epoll_handler); -+ return -1; -+} -+ -+/* epoll loop del handler */ -+int epoll_loop_del_handler(struct epoll_descr *descr, int fd) -+{ -+ struct epoll_loop_handler *epoll_handler = NULL; -+ struct linked_list *index = NULL; -+ -+ linked_list_for_each(index, &descr->handler_list) { -+ epoll_handler = index->elem; -+ -+ if (fd == epoll_handler->cbfd) { -+ if (epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, NULL)) { -+ goto fail_out; -+ } -+ -+ linked_list_del(index); -+ free(index->elem); -+ free(index); -+ return 0; -+ } -+ } -+ -+fail_out: -+ return -1; -+} -+ -+/* epoll loop open */ -+int epoll_loop_open(struct epoll_descr *descr) -+{ -+ descr->fd = epoll_create1(EPOLL_CLOEXEC); -+ if (descr->fd < 0) { -+ return -1; -+ } -+ -+ linked_list_init(&(descr->handler_list)); -+ descr->timeout_cb = NULL; -+ descr->timeout_cbdata = NULL; -+ return 0; -+} -+ -+/* epoll loop close */ -+int epoll_loop_close(struct epoll_descr *descr) -+{ -+ struct linked_list *index = NULL; -+ struct linked_list *next = NULL; -+ -+ linked_list_for_each_safe(index, &(descr->handler_list), next) { -+ linked_list_del(index); -+ free(index->elem); -+ free(index); -+ } -+ -+ return close(descr->fd); -+} -diff --git a/src/cmd/isulad-shim/mainloop.h b/src/cmd/isulad-shim/mainloop.h -new file mode 100644 -index 00000000..7a4f1cfd ---- /dev/null -+++ b/src/cmd/isulad-shim/mainloop.h -@@ -0,0 +1,53 @@ -+/****************************************************************************** -+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. -+ * iSulad licensed under the Mulan PSL v2. -+ * You can use this software according to the terms and conditions of the Mulan PSL v2. -+ * You may obtain a copy of Mulan PSL v2 at: -+ * http://license.coscl.org.cn/MulanPSL2 -+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+ * PURPOSE. -+ * See the Mulan PSL v2 for more details. -+ * Author: tanyifeng -+ * Create: 2018-11-08 -+ * Description: provide container mainloop definition -+ ******************************************************************************/ -+#ifndef UTILS_CUTILS_MAINLOOP_H -+#define UTILS_CUTILS_MAINLOOP_H -+ -+#include -+#include "linked_list.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+typedef void (*epoll_timeout_callback_t)(void *data); -+ -+struct epoll_descr { -+ int fd; -+ struct linked_list handler_list; -+ epoll_timeout_callback_t timeout_cb; -+ void *timeout_cbdata; -+}; -+ -+#define EPOLL_LOOP_HANDLE_CONTINUE 0 -+#define EPOLL_LOOP_HANDLE_CLOSE 1 -+ -+typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data, struct epoll_descr *descr); -+ -+extern int epoll_loop(struct epoll_descr *descr, int t); -+ -+extern int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data); -+ -+extern int epoll_loop_del_handler(struct epoll_descr *descr, int fd); -+ -+extern int epoll_loop_open(struct epoll_descr *descr); -+ -+extern int epoll_loop_close(struct epoll_descr *descr); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index a5e0bd39..c31f8af6 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -28,6 +28,7 @@ - #include - #include - #include -+#include - #include // IWYU pragma: keep - #include // IWYU pragma: keep - #include -@@ -37,6 +38,7 @@ - - #include "common.h" - #include "terminal.h" -+#include "mainloop.h" - - #define MAX_EVENTS 100 - #define DEFAULT_IO_COPY_BUF (16 * 1024) -@@ -78,7 +80,7 @@ static int receive_fd(int sock) - u_char *pfd = NULL; - int fd = -1; - int cmsgsize = CMSG_LEN(sizeof(int)); -- struct cmsghdr *cmptr = (struct cmsghdr *)calloc(1, cmsgsize); -+ struct cmsghdr *cmptr = (struct cmsghdr *)util_common_calloc_s(cmsgsize); - if (cmptr == NULL) { - return -1; - } -@@ -128,107 +130,6 @@ static bool check_fd(int fd) - return true; - } - --static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to) --{ -- int ret = SHIM_ERR; -- -- if (io_thd == NULL || io_thd->ioc == NULL) { -- return SHIM_ERR; -- } -- -- io_copy_t *ioc = io_thd->ioc; -- -- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { -- return SHIM_ERR; -- } -- /* add src fd */ -- if (from != -1 && ioc->fd_from == -1) { -- ioc->fd_from = from; -- struct epoll_event ev; -- ev.events = EPOLLIN; -- ev.data.ptr = io_thd; -- -- ret = epoll_ctl(epfd, EPOLL_CTL_ADD, from, &ev); -- if (ret != SHIM_OK) { -- write_message(g_log_fd, ERR_MSG, "add fd %d to epoll loop failed:%d", from, SHIM_SYS_ERR(errno)); -- pthread_mutex_unlock(&(ioc->mutex)); -- return SHIM_ERR; -- } -- } -- -- /* add dest fd */ -- if (to != -1) { -- /* new fd_node_t for dest fd */ -- fd_node_t *fn = (fd_node_t *)calloc(1, sizeof(fd_node_t)); -- if (fn == NULL) { -- pthread_mutex_unlock(&(ioc->mutex)); -- return SHIM_ERR; -- } -- fn->fd = to; -- fn->is_log = false; -- if (io_thd->terminal != NULL && to == io_thd->terminal->fd) { -- fn->is_log = true; -- } -- fn->next = NULL; -- -- if (ioc->fd_to == NULL) { -- ioc->fd_to = fn; -- } else { -- fd_node_t *tmp = ioc->fd_to; -- while (tmp->next != NULL) { -- tmp = tmp->next; -- } -- tmp->next = fn; -- } -- } -- pthread_mutex_unlock(&(ioc->mutex)); -- -- return SHIM_OK; --} -- --static void remove_io_dispatch(io_thread_t *io_thd, int from, int to) --{ -- if (io_thd == NULL || io_thd->ioc == NULL) { -- return; -- } -- io_copy_t *ioc = io_thd->ioc; -- -- fd_node_t *tmp = NULL; -- do { -- /* remove src fd */ -- if (from != -1 && from == ioc->fd_from) { -- struct epoll_event ev; -- ev.events = EPOLLIN; -- ev.data.fd = ioc->fd_from; -- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); -- } -- -- /* remove dest fd */ -- if (ioc->fd_to == NULL) { -- break; -- } -- if (ioc->fd_to->fd == to) { -- /* remove the first fd node */ -- tmp = ioc->fd_to; -- ioc->fd_to = ioc->fd_to->next; -- break; -- } -- fd_node_t *pre = ioc->fd_to; -- tmp = ioc->fd_to->next; -- while (tmp != NULL && tmp->fd != to) { -- pre = tmp; -- tmp = tmp->next; -- } -- if (tmp != NULL) { -- pre->next = tmp->next; -- } -- } while (0); -- if (tmp != NULL) { -- free(tmp); -- tmp = NULL; -- } --} -- - static int get_exec_winsize(const char *buf, struct winsize *wsize) - { - char **array = NULL; -@@ -261,347 +162,287 @@ out: - return ret; - } - --static void *do_io_copy(void *data) -+static int sync_exit_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) - { -- io_thread_t *io_thd = (io_thread_t *)data; -- if (io_thd == NULL || io_thd->ioc == NULL) { -- return NULL; -- } -- io_copy_t *ioc = io_thd->ioc; -- char *buf = calloc(1, DEFAULT_IO_COPY_BUF + 1); -- if (buf == NULL) { -- return NULL; -- } -+ return EPOLL_LOOP_HANDLE_CLOSE; -+} - -- for (;;) { -- memset(buf, 0, DEFAULT_IO_COPY_BUF); -- (void)sem_wait(&(io_thd->sem_thd)); -- if (io_thd->is_stdin && io_thd->shutdown) { -- break; -- } -+static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) -+{ -+ process_t *p = (process_t *)cbdata; -+ int r_count = 0; -+ int w_count = 0; -+ int *fd_to = NULL; - -- int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); -- if (r_count == -1) { -- if (errno == EAGAIN || errno == EINTR) { -- continue; -- } -- break; -- } else if (r_count == 0) { -- /* End of file. The remote has closed the connection */ -- break; -- } else if (ioc->id != EXEC_RESIZE) { -- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { -- continue; -- } -+ if (events & EPOLLHUP) { -+ return EPOLL_LOOP_HANDLE_CLOSE; -+ } - -- fd_node_t *fn = ioc->fd_to; -- fd_node_t *next = NULL; -- for (; fn != NULL; fn = next) { -- next = fn->next; -- if (fn->is_log) { -- shim_write_container_log_file(io_thd->terminal, ioc->id, buf, r_count); -- } else { -- int w_count = write_nointr_in_total(fn->fd, buf, r_count); -- if (w_count < 0) { -- /* When any error occurs, remove the write fd */ -- remove_io_dispatch(io_thd, -1, fn->fd); -- } -- } -- } -- pthread_mutex_unlock(&(ioc->mutex)); -- } else { -- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { -- continue; -- } -+ if (!(events & EPOLLIN)) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; -+ } - -- int resize_fd = ioc->fd_to->fd; -- struct winsize wsize = { 0x00 }; -- if (get_exec_winsize(buf, &wsize) < 0) { -- break; -- } -- if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { -- break; -- } -- pthread_mutex_unlock(&(ioc->mutex)); -- } -+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); - -- /* -- In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe. -- So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs. -- */ -- if (io_thd->shutdown && r_count <= 0) { -- break; -- } -+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); -+ if (r_count <= 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } -- struct epoll_event ev; -- ev.events = EPOLLIN; -- ev.data.fd = ioc->fd_from; -- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); - -- free(buf); -+ if (p->state->terminal) { -+ fd_to = &(p->recv_fd); -+ } else { -+ fd_to = &(p->shim_io->in); -+ } -+ w_count = write_nointr_in_total(*fd_to, p->buf, r_count); -+ if (w_count < 0) { -+ /* When any error occurs, set the write fd -1 */ -+ write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); -+ close(*fd_to); -+ *fd_to = -1; -+ } - -- return NULL; -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } - --static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data) -+static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) - { -- io_thread_t *thd = (io_thread_t *)data; -- if (thd->ioc == NULL || fd != thd->ioc->fd_from) { -- return; -+ process_t *p = (process_t *)cbdata; -+ int r_count = 0; -+ int w_count = 0; -+ -+ if (events & EPOLLHUP) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- if (event & EPOLLIN) { -- (void)sem_post(&thd->sem_thd); -- } else if (event & EPOLLHUP) { -- thd->shutdown = true; -- (void)sem_post(&thd->sem_thd); -+ if (!(events & EPOLLIN)) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } --} - --static int create_io_copy_thread(process_t *p, int std_id) --{ -- int ret = SHIM_ERR; -- io_thread_t *io_thd = NULL; -- io_copy_t *ioc = NULL; -+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); - -- ioc = (io_copy_t *)calloc(1, sizeof(io_copy_t)); -- if (ioc == NULL) { -- goto failure; -+ if (p->block_read) { -+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); -+ } else { -+ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); - } -- ioc->id = std_id; -- ioc->fd_from = -1; -- ioc->fd_to = NULL; -- if (pthread_mutex_init(&(ioc->mutex), NULL) != 0) { -- goto failure; -+ if (r_count <= 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- io_thd = (io_thread_t *)calloc(1, sizeof(io_thread_t)); -- if (io_thd == NULL) { -- goto failure; -+ shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); -+ -+ if (p->isulad_io->out == -1) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } -- if (sem_init(&io_thd->sem_thd, 0, 0) != 0) { -- write_message(g_log_fd, ERR_MSG, "sem init failed:%d", SHIM_SYS_ERR(errno)); -- goto failure; -+ -+ w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count); -+ if (w_count < 0) { -+ /* When any error occurs, set the write fd -1 */ -+ write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); -+ close(p->isulad_io->out); -+ p->isulad_io->out = -1; - } -- io_thd->epfd = p->io_loop_fd; -- io_thd->ioc = ioc; -- io_thd->shutdown = false; -- io_thd->is_stdin = std_id == STDID_IN ? true : false; -- io_thd->terminal = std_id != STDID_IN ? p->terminal : NULL; - -- p->io_threads[std_id] = io_thd; -+ return EPOLL_LOOP_HANDLE_CONTINUE; -+} - -- ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd); -- if (ret != SHIM_OK) { -- write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno)); -- goto failure; -+static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) -+{ -+ process_t *p = (process_t *)cbdata; -+ int r_count = 0; -+ int w_count = 0; -+ -+ if (events & EPOLLHUP) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- ret = SHIM_OK; -+ if (!(events & EPOLLIN)) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; -+ } - -- return ret; -+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); - --failure: -- if (ioc != NULL) { -- pthread_mutex_destroy(&(ioc->mutex)); -- free(ioc); -+ if (p->block_read) { -+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); -+ } else { -+ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); - } -- if (io_thd != NULL) { -- free(io_thd); -+ if (r_count <= 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- return SHIM_ERR; --} -- --static int start_io_copy_threads(process_t *p) --{ -- int ret = SHIM_ERR; -- int i; -- -- /* 4 threads for stdin, stdout, stderr and exec resize */ -- for (i = 0; i < 4; i++) { -- /* -- * if the terminal is used, we do not need to active the io copy of stderr pipe, -- * for stderr and stdout are mixed together -- */ -- if (i == STDID_ERR && p->state->terminal) { -- continue; -- } -+ shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); - -- ret = create_io_copy_thread(p, i); -- if (ret != SHIM_OK) { -- return SHIM_ERR; -- } -+ if (p->isulad_io->err == -1) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } -- return SHIM_OK; --} - --static void destroy_io_thread(process_t *p, int std_id) --{ -- io_thread_t *io_thd = p->io_threads[std_id]; -- if (io_thd == NULL) { -- return; -+ w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count); -+ if (w_count < 0) { -+ /* When any error occurs, set the write fd -1 */ -+ write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); -+ close(p->isulad_io->err); -+ p->isulad_io->err = -1; - } - -- io_thd->shutdown = true; -- (void)sem_post(&io_thd->sem_thd); -- pthread_join(io_thd->tid, NULL); -- if (io_thd->ioc != NULL) { -- free(io_thd->ioc); -- } -- free(io_thd); -- p->io_threads[std_id] = NULL; -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } - --/* -- std_id: channel type -- isulad_stdio: one side of the isulad fifo file -- fd: one side of the shim io pipe -- --------------------------------------------------------------- -- | CHANNEL | iSulad Fifo Side | Flow Direction | fd | -- --------------------------------------------------------------- -- | STDIN | READ | --> | WRITE | -- --------------------------------------------------------------- -- | STDOUT | WRITE | <-- | READ | -- --------------------------------------------------------------- -- | STDERR | WRITE | <-- | READ | -- --------------------------------------------------------------- -- | RESIZE | READ | --> | WRITE | -- --------------------------------------------------------------- --*/ --static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd) -+static int resize_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) - { -- mode_t mode; -- int fd_isulad = -1; -- int *fd_from = NULL; -- int *fd_to = NULL; -+ process_t *p = (process_t *)cbdata; -+ int r_count = 0; -+ int resize_fd = -1; - -- if (std_id == STDID_IN || std_id == EXEC_RESIZE) { -- mode = O_RDONLY; -- fd_from = &fd_isulad; -- fd_to = &fd; -- } else { -- mode = O_WRONLY; -- fd_from = &fd; -- fd_to = &fd_isulad; -+ if (events & EPOLLHUP) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- if (isulad_stdio != NULL && file_exists(isulad_stdio)) { -- fd_isulad = open_fifo_noblock(isulad_stdio, mode); -- if (fd_isulad < 0) { -- return SHIM_ERR; -- } -- /* open dummy fd to avoid resize epoll hub */ -- if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { -- return SHIM_ERR; -- } -+ if (!(events & EPOLLIN)) { -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } - -- if (*fd_from != -1) { -- if (std_id != STDID_IN && std_id != EXEC_RESIZE && p->io_threads[std_id]->terminal != NULL) { -- (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd); -- } -- return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to); -+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); -+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); -+ if (r_count <= 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; - } - -- /* if no I/O source is available, the I/O thread nead to be destroyed */ -- destroy_io_thread(p, std_id); -+ resize_fd = p->recv_fd; -+ struct winsize wsize = { 0x00 }; -+ if (get_exec_winsize(p->buf, &wsize) < 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; -+ } -+ if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { -+ return EPOLL_LOOP_HANDLE_CLOSE; -+ } - -- return SHIM_OK; -+ return EPOLL_LOOP_HANDLE_CONTINUE; - } - --static void *task_console_accept(void *data) -+static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) - { -+ process_t *p = (process_t *)cbdata; - int conn_fd = -1; -- int recv_fd = -1; - int ret = SHIM_ERR; -- console_accept_t *ac = (console_accept_t *)data; - -- conn_fd = accept(ac->listen_fd, NULL, NULL); -+ conn_fd = accept(p->listen_fd, NULL, NULL); - if (conn_fd < 0) { -- write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno)); -+ write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); - goto out; - } - -- recv_fd = receive_fd(conn_fd); -- if (check_fd(recv_fd) != true) { -+ p->recv_fd = receive_fd(conn_fd); -+ if (check_fd(p->recv_fd) != true) { - write_message(g_log_fd, ERR_MSG, "check console fd failed"); - goto out; - } - - /* do console io copy */ - -- /* p.state.stdin---->runtime.console */ -- ret = connect_to_isulad(ac->p, STDID_IN, ac->p->state->isulad_stdin, recv_fd); -+ // p->isulad_io->in ----> p->recv_fd -+ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); - if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); - goto out; - } -- -- /* p.state.stdout<------runtime.console */ -- ret = connect_to_isulad(ac->p, STDID_OUT, ac->p->state->isulad_stdout, recv_fd); -+ // p->recv_fd ----> p->isulad_io->out -+ ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p); - if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); - goto out; - } -- -- /* p.state.resize_fifo------>runtime.console */ -- ret = connect_to_isulad(ac->p, EXEC_RESIZE, ac->p->state->resize_fifo, recv_fd); -+ // p->isulad_io->resize ----> p->recv_fd -+ ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p); - if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); - goto out; - } - - out: - /* release listen socket at the first time */ -- close_fd(&ac->listen_fd); -- if (ac->p->console_sock_path != NULL) { -- (void)unlink(ac->p->console_sock_path); -- free(ac->p->console_sock_path); -- ac->p->console_sock_path = NULL; -- } -- free(ac); -- if (ret != SHIM_OK) { -- /* -- * When an error occurs during the receiving of the fd , the process -- * exits directly. The files created in the working directory will be -- * deleted by its parent process isulad -- */ -- exit(EXIT_FAILURE); -+ close_fd(&p->listen_fd); -+ if (p->console_sock_path != NULL) { -+ (void)unlink(p->console_sock_path); -+ free(p->console_sock_path); -+ p->console_sock_path = NULL; - } -- return NULL; -+ return ret; - } - --static void *io_epoll_loop(void *data) -+static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) - { -- process_t *p = (process_t *)data; -- int wait_fds = 0; -- struct epoll_event evs[MAX_EVENTS]; -- int i; -+ int i, j; - -- if ((pthread_detach(pthread_self())) != 0) { -- write_message(g_log_fd, ERR_MSG, "detach thread failed"); -- return NULL; -+ for (i = 0; i < 3; i++) { -+ for (j = 0; j < 2; j++) { -+ int ret = fchown(stdio_fd[i][j], uid, gid); -+ if (ret != SHIM_OK) { -+ return SHIM_ERR; -+ } -+ } - } -+ return SHIM_OK; -+} - -- p->io_loop_fd = epoll_create1(EPOLL_CLOEXEC); -- if (p->io_loop_fd < 0) { -- write_message(g_log_fd, ERR_MSG, "epoll create failed:%d", SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -- } -- (void)sem_post(&p->sem_mainloop); -+static void stdio_release(int (*stdio_fd)[2]) -+{ -+ int i, j; - -- for (;;) { -- wait_fds = epoll_wait(p->io_loop_fd, evs, MAX_EVENTS, -1); -- if (wait_fds < 0) { -- if (errno == EINTR) { -- continue; -+ for (i = 0; i < 3; i++) { -+ for (j = 0; j < 2; j++) { -+ if (stdio_fd[i][j] > 0) { -+ close(stdio_fd[i][j]); - } -- _exit(EXIT_FAILURE); - } -+ } -+} - -- for (i = 0; i < wait_fds; i++) { -- io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr; -- sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io); -- } -+static stdio_t *initialize_io(process_t *p) -+{ -+ int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; -+ -+ stdio_t *stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); -+ p->stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); -+ if (p->stdio == NULL || stdio == NULL) { -+ goto failure; -+ } -+ -+ /* don't open resize pipe */ -+ if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || -+ (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { -+ write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); -+ goto failure; - } -+ -+ p->stdio->in = stdio_fd[0][0]; // r -+ stdio->in = stdio_fd[0][1]; // w -+ p->stdio->out = stdio_fd[1][1]; // w -+ stdio->out = stdio_fd[1][0]; // r -+ p->stdio->err = stdio_fd[2][1]; // w -+ stdio->err = stdio_fd[2][0]; // r -+ p->stdio->resize = stdio_fd[3][0]; // r -+ stdio->resize = stdio_fd[3][1]; // w -+ -+ if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { -+ goto failure; -+ } -+ -+ return stdio; -+ -+failure: -+ if (stdio != NULL) { -+ free(stdio); -+ stdio = NULL; -+ } -+ if (p->stdio != NULL) { -+ free(p->stdio); -+ p->stdio = NULL; -+ } -+ stdio_release(stdio_fd); -+ -+ return NULL; - } - - static int new_temp_console_path(process_t *p) -@@ -614,7 +455,7 @@ static int new_temp_console_path(process_t *p) - if (ret != SHIM_OK) { - return SHIM_ERR; - } -- p->console_sock_path = (char *)calloc(1, MAX_CONSOLE_SOCK_LEN + 1); -+ p->console_sock_path = (char *)util_common_calloc_s(MAX_CONSOLE_SOCK_LEN + 1); - if (p->console_sock_path == NULL) { - return SHIM_ERR; - } -@@ -628,12 +469,11 @@ static int new_temp_console_path(process_t *p) - return SHIM_OK; - } - --static int console_init(process_t *p, pthread_t *tid_accept) -+static int console_init(process_t *p, struct epoll_descr *descr) - { - int ret = SHIM_ERR; - int fd = -1; - struct sockaddr_un addr; -- console_accept_t *ac = NULL; - - fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd < 0) { -@@ -657,145 +497,182 @@ static int console_init(process_t *p, pthread_t *tid_accept) - goto failure; - } - -- ac = (console_accept_t *)calloc(1, sizeof(console_accept_t)); -- if (ac == NULL) { -- goto failure; -- } -- ac->p = p; -- ac->listen_fd = fd; -+ p->listen_fd = fd; - -- ret = pthread_create(tid_accept, NULL, task_console_accept, ac); -+ ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p); - if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); - goto failure; - } - - return SHIM_OK; - failure: - close_fd(&fd); -- if (ac != NULL) { -- free(ac); -- ac = NULL; -- } - (void)unlink(p->console_sock_path); - - return SHIM_ERR; - } - --static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) -+static int open_terminal_io(process_t *p, struct epoll_descr *descr) - { -- int i, j; -+ int ret = SHIM_ERR; - -- for (i = 0; i < 3; i++) { -- for (j = 0; j < 2; j++) { -- int ret = fchown(stdio_fd[i][j], uid, gid); -- if (ret != SHIM_OK) { -- return SHIM_ERR; -- } -- } -+ ret = new_temp_console_path(p); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); -+ return SHIM_ERR; - } -- return SHIM_OK; --} -- --static void stdio_release(int (*stdio_fd)[2]) --{ -- int i, j; - -- for (i = 0; i < 3; i++) { -- for (j = 0; j < 2; j++) { -- if (stdio_fd[i][j] > 0) { -- close(stdio_fd[i][j]); -- } -- } -- } -+ /* begin listen from p->console_sock_path */ -+ return console_init(p, descr); - } - --static stdio_t *initialize_io(process_t *p) -+static int open_generic_io(process_t *p, struct epoll_descr *descr) - { -- int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; -- -- stdio_t *stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); -- p->stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); -- if (p->stdio == NULL || stdio == NULL) { -- goto failure; -- } -+ int ret = SHIM_ERR; - -- /* don't open resize pipe */ -- if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || -- (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { -- write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); -- goto failure; -+ // io: in: w out/err: r -+ stdio_t *io = initialize_io(p); -+ if (io == NULL) { -+ return SHIM_ERR; - } -+ p->shim_io = io; - -- p->stdio->in = stdio_fd[0][0]; // r -- stdio->in = stdio_fd[0][1]; // w -- p->stdio->out = stdio_fd[1][1]; // w -- stdio->out = stdio_fd[1][0]; // r -- p->stdio->err = stdio_fd[2][1]; // w -- stdio->err = stdio_fd[2][0]; // r -- p->stdio->resize = stdio_fd[3][0]; // r -- stdio->resize = stdio_fd[3][1]; // w -- -- if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { -- goto failure; -+ // p->isulad_io->in ----> p->shim_io->in -+ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); -+ return SHIM_ERR; - } -- -- return stdio; -- --failure: -- if (stdio != NULL) { -- free(stdio); -- stdio = NULL; -+ // p->shim_io->out ----> p->isulad_io->out -+ ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); -+ return SHIM_ERR; - } -- if (p->stdio != NULL) { -- free(p->stdio); -- p->stdio = NULL; -+ // p->shim_io->err ----> p->isulad_io->err -+ ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); -+ return SHIM_ERR; - } -- stdio_release(stdio_fd); - -- return NULL; -+ return SHIM_OK; - } - --static int open_terminal_io(process_t *p, pthread_t *tid_accept) -+static int set_non_block(int fd) - { -+ int flag = -1; - int ret = SHIM_ERR; - -- ret = new_temp_console_path(p); -- if (ret != SHIM_OK) { -- write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); -+ flag = fcntl(fd, F_GETFL, 0); -+ if (flag < 0) { -+ return SHIM_ERR; -+ } -+ -+ ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK); -+ if (ret != 0) { - return SHIM_ERR; - } - -- /* begin listen and accept fd from p->console_sock_path */ -- return console_init(p, tid_accept); -+ return SHIM_OK; - } - --static int open_generic_io(process_t *p) -+/* -+ std_id: channel type -+ isulad_stdio: one side of the isulad fifo file -+ fd: one side of the shim io pipe -+ --------------------------------------------------------------- -+ | CHANNEL | iSulad Fifo Side | Flow Direction | fd | -+ --------------------------------------------------------------- -+ | STDIN | READ | --> | WRITE | -+ --------------------------------------------------------------- -+ | STDOUT | WRITE | <-- | READ | -+ --------------------------------------------------------------- -+ | STDERR | WRITE | <-- | READ | -+ --------------------------------------------------------------- -+ | RESIZE | READ | --> | WRITE | -+ --------------------------------------------------------------- -+*/ -+static void *io_epoll_loop(void *data) - { -- int ret = SHIM_ERR; -+ int ret = 0; -+ int fd_out = -1; -+ int fd_err = -1; -+ process_t *p = (process_t *)data; -+ struct epoll_descr descr; - -- // io: in: w out/err: r -- stdio_t *io = initialize_io(p); -- if (io == NULL) { -- return SHIM_ERR; -+ ret = epoll_loop_open(&descr); -+ if (ret != 0) { -+ write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno)); -+ exit(EXIT_FAILURE); - } -- p->shim_io = io; -- /* stdin */ -- ret = connect_to_isulad(p, STDID_IN, p->state->isulad_stdin, io->in); -- if (ret != SHIM_OK) { -- return SHIM_ERR; -+ -+ // sync fd: epoll loop will exit when recive sync fd event. -+ ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p); -+ if (ret != 0) { -+ write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); -+ exit(EXIT_FAILURE); - } -- /* stdout */ -- ret = connect_to_isulad(p, STDID_OUT, p->state->isulad_stdout, io->out); -- if (ret != SHIM_OK) { -- return SHIM_ERR; -+ -+ if (p->state->terminal) { -+ ret = open_terminal_io(p, &descr); -+ } else { -+ ret = open_generic_io(p, &descr); - } -- /* stderr */ -- ret = connect_to_isulad(p, STDID_ERR, p->state->isulad_stderr, io->err); - if (ret != SHIM_OK) { -- return SHIM_ERR; -+ write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno)); -+ exit(EXIT_FAILURE); - } - -- return SHIM_OK; -+ (void)sem_post(&p->sem_mainloop); -+ -+ ret = epoll_loop(&descr, -1); -+ if (ret != 0) { -+ write_message(g_log_fd, ERR_MSG, "epoll loop failed"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // in order to avoid data loss, set fd non-block and read it -+ p->block_read = false; -+ if (p->state->terminal) { -+ fd_out = p->recv_fd; -+ } else { -+ fd_out = p->shim_io->out; -+ fd_err = p->shim_io->err; -+ } -+ -+ if (fd_out > 0) { -+ ret = set_non_block(fd_out); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); -+ exit(EXIT_FAILURE); -+ } -+ -+ for (;;) { -+ ret = stdout_cb(fd_out, EPOLLIN, p, &descr); -+ if (ret == EPOLL_LOOP_HANDLE_CLOSE) { -+ break; -+ } -+ } -+ } -+ -+ if (fd_err > 0) { -+ ret = set_non_block(fd_err); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); -+ exit(EXIT_FAILURE); -+ } -+ -+ for (;;) { -+ ret = stderr_cb(fd_err, EPOLLIN, p, &descr); -+ if (ret == EPOLL_LOOP_HANDLE_CLOSE) { -+ break; -+ } -+ } -+ } -+ -+ return NULL; - } - - static void adapt_for_isulad_stdin(process_t *p) -@@ -813,7 +690,7 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s - { - log_terminal *log_term = NULL; - -- log_term = calloc(1, sizeof(log_terminal)); -+ log_term = util_common_calloc_s(sizeof(log_terminal)); - if (log_term == NULL) { - write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal"); - goto clean_out; -@@ -858,11 +735,90 @@ clean_out: - return SHIM_ERR; - } - -+static int open_isulad_fd(int std_id, const char *isulad_stdio, int *fd) -+{ -+ mode_t mode = O_WRONLY; -+ -+ if (std_id == STDID_IN || std_id == EXEC_RESIZE) { -+ mode = O_RDONLY; -+ } -+ -+ if (isulad_stdio != NULL && file_exists(isulad_stdio)) { -+ *(fd) = open_fifo_noblock(isulad_stdio, mode); -+ if (*(fd) < 0) { -+ return -1; -+ } -+ /* open dummy fd to avoid resize epoll hub */ -+ if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { -+ return -1; -+ } -+ } -+ -+ return 0; -+} -+ -+ -+static int init_isulad_stdio(process_t *p) -+{ -+ int ret = SHIM_OK; -+ p->isulad_io = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); -+ if (p->isulad_io == NULL) { -+ return SHIM_ERR; -+ } -+ -+ p->isulad_io->in = -1; -+ p->isulad_io->out = -1; -+ p->isulad_io->err = -1; -+ p->isulad_io->resize = -1; -+ -+ ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin); -+ goto failure; -+ } -+ -+ ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout); -+ goto failure; -+ } -+ -+ ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr); -+ goto failure; -+ } -+ -+ ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize); -+ if (ret != SHIM_OK) { -+ write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo); -+ goto failure; -+ } -+ return SHIM_OK; -+failure: -+ if (p->isulad_io != NULL) { -+ free(p->isulad_io); -+ p->isulad_io = NULL; -+ } -+ if (p->isulad_io->in > 0) { -+ close(p->isulad_io->in); -+ } -+ if (p->isulad_io->out > 0) { -+ close(p->isulad_io->out); -+ } -+ if (p->isulad_io->err > 0) { -+ close(p->isulad_io->err); -+ } -+ if (p->isulad_io->resize > 0) { -+ close(p->isulad_io->resize); -+ } -+ return SHIM_ERR; -+} -+ - process_t *new_process(char *id, char *bundle, char *runtime) - { - shim_client_process_state *p_state; - process_t *p = NULL; -- int i; - int ret; - - p_state = load_process(); -@@ -870,7 +826,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) - return NULL; - } - -- p = (process_t *)calloc(1, sizeof(process_t)); -+ p = (process_t *)util_common_calloc_s(sizeof(process_t)); - if (p == NULL) { - return NULL; - } -@@ -889,16 +845,31 @@ process_t *new_process(char *id, char *bundle, char *runtime) - p->bundle = bundle; - p->runtime = runtime; - p->state = p_state; -- -+ p->block_read = true; - p->console_sock_path = NULL; - p->exit_fd = -1; - p->io_loop_fd = -1; - p->ctr_pid = -1; -+ p->listen_fd = -1; -+ p->recv_fd = -1; - p->stdio = NULL; - p->shim_io = NULL; -+ p->isulad_io = NULL; - -- for (i = 0; i < 3; i++) { -- p->io_threads[i] = NULL; -+ p->sync_fd = eventfd(0, EFD_CLOEXEC); -+ if (p->sync_fd < 0) { -+ write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno)); -+ goto failure; -+ } -+ -+ ret = init_isulad_stdio(p); -+ if (ret != SHIM_OK) { -+ goto failure; -+ } -+ -+ p->buf = util_common_calloc_s(DEFAULT_IO_COPY_BUF + 1); -+ if (p->buf == NULL) { -+ goto failure; - } - - return p; -@@ -909,28 +880,11 @@ failure: - return NULL; - } - --int open_io(process_t *p, pthread_t *tid_accept) -+int process_io_start(process_t *p, pthread_t *tid_epoll) - { - int ret = SHIM_ERR; - -- ret = start_io_copy_threads(p); -- if (ret != SHIM_OK) { -- return SHIM_ERR; -- } -- -- if (p->state->terminal) { -- return open_terminal_io(p, tid_accept); -- } -- -- return open_generic_io(p); --} -- --int process_io_init(process_t *p) --{ -- int ret = SHIM_ERR; -- -- pthread_t tid_loop; -- ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p); -+ ret = pthread_create(tid_epoll, NULL, io_epoll_loop, p); - if (ret != SHIM_OK) { - return SHIM_SYS_ERR(errno); - } -@@ -1091,8 +1045,8 @@ static void exec_runtime_process(process_t *p, int exec_fd) - } - - char *cwd = getcwd(NULL, 0); -- char *log_path = (char *)calloc(1, PATH_MAX); -- char *pid_path = (char *)calloc(1, PATH_MAX); -+ char *log_path = (char *)util_common_calloc_s(PATH_MAX); -+ char *pid_path = (char *)util_common_calloc_s(PATH_MAX); - if (cwd == NULL || log_path == NULL || pid_path == NULL) { - (void)dprintf(exec_fd, "memory error: %s", strerror(errno)); - _exit(EXIT_FAILURE); -@@ -1294,13 +1248,11 @@ static int wait_container_process_with_timeout(process_t *p, const unsigned int - - } - --int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout) -+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout) - { -- int i; - int nret = 0; - int ret = 0; - int status = 0; -- struct timespec ts; - - ret = wait_container_process_with_timeout(p, timeout, &status); - if (ret == SHIM_ERR_TIMEOUT) { -@@ -1324,27 +1276,20 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons - if (p->exit_fd > 0) { - (void)write_nointr(p->exit_fd, &status, sizeof(int)); - } -- // wait for task_console_accept thread termination. In order to make sure that -- // the io_copy connection is established and io_thread is not used by multiple threads. -- if (p->state->terminal) { -- if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { -- write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); -- nret = pthread_join(tid_accept, NULL); -- } else { -- // Set the maximum waiting time to 60s to prevent stuck. -- ts.tv_sec += 60; -- nret = pthread_timedjoin_np(tid_accept, NULL, &ts); -- } - -- if (nret != 0) { -- write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); -+ if (p->sync_fd > 0) { -+ if (eventfd_write(p->sync_fd, 1)) { -+ write_message(g_log_fd, ERR_MSG, "Failed to write sync fd"); - } - } - -- for (i = 0; i < 3; i++) { -- destroy_io_thread(p, i); -+ nret = pthread_join(tid_epoll, NULL); -+ if (nret != 0) { -+ write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread"); - } - -+ close(p->sync_fd); -+ - if (!p->state->exec) { - // if log did not contain "/n", print remaind container log when exit isulad-shim - shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0); -diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h -index 7e3259e8..4cb6462c 100644 ---- a/src/cmd/isulad-shim/process.h -+++ b/src/cmd/isulad-shim/process.h -@@ -40,64 +40,37 @@ typedef struct { - int resize; - } stdio_t; - --typedef struct fd_node { -- int fd; -- bool is_log; -- struct fd_node *next; --} fd_node_t; -- --typedef struct { -- int fd_from; -- fd_node_t *fd_to; -- int id;// 0,1,2,3 -- pthread_mutex_t mutex; --} io_copy_t; -- --typedef struct { -- int epfd; -- pthread_t tid; -- pthread_attr_t attr; -- sem_t sem_thd; -- io_copy_t *ioc; -- bool shutdown; -- bool is_stdin; -- log_terminal *terminal;// just used by stdout and stderr --} io_thread_t; -- - typedef struct process { - char *id; - char *bundle; - char *runtime; -- char *console_sock_path;// pty socket path -+ char *console_sock_path; // pty socket path - int io_loop_fd; - int exit_fd; - int ctr_pid; -+ int sync_fd; -+ int listen_fd; -+ int recv_fd; -+ bool block_read; - log_terminal *terminal; -- stdio_t *stdio;// shim to on runtime side, in:r out/err: w -+ stdio_t *stdio; // shim to on runtime side, in:r out/err: w - stdio_t *shim_io; // shim io on isulad side, in: w out/err: r -- io_thread_t *io_threads[4];// stdin,stdout,stderr,exec_resize -+ stdio_t *isulad_io; // isulad io, in:r out/err: w - shim_client_process_state *state; - sem_t sem_mainloop; -+ char *buf; - } process_t; - --typedef struct { -- int listen_fd; -- process_t *p; --} console_accept_t; -- - typedef struct { - int pid; - int status; - } process_exit_t; - -- -- - process_t* new_process(char *id, char *bundle, char *runtime); - --int open_io(process_t *p, pthread_t *tid_accept); --int process_io_init(process_t *p); -+int process_io_start(process_t *p, pthread_t *tid_epoll); - int create_process(process_t *p); --int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout); -+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout); - - #ifdef __cplusplus - } -diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt -index e5c1cd6e..122538ff 100644 ---- a/test/cmd/isulad-shim/CMakeLists.txt -+++ b/test/cmd/isulad-shim/CMakeLists.txt -@@ -6,6 +6,7 @@ add_executable(${EXE} - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c - ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/mainloop.c - isulad-shim_ut.cc) - - target_include_directories(${EXE} PUBLIC -diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc -index 34ecd452..8c116ac8 100644 ---- a/test/cmd/isulad-shim/isulad-shim_ut.cc -+++ b/test/cmd/isulad-shim/isulad-shim_ut.cc -@@ -11,6 +11,7 @@ - #include - #include - -+#include "mainloop.h" - #include "process.h" - #include "common.h" - --- -2.25.1 - diff --git a/0058-fix-memrealloc-size-error.patch b/0058-fix-memrealloc-size-error.patch deleted file mode 100644 index 9d994d7..0000000 --- a/0058-fix-memrealloc-size-error.patch +++ /dev/null @@ -1,27 +0,0 @@ -From c376da6d86e52bb5eceaa8357b80e81591e05f7b Mon Sep 17 00:00:00 2001 -From: "Neil.wrz" -Date: Thu, 27 Apr 2023 05:20:31 -0700 -Subject: [PATCH 58/58] fix memrealloc size error - -Signed-off-by: Neil.wrz ---- - src/daemon/modules/service/service_container.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c -index 6dc06c83..9960ace1 100644 ---- a/src/daemon/modules/service/service_container.c -+++ b/src/daemon/modules/service/service_container.c -@@ -1580,7 +1580,8 @@ static int do_append_process_exec_env(const char **default_env, defs_process *sp - ret = -1; - goto out; - } -- new_size = (spec->env_len + default_env_len) * sizeof(char *); -+ // new_size = old_size + default_env_len + 1(null as terminator) -+ new_size = (spec->env_len + default_env_len + 1) * sizeof(char *); - old_size = spec->env_len * sizeof(char *); - ret = util_mem_realloc((void **)&temp, new_size, spec->env, old_size); - if (ret != 0) { --- -2.25.1 - diff --git a/0059-convert-struct-lcr-start-exec-request.patch b/0059-convert-struct-lcr-start-exec-request.patch deleted file mode 100644 index 8ef73fa..0000000 --- a/0059-convert-struct-lcr-start-exec-request.patch +++ /dev/null @@ -1,88 +0,0 @@ -From 17abc741624f084d5c2d7cf16acb28109bf0a72a Mon Sep 17 00:00:00 2001 -From: zhangxiaoyu -Date: Fri, 5 May 2023 15:13:52 +0800 -Subject: [PATCH 59/59] convert struct lcr start/exec request - -Signed-off-by: zhangxiaoyu ---- - .../modules/runtime/engines/lcr/lcr_engine.c | 58 +++++++++++++++++-- - 1 file changed, 54 insertions(+), 4 deletions(-) - -diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c -index 2ca12545..34292ce3 100644 ---- a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c -+++ b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c -@@ -102,16 +102,66 @@ static bool lcr_update_container(const char *name, const char *lcrpath, const st - - static bool lcr_start_container(const engine_start_request_t *request) - { -- struct lcr_start_request *lcr_request = (struct lcr_start_request *)request; -+ struct lcr_start_request lcr_request = { 0 }; - -- return g_lcr_start_op(lcr_request); -+ if (g_lcr_start_op == NULL) { -+ ERROR("Not supported start operation"); -+ return false; -+ } -+ -+ if (request == NULL) { -+ ERROR("Empty start request"); -+ return false; -+ } -+ -+ lcr_request.name = request->name; -+ lcr_request.lcrpath = request->lcrpath; -+ lcr_request.logpath = request->logpath; -+ lcr_request.loglevel = request->loglevel; -+ lcr_request.daemonize = request->daemonize; -+ lcr_request.tty = request->tty; -+ lcr_request.open_stdin = request->open_stdin; -+ lcr_request.console_fifos = request->console_fifos; -+ lcr_request.start_timeout = request->start_timeout; -+ lcr_request.container_pidfile = request->container_pidfile; -+ lcr_request.exit_fifo = request->exit_fifo; -+ lcr_request.image_type_oci = request->image_type_oci; -+ -+ return g_lcr_start_op(&lcr_request); - } - - static bool lcr_exec_container(const engine_exec_request_t *request, int *exit_code) - { -- struct lcr_exec_request *lcr_request = (struct lcr_exec_request *)request; -+ struct lcr_exec_request lcr_request = { 0 }; -+ -+ if (g_lcr_exec_op == NULL) { -+ ERROR("Not supported exec operation"); -+ return false; -+ } -+ -+ if (request == NULL) { -+ ERROR("Empty exec request"); -+ return false; -+ } - -- return g_lcr_exec_op(lcr_request, exit_code); -+ lcr_request.name = request->name; -+ lcr_request.lcrpath = request->lcrpath; -+ lcr_request.logpath = request->logpath; -+ lcr_request.loglevel = request->loglevel; -+ lcr_request.console_fifos = request->console_fifos; -+ lcr_request.user = request->user; -+ lcr_request.add_gids = request->add_gids; -+ lcr_request.env = request->env; -+ lcr_request.env_len = request->env_len; -+ lcr_request.args = request->args; -+ lcr_request.args_len = request->args_len; -+ lcr_request.timeout = request->timeout; -+ lcr_request.suffix = request->suffix; -+ lcr_request.tty = request->tty; -+ lcr_request.open_stdin = request->open_stdin; -+ lcr_request.workdir = (char *)request->workdir; -+ -+ return g_lcr_exec_op(&lcr_request, exit_code); - } - - /* --- -2.25.1 - diff --git a/iSulad.spec b/iSulad.spec index 6b90c43..9bea5bb 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ -%global _version 2.0.18 -%global _release 7 +%global _version 2.1.2 +%global _release 2 %global is_systemd 1 %global enable_shimv2 1 %global is_embedded 1 @@ -13,69 +13,9 @@ URL: https://gitee.com/openeuler/iSulad Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz BuildRoot: {_tmppath}/iSulad-%{version} -Patch0001: 0001-add-omitted-musl-adaption-code.patch -Patch0002: 0002-add-cpu-rt-CI.patch -Patch0003: 0003-add-cpu-rt-ut.patch -Patch0004: 0004-add-info-log-when-isulad-shutdown.patch -Patch0005: 0005-create-a-log-file-for-shim-v2-and-remove-10-229-devi.patch -Patch0006: 0006-ensure-isula-exec-inherits-the-config-of-create.patch -Patch0007: 0007-Delete-meaningless-thread-creation-and-ensure-the-ta.patch -Patch0008: 0008-fix-cpu-rt-review-comments.patch -Patch0009: 0009-fix-inspect.sh-failed.patch -Patch0010: 0010-ensure-list-name-is-not-null.patch -Patch0011: 0011-Bugfix-in-config-and-executor.patch -Patch0012: 0012-fix-isula-cpu-rt-CI.patch -Patch0013: 0013-add-CRI-ContainerStats-Service.patch -Patch0014: 0014-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch -Patch0015: 0015-fix-cpu-rt-disable-after-reboot-machine.patch -Patch0016: 0016-fix-code-style.patch -Patch0017: 0017-add-retry-for-read-write.patch -Patch0018: 0018-add-crictl-timeout-and-sync-for-CI.patch -Patch0019: 0019-unlock-m_podsLock-if-new-failed.patch -Patch0020: 0020-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch -Patch0021: 0021-support-pull-image-with-digest.patch -Patch0022: 0022-isulad-shim-support-execSync-with-timeout.patch -Patch0023: 0023-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch -Patch0024: 0024-adapt-to-repo-of-openeuler-url-changed.patch -Patch0025: 0025-modify-sleep-time.patch -Patch0026: 0026-change-goto-branch.patch -Patch0027: 0027-modifying-cpurt-file-permissions.patch -Patch0028: 0028-add-design-docs-for-cri-manager.patch -Patch0029: 0029-improve-check-of-process-failure.patch -Patch0030: 0030-support-isula-update-when-runtime-is-runc.patch -Patch0031: 0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch -Patch0032: 0032-add-CRI-container-design-doc.patch -Patch0033: 0033-fix-util_getgrent_r-overflow.patch -Patch0034: 0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch -Patch0035: 0035-fix-inspect-data-memleak.patch -Patch0036: 0036-containers-in-same-sandbox-should-have-same-process-.patch -Patch0037: 0037-clean-container-process-after-execSync-timeout-exit.patch -Patch0038: 0038-support-to-config-selinux-label-in-cri.patch -Patch0039: 0039-add-files_limit-to-oci-spec.patch -Patch0040: 0040-support-setting-pod-to-privilege.patch -Patch0041: 0041-add-hugepage_limit.patch -Patch0042: 0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch -Patch0043: 0043-isulad-shim-fix-log-loss-bug.patch -Patch0044: 0044-remove-unused-func.patch -Patch0045: 0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch -Patch0046: 0046-free-timeout-when-shim_create-finished.patch -Patch0047: 0047-clean-isulad-shim-compile-relies.patch -Patch0048: 0048-remote-layer-store-demo.patch -Patch0049: 0049-add-ci-for-remote-ro.patch -Patch0050: 0050-fix-compile-error-when-not-enable-remote-ro.patch -Patch0051: 0051-CI-not-enable-remote-ro-for-ut.patch -Patch0052: 0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch -Patch0053: 0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch -Patch0054: 0054-refactor-remote-ro-code.patch -Patch0055: 0055-bugfix-when-refresh-can-t-load-or-pull-images.patch -Patch0056: 0056-remove-unused-headers.patch -Patch0057: 0057-change-isulad-shim-epoll-struct.patch -Patch0058: 0058-fix-memrealloc-size-error.patch -Patch0059: 0059-convert-struct-lcr-start-exec-request.patch - %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) -Provides: libisula.so()(64bit) +Provides: libisula_client.so()(64bit) Provides: libisulad_img.so()(64bit) Provides: libisulad_tools.so()(64bit) %endif @@ -100,13 +40,11 @@ Requires: sqlite BuildRequires: gtest-devel gmock-devel %endif -%define lcrver_lower 2.0.9-0 -%define lcrver_upper 2.0.10-0 -%define clibcniver_lower 2.0.7-0 -%define clibcniver_upper 2.0.8-0 +%define lcrver_lower 2.1.1-0 +%define lcrver_upper 2.1.2-0 BuildRequires: lcr-devel > %{lcrver_lower} lcr-devel < %{lcrver_upper} -BuildRequires: clibcni-devel > %{clibcniver_lower} clibcni-devel < %{clibcniver_upper} +BuildRequires: libisula-devel > %{lcrver_lower} libisula-devel < %{lcrver_upper} BuildRequires: cmake gcc-c++ yajl-devel lxc lxc-devel BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel BuildRequires: libcurl libcurl-devel libarchive-devel device-mapper-devel @@ -118,8 +56,8 @@ BuildRequires: lib-shim-v2 lib-shim-v2-devel %endif +Requires: libisula > %{lcrver_lower} libisula < %{lcrver_upper} Requires: lcr > %{lcrver_lower} lcr < %{lcrver_upper} -Requires: clibcni > %{clibcniver_lower} clibcni < %{clibcniver_upper} Requires: grpc protobuf lxc Requires: libcurl Requires: http-parser libseccomp @@ -141,15 +79,15 @@ mkdir -p build cd build %if 0%{?enable_shimv2} %if %{defined openeuler} -%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON ../ +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ../ %else -%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON ../ +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ../ %endif %else %if %{defined openeuler} -%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON ../ +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ../ %else -%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr ../ +%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_GRPC_REMOTE_CONNECT=ON ../ %endif %endif %make_build @@ -165,7 +103,7 @@ ctest -E "registry_images_ut|volume_ut" rm -rf %{buildroot} cd build install -d $RPM_BUILD_ROOT/%{_libdir} -install -m 0755 ./src/libisula.so %{buildroot}/%{_libdir}/libisula.so +install -m 0755 ./src/libisula_client.so %{buildroot}/%{_libdir}/libisula_client.so install -m 0755 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so install -m 0755 ./src/libisulad_tools.so %{buildroot}/%{_libdir}/libisulad_tools.so install -m 0755 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so @@ -316,6 +254,18 @@ fi %endif %changelog +* Fri May 12 2023 zhangxiaoyu - 2.1.2-2 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix remote grpc macro + +* Thu May 11 2023 zhangxiaoyu - 2.1.2-1 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade to v2.1.2 + * Fri May 05 2023 wangrunze - 2.0.18-7 - Type: bugfix - ID: NA diff --git a/v2.0.18.tar.gz b/v2.0.18.tar.gz deleted file mode 100644 index c116ce8..0000000 Binary files a/v2.0.18.tar.gz and /dev/null differ diff --git a/v2.1.2.tar.gz b/v2.1.2.tar.gz new file mode 100644 index 0000000..05d616b Binary files /dev/null and b/v2.1.2.tar.gz differ