From 22ae3de54780a4700f345559ab0adec2288f134a Mon Sep 17 00:00:00 2001 From: zhangxiaoyu Date: Thu, 4 May 2023 16:44:20 +0800 Subject: [PATCH] upgrade from upstream Signed-off-by: zhangxiaoyu (cherry picked from commit 4b64b19964f903cdbea9493ab2eaeace8c823cad) --- 0047-clean-isulad-shim-compile-relies.patch | 347 +++ 0048-remote-layer-store-demo.patch | 2290 +++++++++++++++++ 0049-add-ci-for-remote-ro.patch | 119 + ...pile-error-when-not-enable-remote-ro.patch | 62 + 0051-CI-not-enable-remote-ro-for-ut.patch | 41 + ...-try-add-or-remove-image-layer-twice.patch | 99 + ...lete-layers-under-dir-overlay-layers.patch | 34 + 0054-refactor-remote-ro-code.patch | 2089 +++++++++++++++ ...en-refresh-can-t-load-or-pull-images.patch | 319 +++ 0056-remove-unused-headers.patch | 40 + 0057-change-isulad-shim-epoll-struct.patch | 1793 +++++++++++++ iSulad.spec | 19 +- 12 files changed, 7251 insertions(+), 1 deletion(-) create mode 100644 0047-clean-isulad-shim-compile-relies.patch create mode 100644 0048-remote-layer-store-demo.patch create mode 100644 0049-add-ci-for-remote-ro.patch create mode 100644 0050-fix-compile-error-when-not-enable-remote-ro.patch create mode 100644 0051-CI-not-enable-remote-ro-for-ut.patch create mode 100644 0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch create mode 100644 0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch create mode 100644 0054-refactor-remote-ro-code.patch create mode 100644 0055-bugfix-when-refresh-can-t-load-or-pull-images.patch create mode 100644 0056-remove-unused-headers.patch create mode 100644 0057-change-isulad-shim-epoll-struct.patch diff --git a/0047-clean-isulad-shim-compile-relies.patch b/0047-clean-isulad-shim-compile-relies.patch new file mode 100644 index 0000000..56ad8be --- /dev/null +++ b/0047-clean-isulad-shim-compile-relies.patch @@ -0,0 +1,347 @@ +From fde1c406a837b849f1182d8943f1f942088b608d Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sun, 23 Apr 2023 15:01:58 +0800 +Subject: [PATCH 47/56] clean isulad shim compile relies + +Signed-off-by: zhongtao +--- + src/CMakeLists.txt | 12 +- + src/cmd/isulad-shim/common.c | 179 ++++++++++++++++++++++++++++ + src/cmd/isulad-shim/common.h | 49 ++++++++ + src/cmd/isulad-shim/process.c | 3 - + test/cmd/isulad-shim/CMakeLists.txt | 13 -- + 5 files changed, 238 insertions(+), 18 deletions(-) + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index f3dd3c19..02d7b13f 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -113,8 +113,16 @@ endif() + add_executable(isulad-shim + ${ISULAD_SHIM_SRCS} + ) +-target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS}) +-target_link_libraries(isulad-shim libisulad_tools) ++target_include_directories(isulad-shim PUBLIC ++ ${ISULAD_SHIM_INCS} ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ ${COMMON_INCS} ++ ${CMAKE_BINARY_DIR}/conf ++ ${CHECKED_INCLUDE_DIRS} ++ ${SHARED_INCS} ++ ${ISULA_LIBUTILS_INCLUDE_DIR} ++ ) ++target_link_libraries(isulad-shim ${ISULA_LIBUTILS_LIBRARY}) + if (ANDROID OR MUSL) + target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY}) + else() +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index f188da1e..e1ca96e1 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + + int set_fd_no_inherited(int fd) + { +@@ -346,3 +347,181 @@ int shim_util_safe_uint64(const char *numstr, uint64_t *converted) + return 0; + } + ++void util_usleep_nointerupt(unsigned long usec) ++{ ++#define SECOND_TO_USECOND_MUTIPLE 1000000 ++ int ret = 0; ++ struct timespec request = { 0 }; ++ struct timespec remain = { 0 }; ++ if (usec == 0) { ++ return; ++ } ++ ++ request.tv_sec = (time_t)(usec / SECOND_TO_USECOND_MUTIPLE); ++ request.tv_nsec = (long)((usec % SECOND_TO_USECOND_MUTIPLE) * 1000); ++ ++ do { ++ ret = nanosleep(&request, &remain); ++ request = remain; ++ } while (ret == -1 && errno == EINTR); ++} ++ ++void *util_smart_calloc_s(size_t unit_size, size_t count) ++{ ++ if (unit_size == 0) { ++ return NULL; ++ } ++ ++ if (count > (MAX_MEMORY_SIZE / unit_size)) { ++ return NULL; ++ } ++ ++ return calloc(count, unit_size); ++} ++ ++size_t util_array_len(const char **array) ++{ ++ const char **pos; ++ size_t len = 0; ++ ++ for (pos = array; pos != NULL && *pos != NULL; pos++) { ++ len++; ++ } ++ ++ return len; ++} ++ ++void util_free_array(char **array) ++{ ++ char **p; ++ ++ for (p = array; p != NULL && *p != NULL; p++) { ++ UTIL_FREE_AND_SET_NULL(*p); ++ } ++ free(array); ++} ++ ++int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment) ++{ ++ size_t add_capacity; ++ char **add_array = NULL; ++ ++ if (orig_array == NULL || orig_capacity == NULL || increment == 0) { ++ return -1; ++ } ++ ++ if (((*orig_array) == NULL) || ((*orig_capacity) == 0)) { ++ UTIL_FREE_AND_SET_NULL(*orig_array); ++ *orig_capacity = 0; ++ } ++ ++ add_capacity = *orig_capacity; ++ while (size + 1 > add_capacity) { ++ add_capacity += increment; ++ } ++ if (add_capacity != *orig_capacity) { ++ add_array = util_smart_calloc_s(sizeof(void *), add_capacity); ++ if (add_array == NULL) { ++ return -1; ++ } ++ if (*orig_array != NULL) { ++ (void)memcpy(add_array, *orig_array, *orig_capacity * sizeof(void *)); ++ UTIL_FREE_AND_SET_NULL(*orig_array); ++ } ++ ++ *orig_array = add_array; ++ *orig_capacity = add_capacity; ++ } ++ ++ return 0; ++} ++ ++char *util_strdup_s(const char *src) ++{ ++ char *dst = NULL; ++ ++ if (src == NULL) { ++ return NULL; ++ } ++ ++ dst = strdup(src); ++ if (dst == NULL) { ++ abort(); ++ } ++ ++ return dst; ++} ++ ++static char **make_empty_array() ++{ ++ char **res_array = NULL; ++ ++ res_array = calloc(2, sizeof(char *)); ++ if (res_array == NULL) { ++ return NULL; ++ } ++ res_array[0] = util_strdup_s(""); ++ return res_array; ++} ++ ++static char **util_shrink_array(char **orig_array, size_t new_size) ++{ ++ char **new_array = NULL; ++ size_t i = 0; ++ ++ if (new_size == 0) { ++ return orig_array; ++ } ++ new_array = util_smart_calloc_s(sizeof(char *), new_size); ++ if (new_array == NULL) { ++ return orig_array; ++ } ++ ++ for (i = 0; i < new_size; i++) { ++ new_array[i] = orig_array[i]; ++ } ++ free(orig_array); ++ return new_array; ++} ++ ++char **util_string_split_multi(const char *src_str, char delim) ++{ ++ int ret, tmp_errno; ++ char *token = NULL; ++ char *cur = NULL; ++ char **res_array = NULL; ++ char deli[2] = { delim, '\0' }; ++ size_t count = 0; ++ size_t capacity = 0; ++ char *tmpstr = NULL; ++ ++ if (src_str == NULL) { ++ return NULL; ++ } ++ ++ if (src_str[0] == '\0') { ++ return make_empty_array(); ++ } ++ ++ tmpstr = util_strdup_s(src_str); ++ cur = tmpstr; ++ token = strsep(&cur, deli); ++ while (token != NULL) { ++ ret = util_grow_array(&res_array, &capacity, count + 1, 16); ++ if (ret < 0) { ++ goto err_out; ++ } ++ res_array[count] = util_strdup_s(token); ++ count++; ++ token = strsep(&cur, deli); ++ } ++ free(tmpstr); ++ return util_shrink_array(res_array, count + 1); ++ ++err_out: ++ tmp_errno = errno; ++ free(tmpstr); ++ util_free_array(res_array); ++ errno = tmp_errno; ++ return NULL; ++} +\ No newline at end of file +diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h +index 91808295..3de16ace 100644 +--- a/src/cmd/isulad-shim/common.h ++++ b/src/cmd/isulad-shim/common.h +@@ -58,6 +58,43 @@ extern "C" { + #define CONTAINER_ACTION_REBOOT 129 + #define CONTAINER_ACTION_SHUTDOWN 130 + ++ ++void util_usleep_nointerupt(unsigned long usec); ++/** ++ * retry_cnt: max count of call cb; ++ * interval_us: how many us to sleep, after call cb; ++ * cb: retry call function; ++ * return: ++ * 0 is cb successful at least once; ++ * 1 is all cb are failure; ++*/ ++#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ ++ size_t i = 0; \ ++ for(; i < retry_cnt; i++) { \ ++ ret = cb(##__VA_ARGS__); \ ++ if (ret == 0) { \ ++ break; \ ++ } \ ++ util_usleep_nointerupt(interval_us); \ ++ } \ ++ } while(0) ++ ++#define UTIL_FREE_AND_SET_NULL(p) \ ++ do { \ ++ if ((p) != NULL) { \ ++ free((void *)(p)); \ ++ (p) = NULL; \ ++ } \ ++ } while (0) ++ ++#if __WORDSIZE == 64 ++// current max user memory for 64-machine is 2^47 B ++#define MAX_MEMORY_SIZE ((size_t)1 << 47) ++#else ++// current max user memory for 32-machine is 2^31 B ++#define MAX_MEMORY_SIZE ((size_t)1 << 31) ++#endif ++ + ssize_t read_nointr(int fd, void *buf, size_t count); + ssize_t write_nointr(int fd, const void *buf, size_t count); + +@@ -79,6 +116,18 @@ int open_no_inherit(const char *path, int flag, mode_t mode); + + int shim_util_safe_uint64(const char *numstr, uint64_t *converted); + ++void *util_smart_calloc_s(size_t unit_size, size_t count); ++ ++size_t util_array_len(const char **array); ++ ++void util_free_array(char **array); ++ ++int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment); ++ ++char *util_strdup_s(const char *src); ++ ++char **util_string_split_multi(const char *src_str, char delim); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index a676e7ce..a5e0bd39 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -37,9 +37,6 @@ + + #include "common.h" + #include "terminal.h" +-#include "utils_array.h" +-#include "utils_string.h" +-#include "utils.h" + + #define MAX_EVENTS 100 + #define DEFAULT_IO_COPY_BUF (16 * 1024) +diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt +index dc293f6d..e5c1cd6e 100644 +--- a/test/cmd/isulad-shim/CMakeLists.txt ++++ b/test/cmd/isulad-shim/CMakeLists.txt +@@ -6,26 +6,13 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c + isulad-shim_ut.cc) + + target_include_directories(${EXE} PUBLIC + ${GTEST_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256 +- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils + ${CMAKE_BINARY_DIR}/conf + ) + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +-- +2.25.1 + diff --git a/0048-remote-layer-store-demo.patch b/0048-remote-layer-store-demo.patch new file mode 100644 index 0000000..c5c93c5 --- /dev/null +++ b/0048-remote-layer-store-demo.patch @@ -0,0 +1,2290 @@ +From fdd49ec5b0cea74acfb05482045042280c2f2eaa Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 9 Jan 2023 17:12:53 -0800 +Subject: [PATCH 48/56] remote layer store demo + +Signed-off-by: Neil +--- + cmake/options.cmake | 6 + + src/daemon/config/isulad_config.c | 1 + + src/daemon/modules/image/oci/oci_image.c | 4 + + .../modules/image/oci/storage/CMakeLists.txt | 6 + + .../storage/image_store/image_remote_impl.c | 173 +++++++++ + .../oci/storage/image_store/image_store.c | 76 +++- + .../oci/storage/image_store/image_store.h | 11 + + .../storage/layer_store/graphdriver/driver.c | 4 +- + .../storage/layer_store/graphdriver/driver.h | 3 + + .../graphdriver/overlay2/driver_overlay2.c | 158 +++++++- + .../graphdriver/overlay2/driver_overlay2.h | 8 + + .../overlay2/overlay_remote_impl.c | 282 ++++++++++++++ + .../storage/layer_store/layer_remote_impl.c | 219 +++++++++++ + .../oci/storage/layer_store/layer_store.c | 200 +++++++++- + .../oci/storage/layer_store/layer_store.h | 11 + + .../remote_layer_support/CMakeLists.txt | 12 + + .../remote_layer_support/remote_support.c | 122 ++++++ + .../remote_layer_support/remote_support.h | 58 +++ + .../ro_symlink_maintain.c | 347 ++++++++++++++++++ + .../ro_symlink_maintain.h | 52 +++ + .../modules/image/oci/storage/storage.c | 11 +- + .../modules/image/oci/storage/storage.h | 3 + + 22 files changed, 1749 insertions(+), 18 deletions(-) + create mode 100644 src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index a6905c17..701082dd 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -121,6 +121,12 @@ if (DISABLE_CLEANUP STREQUAL "ON") + message("${Green}-- Disable cleanup module") + endif() + ++option(ENABLE_REMOTE_LAYER_STORE "enable remote layer store" OFF) ++if (ENABLE_REMOTE_LAYER_STORE STREQUAL "ON") ++ add_definitions(-DENABLE_REMOTE_LAYER_STORE) ++ message("${Green}-- Enable remote layer store") ++endif() ++ + option(MUSL "available for musl" OFF) + if (MUSL) + add_definitions(-D__MUSL__) +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index ad82aeff..996917c4 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1515,6 +1515,7 @@ static int merge_authorization_conf_into_global(struct service_arguments *args, + static int merge_storage_conf_into_global(struct service_arguments *args, isulad_daemon_configs *tmp_json_confs) + { + override_string_value(&args->json_confs->storage_driver, &tmp_json_confs->storage_driver); ++ args->json_confs->storage_enable_remote_layer = tmp_json_confs->storage_enable_remote_layer; + + if (string_array_append(tmp_json_confs->storage_opts, tmp_json_confs->storage_opts_len, + &(args->json_confs->storage_opts_len), &(args->json_confs->storage_opts)) != 0) { +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index 0fc0b7fe..e4442a79 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -214,6 +214,10 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++#endif ++ + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, + &storage_opts->driver_opts_len) != 0) { + ERROR("Failed to get storage storage opts"); +diff --git a/src/daemon/modules/image/oci/storage/CMakeLists.txt b/src/daemon/modules/image/oci/storage/CMakeLists.txt +index 06c2f378..24b71450 100644 +--- a/src/daemon/modules/image/oci/storage/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/CMakeLists.txt +@@ -3,12 +3,17 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_storage_srcs) + add_subdirectory(image_store) + add_subdirectory(layer_store) + add_subdirectory(rootfs_store) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() ++ + + set(STORAGE_SRCS + ${local_storage_srcs} + ${IMAGE_STORE_SRCS} + ${LAYER_STORE_SRCS} + ${ROOTFS_STORE_SRCS} ++ ${REMOTE_LAYER_SUPPORT_SRCS} + PARENT_SCOPE + ) + +@@ -17,5 +22,6 @@ set(STORAGE_INCS + ${IMAGE_STORE_INCS} + ${LAYER_STORE_INCS} + ${ROOTFS_STORE_INCS} ++ ${REMOTE_LAYER_SUPPORT_INCS} + PARENT_SCOPE + ) +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +new file mode 100644 +index 00000000..20da8116 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +@@ -0,0 +1,173 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote image store functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "image_store.h" ++ ++#include ++#include ++ ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "map.h" ++#include "utils_file.h" ++#include "utils.h" ++#include "layer_store.h" ++#include "utils_array.h" ++ ++struct remote_image_data { ++ const char *image_home; ++}; ++ ++static map_t *image_byid_old = NULL; ++static map_t *image_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->image_home = remote_home; ++ image_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ image_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(image_byid_old); ++ map_free(image_byid_new); ++ ++ free(data); ++ return; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ int ret = 0; ++ int nret; ++ char **image_dirs = NULL; ++ size_t image_dirs_num = 0; ++ size_t i; ++ char *id_patten = "^[a-f0-9]{64}$"; ++ char image_path[PATH_MAX] = { 0x00 }; ++ bool exist = true; ++ struct remote_image_data *img_data = (struct remote_image_data *)data; ++ ++ ret = util_list_all_subdir(img_data->image_home, &image_dirs); ++ if (ret != 0) { ++ ERROR("Failed to get images directory"); ++ goto out; ++ } ++ image_dirs_num = util_array_len((const char **)image_dirs); ++ ++ for (i = 0; i < image_dirs_num; i++) { ++ bool valid_v1_image = false; ++ ++ if (util_reg_match(id_patten, image_dirs[i]) != 0) { ++ DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", ++ image_dirs[i]); ++ continue; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", img_data->image_home, image_dirs[i]); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ continue; ++ } ++ ++ if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ ERROR("Failed to validate manifest schema version 1 format"); ++ continue; ++ } ++ ++ if (!valid_v1_image) { ++ map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); ++ } ++ } ++ ++out: ++ util_free_array(image_dirs); ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ char *top_layer = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ int ret = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(image_byid_old, image_byid_new); ++ array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ top_layer = get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { ++ ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); ++ continue; ++ } ++ ++ if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ ERROR("Failed to load image into memrory: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ ERROR("Failed to remove remote memory store"); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = image_byid_old; ++ image_byid_old = image_byid_new; ++ image_byid_new = tmp_map; ++ empty_map(image_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ free(top_layer); ++ ++ return ret; ++} ++ ++remote_support *image_store_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index 7e1a5373..c6836e46 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -50,6 +50,9 @@ + #include "image_type.h" + #include "linked_list.h" + #include "utils_verify.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + // the name of the big data item whose contents we consider useful for computing a "digest" of the + // image, by which we can locate the image later. +@@ -3099,7 +3102,7 @@ out: + return ret; + } + +-static int validate_manifest_schema_version_1(const char *path, bool *valid) ++int validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3643,6 +3646,10 @@ int image_store_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ remote_image_init(g_image_store->dir); ++#endif ++ + out: + if (ret != 0) { + free_image_store(g_image_store); +@@ -3651,3 +3658,70 @@ out: + free(root_dir); + return ret; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int append_image_by_directory_with_lock(const char *id) ++{ ++ int ret = 0; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return -1; ++ } ++ ++ ret = append_image_by_directory(image_path); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++int remove_image_from_memory_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!image_store_lock(EXCLUSIVE)) { ++ ERROR("Failed to lock remote image store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_image_from_memory(id); ++ image_store_unlock(); ++ ++ return ret; ++} ++ ++char *get_top_layer_from_json(const char *img_id) ++{ ++ ++ char *ret = NULL; ++ int nret = 0; ++ char image_path[PATH_MAX] = { 0x00 }; ++ storage_image *im = NULL; ++ parser_error err = NULL; ++ ++ nret = snprintf(image_path, sizeof(image_path), "%s/%s/%s", g_image_store->dir, img_id, IMAGE_JSON); ++ if (nret < 0 || (size_t)nret >= sizeof(image_path)) { ++ ERROR("Failed to get image path"); ++ return NULL; ++ } ++ ++ im = storage_image_parse_file(image_path, NULL, &err); ++ if (im == NULL) { ++ ERROR("Failed to parse images path: %s", err); ++ return NULL; ++ } ++ ++ ret = util_strdup_s(im->layer); ++ free_storage_image(im); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index edd4fa2c..c3cb50e3 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,6 +28,9 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct storage_module_init_options; + +@@ -108,6 +111,14 @@ void image_store_free(); + + imagetool_image_summary *image_store_get_image_summary(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *image_store_impl_remote_support(); ++int validate_manifest_schema_version_1(const char *path, bool *valid); ++int append_image_by_directory_with_lock(const char *image_dir); ++int remove_image_from_memory_with_lock(const char *id); ++char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +index d3b5209a..b83c63b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c +@@ -141,7 +141,9 @@ int graphdriver_init(const struct storage_module_init_options *opts) + ret = -1; + goto out; + } +- ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_drivers[i].enable_remote_layer = opts->enable_remote_layer; ++#endif + if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts, + opts->driver_opts_len) != 0) { + ret = -1; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +index 7faf70c8..acd847cc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h +@@ -90,6 +90,9 @@ struct graphdriver { + bool support_dtype; + + bool support_quota; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + struct pquota_control *quota_ctrl; + + // options for overlay2 +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +index 330c230a..eedbeef2 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +@@ -12,6 +12,7 @@ + * Create: 2020-04-02 + * Description: provide overlay2 function definition + ******************************************************************************/ ++#define _GNU_SOURCE + #include "driver_overlay2.h" + + #include +@@ -44,6 +45,9 @@ + #include "utils_timestamp.h" + #include "selinux_label.h" + #include "err_msg.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + struct io_read_wrapper; + +@@ -343,6 +347,13 @@ int overlay2_init(struct graphdriver *driver, const char *driver_home, const cha + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer && remote_overlay_init(driver_home) != 0) { ++ ERROR("Failed to init overlay remote"); ++ return -1; ++ } ++#endif ++ + driver->home = util_strdup_s(driver_home); + + root_dir = util_path_dir(driver_home); +@@ -423,7 +434,7 @@ static int mk_diff_directory(const char *layer_dir) + int ret = 0; + char *diff_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + diff_dir = util_path_join(layer_dir, OVERLAY_LAYER_DIFF); +@@ -538,7 +549,7 @@ static int mk_work_directory(const char *layer_dir) + int ret = 0; + char *work_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + work_dir = util_path_join(layer_dir, OVERLAY_LAYER_WORK); +@@ -575,7 +586,7 @@ static int mk_merged_directory(const char *layer_dir) + int ret = 0; + char *merged_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + merged_dir = util_path_join(layer_dir, OVERLAY_LAYER_MERGED); +@@ -852,13 +863,115 @@ out: + return ret; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static int do_create_remote_ro(const char *id, const char *parent, const struct graphdriver *driver, ++ const struct driver_create_opts *create_opts) ++{ ++ int ret = 0; ++ int get_err = 0; ++ char *ro_symlink = NULL; ++ char *ro_home = NULL; ++ char *layer_dir = NULL; ++#ifdef ENABLE_USERNS_REMAP ++ char *userns_remap = conf_get_isulad_userns_remap(); ++#endif ++ ++ ro_home = util_path_join(driver->home, OVERLAY_RO_DIR); ++ if (ro_home == NULL) { ++ ERROR("Failed to join ro_home"); ++ ret = -1; ++ goto out; ++ } ++ ++ layer_dir = util_path_join(ro_home, id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer_dir"); ++ ret = -1; ++ goto out; ++ } ++ ++ ro_symlink = util_path_join(driver->home, id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro_symlink"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join layer dir:%s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (check_parent_valid(parent, driver) != 0) { ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_mkdir_p(layer_dir, 0700) != 0) { ++ ERROR("Unable to create layer directory %s.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ // mk symbol link ++ if (symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ ++#ifdef ENABLE_USERNS_REMAP ++ if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) { ++ ERROR("Unable to change directory %s owner for user remap.", layer_dir); ++ ret = -1; ++ goto out; ++ } ++#endif ++ ++ if (create_opts->storage_opt != NULL && create_opts->storage_opt->len != 0) { ++ if (set_layer_quota(layer_dir, create_opts->storage_opt, driver) != 0) { ++ ERROR("Unable to set layer quota %s", layer_dir); ++ ret = -1; ++ goto err_out; ++ } ++ } ++ ++ if (mk_sub_directories(id, parent, layer_dir, driver->home) != 0) { ++ ret = -1; ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", layer_dir); ++ } ++ ++ // to remove a file ++ if (util_fileself_exists(ro_symlink) && !util_force_remove_file(ro_symlink, &get_err)) { ++ ERROR("Failed to remove symbol link %s", ro_symlink); ++ } ++ ++out: ++ free(layer_dir); ++ free(ro_home); ++ free(ro_symlink); ++#ifdef ENABLE_USERNS_REMAP ++ free(userns_remap); ++#endif ++ return ret; ++} ++#endif ++ + static int do_create(const char *id, const char *parent, const struct graphdriver *driver, + const struct driver_create_opts *create_opts) + { + int ret = 0; + char *layer_dir = NULL; + #ifdef ENABLE_USERNS_REMAP +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + + layer_dir = util_path_join(driver->home, id); +@@ -1002,7 +1115,15 @@ int overlay2_create_ro(const char *id, const char *parent, const struct graphdri + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (driver->enable_remote_layer) { ++ return do_create_remote_ro(id, parent, driver, create_opts); ++ } else { ++ return do_create(id, parent, driver, create_opts); ++ } ++#else + return do_create(id, parent, driver, create_opts); ++#endif + } + + static char *read_layer_link_file(const char *layer_dir) +@@ -1047,6 +1168,9 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + char *link_id = NULL; + char link_path[PATH_MAX] = { 0 }; + char clean_path[PATH_MAX] = { 0 }; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ struct stat stat_buf; ++#endif + + if (id == NULL || driver == NULL) { + ERROR("Invalid input arguments"); +@@ -1079,11 +1203,34 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver) + } + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (lstat(layer_dir, &stat_buf) < 0) { ++ SYSERROR("Failed to lstat path: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ ++ if (driver->enable_remote_layer && S_ISLNK(stat_buf.st_mode)) { ++ // jusdge if the dir is symlink? ++ if (remote_overlay_remove_ro_dir(id) != 0) { ++ ERROR("Failed to delete symlink to layer dir: %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (util_recursive_rmdir(layer_dir, 0) != 0) { ++ SYSERROR("Failed to remove layer directory %s", layer_dir); ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (util_recursive_rmdir(layer_dir, 0) != 0) { + SYSERROR("Failed to remove layer directory %s", layer_dir); + ret = -1; + goto out; + } ++#endif + + out: + free(layer_dir); +@@ -1747,7 +1894,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const + int ret = 0; + #ifdef ENABLE_USERNS_REMAP + unsigned int size = 0; +- char* userns_remap = conf_get_isulad_userns_remap(); ++ char *userns_remap = conf_get_isulad_userns_remap(); + #endif + char *layer_dir = NULL; + char *layer_diff = NULL; +@@ -2166,3 +2313,4 @@ out: + free(layer_diff); + return ret; + } ++ +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index e14271b1..5c1d93fb 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,6 +22,9 @@ + #include + + #include "driver.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -68,6 +71,11 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *overlay_driver_impl_remote_support(void); ++bool overlay_remote_layer_valid(const char *layer_id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +new file mode 100644 +index 00000000..a674a00f +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +@@ -0,0 +1,282 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: provide remote implementation for driver overlay ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "driver_overlay2.h" ++ ++#include ++ ++#include "map.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "isula_libutils/log.h" ++#include "utils.h" ++#include "utils_array.h" ++#include "utils_file.h" ++#include "path.h" ++ ++#define OVERLAY_LINK_DIR "l" ++#define OVERLAY_LAYER_LINK "link" ++ ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; ++ ++static map_t *overlay_byid_old = NULL; ++static map_t *overlay_byid_new = NULL; ++ ++static void *remote_support_create(const char *remote_home, const char *remote_ro) ++{ ++ struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->overlay_home = remote_home; ++ data->overlay_ro = remote_ro; ++ overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++} ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(overlay_byid_old); ++ map_free(overlay_byid_new); ++ free(data); ++} ++ ++static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ if (!map_insert(overlay_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_overlay_data *remote_data = data; ++ return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++} ++ ++static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) ++{ ++ int ret = 0; ++ int nret = 0; ++ char target_path[PATH_MAX] = { 0 }; ++ char link_path[PATH_MAX] = { 0 }; ++ char clean_path[PATH_MAX] = { 0 }; ++ ++ nret = snprintf(target_path, PATH_MAX, "../%s/diff", id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get target path %s", id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = snprintf(link_path, PATH_MAX, "%s/%s/%s", driver_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Failed to get link path %s", link_id); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("failed to get clean path %s", link_path); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_fileself_exists(clean_path) && util_path_remove(clean_path) != 0) { ++ ERROR("failed to remove old symbol link"); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = symlink(target_path, clean_path); ++ if (nret < 0) { ++ SYSERROR("Failed to create symlink from \"%s\" to \"%s\"", clean_path, target_path); ++ ret = -1; ++ goto out; ++ } ++ ++out: ++ return ret; ++} ++ ++static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++} ++ ++static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ char *link_file = NULL; ++ char *diff_symlink = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->overlay_home, overlay_id); ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ layer_dir = util_path_join(data->overlay_ro, overlay_id); ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", overlay_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // add RO symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ // maintain link ++ // try read link file in layer_dir ++ // mk symlink between ro_symlink ++ link_file = util_path_join(layer_dir, OVERLAY_LAYER_LINK); ++ if (link_file == NULL) { ++ ERROR("Failed to get layer link file %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (!util_fileself_exists(link_file)) { ++ ERROR("link file for layer %s not exist", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ diff_symlink = util_read_content_from_file(link_file); ++ if (link_file == NULL) { ++ ERROR("Failed to read content from link file of layer %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (do_diff_symlink(overlay_id, diff_symlink, data->overlay_home) != 0) { ++ ERROR("Failed to add diff link for layer %s", overlay_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ free(link_file); ++ free(diff_symlink); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = overlay_byid_old; ++ overlay_byid_old = overlay_byid_new; ++ overlay_byid_new = tmp_map; ++ empty_map(overlay_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *overlay_driver_impl_remote_support(void) ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ ERROR("Failed to calloc overlay supporter"); ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool overlay_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(overlay_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +new file mode 100644 +index 00000000..d03fc20b +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -0,0 +1,219 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-02-27 ++ * Description: remote layer store implementation ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "layer_store.h" ++ ++#include ++#include ++#include ++ ++#include "map.h" ++#include "utils.h" ++#include "remote_support.h" ++#include "ro_symlink_maintain.h" ++#include "path.h" ++#include "driver_overlay2.h" ++ ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; ++ ++static map_t *layer_byid_old = NULL; ++static map_t *layer_byid_new = NULL; ++ ++static void *remote_support_create(const char *layer_home, const char *layer_ro) ++{ ++ struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); ++ if (data == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ data->layer_home = util_strdup_s(layer_home); ++ data->layer_ro = util_strdup_s(layer_ro); ++ layer_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ layer_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ ++ return data; ++}; ++ ++static void remote_support_destroy(void *data) ++{ ++ if (data == NULL) { ++ return; ++ } ++ ++ map_free(layer_byid_old); ++ map_free(layer_byid_new); ++ free(data); ++} ++ ++static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context) ++{ ++ bool exist = true; ++ ++ if (!map_insert(layer_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) { ++ ERROR("can't insert remote layer into map"); ++ return false; ++ } ++ ++ return true; ++} ++ ++static int remote_support_scan(void *data) ++{ ++ struct remote_layer_data *remote_data = data; ++ return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++} ++ ++static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int nret = 0; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer symbol link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (remove_memory_stores_with_lock(layer_id) != 0) { ++ ERROR("Failed to remove remote layer store memory"); ++ ret = -1; ++ } ++ ++out: ++ free(ro_symlink); ++ return ret; ++ ++} ++ ++static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) ++{ ++ char *ro_symlink = NULL; ++ char *layer_dir = NULL; ++ int ret = 0; ++ ++ ro_symlink = util_path_join(data->layer_home, layer_id); ++ layer_dir = util_path_join(data->layer_ro, layer_id); ++ ++ if (ro_symlink == NULL) { ++ ERROR("Failed to join ro symlink path: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ ++ if (layer_dir == NULL) { ++ ERROR("Failed to join ro layer dir: %s", layer_id); ++ ret = -1; ++ goto free_out; ++ } ++ // add symbol link first ++ if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) { ++ SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir); ++ ret = -1; ++ goto free_out; ++ } ++ // insert layer into memory ++ if (load_one_layer(layer_id) != 0) { ++ ERROR("Failed to load new layer: %s into memory", layer_id); ++ ret = -1; ++ } ++ ++free_out: ++ free(ro_symlink); ++ free(layer_dir); ++ ++ return ret; ++} ++ ++static int remote_support_add(void *data) ++{ ++ int ret = 0; ++ char **array_added = NULL; ++ char **array_deleted = NULL; ++ map_t *tmp_map = NULL; ++ int i = 0; ++ ++ if (data == NULL) { ++ return -1; ++ } ++ ++ array_added = added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ ++ for (i = 0; i < util_array_len((const char **)array_added); i++) { ++ if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ map_remove(layer_byid_new, (void *)array_added[i]); ++ ERROR("remote overlay layer current not valid: %s", array_added[i]); ++ continue; ++ } ++ ++ if (add_one_remote_layer(data, array_added[i]) != 0) { ++ ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ret = -1; ++ } ++ } ++ ++ for (i = 0; i < util_array_len((const char **)array_deleted); i++) { ++ if (remove_one_remote_layer(data, array_deleted[i]) != 0) { ++ ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ ret = -1; ++ } ++ } ++ ++ tmp_map = layer_byid_old; ++ layer_byid_old = layer_byid_new; ++ layer_byid_new = tmp_map; ++ empty_map(layer_byid_new); ++ ++ util_free_array(array_added); ++ util_free_array(array_deleted); ++ ++ return ret; ++} ++ ++remote_support *layer_store_impl_remote_support() ++{ ++ remote_support *rs = util_common_calloc_s(sizeof(remote_support)); ++ if (rs == NULL) { ++ return NULL; ++ } ++ ++ rs->create = remote_support_create; ++ rs->destroy = remote_support_destroy; ++ rs->scan_remote_dir = remote_support_scan; ++ rs->load_item = remote_support_add; ++ ++ return rs; ++} ++ ++bool layer_remote_layer_valid(const char *layer_id) ++{ ++ return map_search(layer_byid_old, (void *)layer_id) != NULL; ++} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 60aaff22..8b8f5f1e 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -31,8 +31,6 @@ + #include + #include + #include +-#include +- + #include + #include + +@@ -50,6 +48,10 @@ + #include "http.h" + #include "utils_base64.h" + #include "constants.h" ++#include "path.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + #define PAYLOAD_CRC_LEN 12 + +@@ -78,6 +80,9 @@ typedef struct { + static layer_store_metadata g_metadata; + static char *g_root_dir; + static char *g_run_dir; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++static bool g_enable_remote_layer; ++#endif + + static inline char *tar_split_path(const char *id); + static inline char *mountpoint_json_path(const char *id); +@@ -131,7 +136,7 @@ void layer_store_cleanup() + map_free(g_metadata.by_uncompress_digest); + g_metadata.by_uncompress_digest = NULL; + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + linked_list_del(item); + layer_ref_dec((layer_t *)item->elem); + free(item); +@@ -162,7 +167,7 @@ static void free_digest_layer_t(digest_layer_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->layer_list), next) { ++ linked_list_for_each_safe (item, &(ptr->layer_list), next) { + linked_list_del(item); + free(item->elem); + item->elem = NULL; +@@ -276,6 +281,10 @@ static bool init_from_conf(const struct storage_module_init_options *conf) + g_root_dir = tmp_path; + tmp_path = NULL; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote_layer = conf->enable_remote_layer; ++#endif ++ + return true; + free_out: + free(g_run_dir); +@@ -613,7 +622,7 @@ static int delete_digest_from_map(map_t *by_digest, const char *digest, const ch + return 0; + } + +- linked_list_for_each_safe(item, &(old_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(old_list->layer_list), next) { + char *t_id = (char *)item->elem; + if (strcmp(t_id, id) == 0) { + linked_list_del(item); +@@ -726,7 +735,7 @@ static int remove_memory_stores(const char *id) + } + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + if (strcmp(tl->slayer->id, id) != 0) { + continue; +@@ -1119,10 +1128,25 @@ static int new_layer_by_opts(const char *id, const struct layer_opts *opts) + ret = -1; + goto out; + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && !opts->writable) { ++ if (remote_layer_build_ro_dir(id) != 0) { ++ ret = -1; ++ goto out; ++ } ++ } else { ++ if (!build_layer_dir(id)) { ++ ret = -1; ++ goto out; ++ } ++ } ++#else + if (!build_layer_dir(id)) { + ret = -1; + goto out; + } ++#endif + + ret = update_layer_datas(id, opts, l); + if (ret != 0) { +@@ -1304,7 +1328,15 @@ clear_memory: + driver_remove: + if (ret != 0) { + (void)graphdriver_rm_layer(lid); ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer) { ++ (void)remote_layer_remove_ro_dir(lid); ++ } else { ++ (void)layer_store_remove_layer(lid); ++ } ++#else + (void)layer_store_remove_layer(lid); ++#endif + } + free_out: + layer_store_unlock(); +@@ -1379,7 +1411,15 @@ static int do_delete_layer(const char *id) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } ++#else + ret = layer_store_remove_layer(l->slayer->id); ++#endif + + free_out: + free(tspath); +@@ -1461,7 +1501,7 @@ int layer_store_list(struct layer_list *resp) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *l = (layer_t *)item->elem; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1504,7 +1544,7 @@ static int layers_by_digest_map(map_t *m, const char *digest, struct layer_list + goto free_out; + } + +- linked_list_for_each_safe(item, &(id_list->layer_list), next) { ++ linked_list_for_each_safe (item, &(id_list->layer_list), next) { + layer_t *l = NULL; + resp->layers[i] = util_common_calloc_s(sizeof(struct layer)); + if (resp->layers[i] == NULL) { +@@ -1748,6 +1788,114 @@ out: + return ret; + } + ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ // if (remove_name(tl->slayer->names[i])) { ++ // should_save = true; ++ // } ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} ++ + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -1764,6 +1912,14 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // skip RO dir ++ // otherwise, RO dir will be treat as invalid layer dir ++ if (strcmp(sub_dir->d_name, REMOTE_RO_LAYER_DIR) == 0) { ++ goto free_out; ++ } ++#endif ++ + if (!util_dir_exists(tmpdir)) { + // ignore non-dir + DEBUG("%s is not directory", sub_dir->d_name); +@@ -1850,7 +2006,7 @@ static int load_layers_from_json_files() + goto unlock_out; + } + +- linked_list_for_each_safe(item, &(g_metadata.layers_list), next) { ++ linked_list_for_each_safe (item, &(g_metadata.layers_list), next) { + layer_t *tl = (layer_t *)item->elem; + size_t i = 0; + +@@ -1957,6 +2113,13 @@ int layer_store_init(const struct storage_module_init_options *conf) + goto free_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote_layer && remote_layer_init(g_root_dir) != 0) { ++ ERROR("Failed to init layer remote"); ++ goto free_out; ++ } ++#endif ++ + if (load_layers_from_json_files() != 0) { + goto free_out; + } +@@ -2129,7 +2292,7 @@ static tar_split *new_tar_split(layer_t *l, const char *tspath) + int ret = 0; + int nret = 0; + tar_split *ts = NULL; +- char path[PATH_MAX] = {0}; ++ char path[PATH_MAX] = { 0 }; + + ts = util_common_calloc_s(sizeof(tar_split)); + if (ts == NULL) { +@@ -2312,3 +2475,20 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + { + return graphdriver_get_metadata(id); + } ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++int remove_memory_stores_with_lock(const char *id) ++{ ++ int ret = 0; ++ ++ if (!layer_store_lock(true)) { ++ ERROR("Failed to lock layer store when handle: %s", id); ++ return -1; ++ } ++ ++ ret = remove_memory_stores(id); ++ layer_store_unlock(); ++ ++ return ret; ++} ++#endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 94d4bf04..44bd297e 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -23,6 +23,10 @@ + + #include "storage.h" + #include "io_wrapper.h" ++#include "map.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "remote_support.h" ++#endif + + struct io_read_wrapper; + struct layer_list; +@@ -79,6 +83,13 @@ int layer_store_check(const char *id); + + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++remote_support *layer_store_impl_remote_support(); ++bool layer_remote_layer_valid(const char *layer_id); ++int load_one_layer(const char *id); ++int remove_memory_stores_with_lock(const char *id); ++#endif ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..06c78678 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,12 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_remote_layer_support_srcs) ++ ++set(REMOTE_LAYER_SUPPORT_SRCS ++ ${local_remote_layer_support_srcs} ++ PARENT_SCOPE ++ ) ++ ++set(REMOTE_LAYER_SUPPORT_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++) +\ No newline at end of file +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +new file mode 100644 +index 00000000..9dc096f7 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -0,0 +1,122 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide image store functions ++ ******************************************************************************/ ++ ++#include "remote_support.h" ++ ++#include "layer_store.h" ++#include "image_store.h" ++#include "isula_libutils/log.h" ++#include "driver_overlay2.h" ++#include "utils.h" ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = layer_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = image_store_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++} ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) ++{ ++ remote_support *handlers = overlay_driver_impl_remote_support(); ++ if (handlers == NULL || handlers->create == NULL) { ++ return NULL; ++ } ++ ++ remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); ++ if (supporter == NULL) { ++ goto err_out; ++ } ++ ++ supporter->handlers = handlers; ++ supporter->data = handlers->create(remote_home, remote_ro); ++ ++ return supporter; ++ ++err_out: ++ free(handlers); ++ free(supporter); ++ return NULL; ++ ++} ++ ++void destroy_suppoter(remote_supporter *supporter) ++{ ++ if (supporter->handlers->destroy == NULL) { ++ ERROR("destroy_supporter operation not supported"); ++ return; ++ } ++ ++ supporter->handlers->destroy(supporter->data); ++ free(supporter->handlers); ++ free(supporter); ++} ++ ++int scan_remote_dir(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("scan_remote_dir operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->scan_remote_dir(supporter->data); ++} ++ ++int load_item(remote_supporter *supporter) ++{ ++ if (supporter->handlers->scan_remote_dir == NULL) { ++ ERROR("load_item operation not supported"); ++ return -1; ++ } ++ return supporter->handlers->load_item(supporter->data); ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +new file mode 100644 +index 00000000..d1f7af35 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -0,0 +1,58 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-03 ++ * Description: provide remote support functions ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H ++ ++#include "linked_list.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct { ++ void *(*create)(const char *remote_home, const char *remote_ro); ++ void (*destroy)(void *data); ++ // populate the list contains all dirs ++ int (*scan_remote_dir)(void *data); ++ // consume the list contains all dirs ++ int (*load_item)(void *data); ++} remote_support; ++ ++typedef struct { ++ void *data; ++ remote_support *handlers; ++} remote_supporter; ++ ++// RemoteSupport *impl_remote_support(); ++remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++ ++remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++ ++void destroy_suppoter(remote_supporter *supporter); ++ ++int scan_remote_dir(remote_supporter *supporter); ++ ++int load_item(remote_supporter *supporter); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +new file mode 100644 +index 00000000..7df7a221 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -0,0 +1,347 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#define _GNU_SOURCE ++#include "ro_symlink_maintain.h" ++ ++#include ++#include ++#include ++#include ++#include "map.h" ++#include "path.h" ++#include "linked_list.h" ++#include "layer_store.h" ++#include "layer.h" ++#include "isula_libutils/log.h" ++#include "image_store.h" ++#include "remote_support.h" ++#include "utils.h" ++#include "utils_file.h" ++#include "stdlib.h" ++ ++#define REMOTE_RO_LAYER_DIR "RO" ++ ++// overlay-layers and overlay-layers/RO ++static char *image_home; ++ ++static char *layer_ro_dir; ++static char *layer_home; ++ ++// overlay and overlay/RO ++static char *overlay_ro_dir; ++static char *overlay_home; ++ ++struct supporters { ++ remote_supporter *image_supporter; ++ remote_supporter *layer_supporter; ++ remote_supporter *overlay_supporter; ++}; ++ ++static struct supporters supporters; ++ ++int remote_image_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ image_home = util_strdup_s(root_dir); ++ if (image_home == NULL) { ++ ERROR("Failed create path for remote image home"); ++ goto out; ++ } ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_layer_init(const char *root_dir) ++{ ++ if (root_dir == NULL) { ++ goto out; ++ } ++ ++ layer_home = util_strdup_s(root_dir); ++ layer_ro_dir = util_path_join(root_dir, REMOTE_RO_LAYER_DIR); ++ if (layer_ro_dir == NULL) { ++ ERROR("Failed join path when init remote layer maintainer"); ++ goto out; ++ } ++ if (!util_file_exists(layer_ro_dir) && util_mkdir_p(layer_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++int remote_overlay_init(const char *driver_home) ++{ ++ if (driver_home == NULL) { ++ goto out; ++ } ++ ++ overlay_home = util_strdup_s(driver_home); ++ overlay_ro_dir = util_path_join(driver_home, REMOTE_RO_LAYER_DIR); ++ if (overlay_ro_dir == NULL) { ++ ERROR("Failed to join path when init remote maintainer"); ++ goto out; ++ } ++ // build RO dir if not exist ++ if (!util_file_exists(overlay_ro_dir) && util_mkdir_p(overlay_ro_dir, 0700) != 0) { ++ ERROR("Failed to create RO dir under overlay"); ++ goto out; ++ } ++ ++ return 0; ++ ++out: ++ remote_maintain_cleanup(); ++ return -1; ++} ++ ++void remote_maintain_cleanup(void) ++{ ++ free(image_home); ++ image_home = NULL; ++ ++ free(layer_home); ++ layer_home = NULL; ++ free(layer_ro_dir); ++ layer_ro_dir = NULL; ++ free(overlay_home); ++ ++ overlay_home = NULL; ++ free(overlay_ro_dir); ++ overlay_ro_dir = NULL; ++} ++ ++// to maintain the symbol links, add new symbol link and delete invalid symbol link ++// arg is const char *driver_home ++// scanning driver->home/RO/ directory, build symlink in driver->home ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); ++ ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); ++ scan_remote_dir(supporters->overlay_supporter); ++ load_item(supporters->overlay_supporter); ++ scan_remote_dir(supporters->layer_supporter); ++ load_item(supporters->layer_supporter); ++ scan_remote_dir(supporters->image_supporter); ++ load_item(supporters->image_supporter); ++ DEBUG("remote refresh end\n"); ++ } ++ return NULL; ++} ++ ++int start_refresh_thread(void) ++{ ++ int res = 0; ++ pthread_t a_thread; ++ ++ supporters.image_supporter = create_image_supporter(image_home, NULL); ++ if (supporters.image_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); ++ if (supporters.layer_supporter == NULL) { ++ goto free_out; ++ } ++ ++ supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); ++ if (supporters.overlay_supporter == NULL) { ++ goto free_out; ++ } ++ ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; ++ } ++ ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } ++ ++ return 0; ++ ++free_out: ++ destroy_suppoter(supporters.image_supporter); ++ destroy_suppoter(supporters.layer_supporter); ++ destroy_suppoter(supporters.overlay_supporter); ++ ++ return -1; ++} ++ ++static int do_build_ro_dir(const char *home, const char *id) ++{ ++ char *ro_symlink = NULL; ++ char *ro_layer_dir = NULL; ++ int nret = 0; ++ // bool ret = true; ++ int ret = 0; ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed create ro layer dir sym link path"); ++ return -1; ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Failed to create ro layer dir path"); ++ return -1; ++ } ++ ++ if (util_mkdir_p(ro_layer_dir, IMAGE_STORE_PATH_MODE) != 0) { ++ ret = -1; ++ ERROR("Failed to create layer direcotry %s", ro_layer_dir); ++ goto out; ++ } ++ ++ if (symlink(ro_layer_dir, ro_symlink) != 0) { ++ ret = -1; ++ SYSERROR("Failed to create symlink to layer dir %s", ro_layer_dir); ++ goto err_out; ++ } ++ ++ goto out; ++ ++err_out: ++ if (util_recursive_rmdir(ro_layer_dir, 0)) { ++ ERROR("Failed to delete layer path: %s", ro_layer_dir); ++ } ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_overlay_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(overlay_home, id); ++} ++ ++int remote_layer_build_ro_dir(const char *id) ++{ ++ return do_build_ro_dir(layer_home, id); ++} ++ ++int do_remove_ro_dir(const char *home, const char *id) ++{ ++ char *ro_layer_dir = NULL; ++ char *ro_symlink = NULL; ++ char clean_path[PATH_MAX] = { 0 }; ++ int ret = 0; ++ int nret = 0; ++ ++ if (id == NULL) { ++ return 0; ++ } ++ ++ nret = asprintf(&ro_symlink, "%s/%s", home, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer sym link path failed"); ++ return -1; ++ } ++ ++ if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create layer json path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ ret = util_recursive_rmdir(ro_layer_dir, 0); ++ ++out: ++ free(ro_layer_dir); ++ free(ro_symlink); ++ return ret; ++} ++ ++int remote_layer_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(layer_home, id); ++} ++ ++int remote_overlay_remove_ro_dir(const char *id) ++{ ++ return do_remove_ro_dir(overlay_home, id); ++} ++ ++static char **map_diff(map_t *map_a, map_t *map_b) ++{ ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } ++ } ++ ++ map_itor_free(itor); ++ ++ return array; ++} ++ ++char **deleted_layers(map_t *old, map_t *new) ++{ ++ return map_diff(old, new); ++} ++ ++char **added_layers(map_t *old, map_t *new) ++{ ++ return map_diff(new, old); ++} ++ ++int empty_map(map_t *mp) ++{ ++ if (mp == NULL) { ++ return -1; ++ } ++ ++ map_clear(mp); ++ mp->store->root = mp->store->nil; ++ return 0; ++} +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +new file mode 100644 +index 00000000..25712d40 +--- /dev/null ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -0,0 +1,52 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-01-12 ++ * Description: provide remote symlink maintain functions ++ ******************************************************************************/ ++#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H ++ ++#include "map.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++int remote_image_init(const char *root_dir); ++ ++int remote_layer_init(const char *root_dir); ++ ++int remote_overlay_init(const char *driver_home); ++ ++void remote_maintain_cleanup(void); ++ ++int start_refresh_thread(void); ++ ++int remote_layer_build_ro_dir(const char *id); ++ ++int remote_overlay_build_ro_dir(const char *id); ++ ++int remote_layer_remove_ro_dir(const char *id); ++ ++int remote_overlay_remove_ro_dir(const char *id); ++ ++char **deleted_layers(map_t *old, map_t *new); ++ ++char **added_layers(map_t *old, map_t *new); ++ ++int empty_map(map_t *mp); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 2f4bdf5f..31812a22 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -42,6 +42,9 @@ + #include "utils_string.h" + #include "utils_verify.h" + #include "sha256.h" ++#ifdef ENABLE_REMOTE_LAYER_STORE ++#include "ro_symlink_maintain.h" ++#endif + + static pthread_rwlock_t g_storage_rwlock; + static char *g_storage_run_root; +@@ -1870,6 +1873,12 @@ int storage_module_init(struct storage_module_init_options *opts) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ ERROR("Failed to start remote refresh thread"); ++ } ++#endif ++ + if (restore_images_size() != 0) { + ERROR("Failed to recal image size"); + ret = -1; +@@ -1906,4 +1915,4 @@ out: + char *storage_rootfs_get_dir() + { + return rootfs_store_get_data_dir(); +-} +\ No newline at end of file ++} +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 3ec47959..7404ee54 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -70,6 +70,9 @@ struct storage_module_init_options { + char **driver_opts; + size_t driver_opts_len; + bool integration_check; ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ bool enable_remote_layer; ++#endif + }; + + struct storage_img_create_options { +-- +2.25.1 + diff --git a/0049-add-ci-for-remote-ro.patch b/0049-add-ci-for-remote-ro.patch new file mode 100644 index 0000000..9e6ece1 --- /dev/null +++ b/0049-add-ci-for-remote-ro.patch @@ -0,0 +1,119 @@ +From 1b1900c183886e13edd86c2e2cb6e5c42cfebd3d Mon Sep 17 00:00:00 2001 +From: Neil +Date: Sun, 5 Mar 2023 12:23:29 +0000 +Subject: [PATCH 49/56] add ci for remote ro + +Signed-off-by: Neil +--- + CI/make-and-install.sh | 4 +- + .../container_cases/test_data/daemon.json | 1 + + CI/test_cases/image_cases/ro_separate.sh | 69 +++++++++++++++++++ + 3 files changed, 72 insertions(+), 2 deletions(-) + create mode 100644 CI/test_cases/image_cases/ro_separate.sh + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 81022d75..e714d206 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -106,9 +106,9 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi + make -j $(nproc) + make install +diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json +index aa88c9da..2664c6b2 100644 +--- a/CI/test_cases/container_cases/test_data/daemon.json ++++ b/CI/test_cases/container_cases/test_data/daemon.json +@@ -19,6 +19,7 @@ + "hook-spec": "/etc/default/isulad/hooks/default.json", + "start-timeout": "2m", + "storage-driver": "overlay2", ++ "storage-enable-remote-layer": false, + "storage-opts": [ + "overlay2.override_kernel_check=true" + ], +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +new file mode 100644 +index 00000000..47e04abb +--- /dev/null ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -0,0 +1,69 @@ ++#!/bin/bash ++# ++# attributes: isulad basic image ++# concurrent: NA ++# spend time: 22 ++ ++####################################################################### ++##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++# - iSulad licensed under the Mulan PSL v2. ++# - You can use this software according to the terms and conditions of the Mulan PSL v2. ++# - You may obtain a copy of Mulan PSL v2 at: ++# - http://license.coscl.org.cn/MulanPSL2 ++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# - PURPOSE. ++# - See the Mulan PSL v2 for more details. ++##- @Description:CI ++##- @Author: wangrunze ++##- @Create: 2023-03-03 ++####################################################################### ++ ++declare -r curr_path=$(dirname $(readlink -f "$0")) ++source ../helpers.sh ++single_image="${curr_path}/busybox.tar" ++ ++function test_separate_ro() ++{ ++ local ret=0 ++ local test="isula separate ro test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ isula rmi busybox ++ ++ isula pull busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++)) ++ ++ isula run -tid --name test_separate busybox /bin/sh ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++)) ++ ++ isula stop test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) ++ ++ isula rmi busybox ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) ++ ++ isula load -i $single_image ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++)) ++ ++ check_valgrind_log ++ [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++)) ++ ++ sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json ++ start_isulad_with_valgrind ++ wait_isulad_running ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ ++declare -i ans=0 ++ ++test_separate_ro || ((ans++)) ++ ++show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0050-fix-compile-error-when-not-enable-remote-ro.patch b/0050-fix-compile-error-when-not-enable-remote-ro.patch new file mode 100644 index 0000000..baba9ea --- /dev/null +++ b/0050-fix-compile-error-when-not-enable-remote-ro.patch @@ -0,0 +1,62 @@ +From e4f309f61b169529c263b7a83a0eda16ebe132f5 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Sun, 5 Mar 2023 18:55:40 -0800 +Subject: [PATCH 50/56] fix compile error when not enable remote ro + +Signed-off-by: Neil.wrz +--- + .../modules/image/oci/storage/image_store/CMakeLists.txt | 3 +++ + .../modules/image/oci/storage/layer_store/CMakeLists.txt | 4 ++++ + .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt | 3 +++ + 3 files changed, 10 insertions(+) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index ecf21caa..7d4fb77c 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") ++ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index f964f709..e04b4ad7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") ++ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS + ${GRAPHDRIVER_SRCS} + PARENT_SCOPE + ) ++ + set(LAYER_STORE_INCS + ${CMAKE_CURRENT_SOURCE_DIR} + ${GRAPHDRIVER_INCS} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index ceed16b7..dd4e82aa 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,5 +1,8 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) ++IF (NOT ENABLE_REMOTE_LAYER_STORE) ++list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") ++ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +-- +2.25.1 + diff --git a/0051-CI-not-enable-remote-ro-for-ut.patch b/0051-CI-not-enable-remote-ro-for-ut.patch new file mode 100644 index 0000000..46a3eb6 --- /dev/null +++ b/0051-CI-not-enable-remote-ro-for-ut.patch @@ -0,0 +1,41 @@ +From 1d5aa99d97df3f4ddec4ae436cb0ccbbba3e863a Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 6 Mar 2023 18:59:43 -0800 +Subject: [PATCH 51/56] CI not enable remote ro for ut + +Signed-off-by: Neil.wrz +--- + CI/make-and-install.sh | 2 +- + CI/test_cases/image_cases/ro_separate.sh | 3 +++ + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index e714d206..7c194826 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -106,7 +106,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh +index 47e04abb..df45e120 100644 +--- a/CI/test_cases/image_cases/ro_separate.sh ++++ b/CI/test_cases/image_cases/ro_separate.sh +@@ -45,6 +45,9 @@ function test_separate_ro() + isula stop test_separate + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++)) + ++ isula rm test_separate ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++)) ++ + isula rmi busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++)) + +-- +2.25.1 + diff --git a/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch b/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch new file mode 100644 index 0000000..adab3d1 --- /dev/null +++ b/0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch @@ -0,0 +1,99 @@ +From 569e2d07cc153f2918868ba58bc7da9a626e4db0 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 7 Mar 2023 23:59:56 -0800 +Subject: [PATCH 52/56] bugfix remote ro try add or remove image/layer twice + +Signed-off-by: Neil.wrz +--- + .../image/oci/storage/image_store/image_store.c | 14 ++++++++++++++ + .../oci/storage/layer_store/layer_remote_impl.c | 2 +- + .../image/oci/storage/layer_store/layer_store.c | 11 +++++++++++ + 3 files changed, 26 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index c6836e46..a44bf362 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -3671,6 +3671,11 @@ int append_image_by_directory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ DEBUG("remote image already exist, not added: %s", id); ++ goto out; ++ } ++ + nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id); + if (nret < 0 || (size_t)nret >= sizeof(image_path)) { + ERROR("Failed to get image path"); +@@ -3678,6 +3683,8 @@ int append_image_by_directory_with_lock(const char *id) + } + + ret = append_image_by_directory(image_path); ++ ++out: + image_store_unlock(); + + return ret; +@@ -3692,7 +3699,14 @@ int remove_image_from_memory_with_lock(const char *id) + return -1; + } + ++ if (map_search(g_image_store->byid, (void *)id) == NULL) { ++ DEBUG("remote image already remvoed, don't delete twice: %s", id); ++ goto out; ++ } ++ + ret = remove_image_from_memory(id); ++ ++out: + image_store_unlock(); + + return ret; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +index d03fc20b..d676458c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +@@ -175,7 +175,7 @@ static int remote_support_add(void *data) + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { +- ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ ERROR("Failed to add remote layer: %s", array_added[i]); + ret = -1; + } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 8b8f5f1e..29ead711 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1855,6 +1855,11 @@ int load_one_layer(const char *id) + return -1; + } + ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ + tl = load_one_layer_from_json(id); + if (tl == NULL) { + ret = -1; +@@ -2485,8 +2490,14 @@ int remove_memory_stores_with_lock(const char *id) + ERROR("Failed to lock layer store when handle: %s", id); + return -1; + } ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } + + ret = remove_memory_stores(id); ++ ++unlock_out: + layer_store_unlock(); + + return ret; +-- +2.25.1 + diff --git a/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch b/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch new file mode 100644 index 0000000..94801b9 --- /dev/null +++ b/0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch @@ -0,0 +1,34 @@ +From bbcc4c58d67aeceee55fcc1126deedafe0e43c20 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 14 Mar 2023 20:32:23 -0700 +Subject: [PATCH 53/56] bugfix can't delete layers under dir overlay-layers + +Signed-off-by: Neil.wrz +--- + .../modules/image/oci/storage/layer_store/layer_store.c | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 29ead711..680b35a2 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1412,10 +1412,14 @@ static int do_delete_layer(const char *id) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (l->slayer->writable) { ++ if (!g_enable_remote_layer) { + ret = layer_store_remove_layer(l->slayer->id); + } else { +- ret = remote_layer_remove_ro_dir(l->slayer->id); ++ if (l->slayer->writable) { ++ ret = layer_store_remove_layer(l->slayer->id); ++ } else { ++ ret = remote_layer_remove_ro_dir(l->slayer->id); ++ } + } + #else + ret = layer_store_remove_layer(l->slayer->id); +-- +2.25.1 + diff --git a/0054-refactor-remote-ro-code.patch b/0054-refactor-remote-ro-code.patch new file mode 100644 index 0000000..86196e8 --- /dev/null +++ b/0054-refactor-remote-ro-code.patch @@ -0,0 +1,2089 @@ +From 2bfce1ac385eabd9614e4d8d6f50c02efb9653fc Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 6 Mar 2023 17:43:17 -0800 +Subject: [PATCH 54/56] refactor remote ro code + +Signed-off-by: Neil.wrz +--- + CI/make-and-install.sh | 2 +- + .../oci/storage/image_store/CMakeLists.txt | 3 - + .../oci/storage/image_store/image_store.c | 37 +-- + .../oci/storage/image_store/image_store.h | 13 +- + .../oci/storage/layer_store/CMakeLists.txt | 3 - + .../graphdriver/overlay2/CMakeLists.txt | 3 - + .../graphdriver/overlay2/driver_overlay2.h | 8 - + .../oci/storage/layer_store/layer_store.c | 230 +++++++++--------- + .../oci/storage/layer_store/layer_store.h | 9 +- + .../image_remote_impl.c | 61 +++-- + .../layer_remote_impl.c | 71 +++--- + .../overlay_remote_impl.c | 103 +++++--- + .../remote_layer_support/remote_support.c | 141 ++++++----- + .../remote_layer_support/remote_support.h | 62 +++-- + .../ro_symlink_maintain.c | 124 +--------- + .../ro_symlink_maintain.h | 19 +- + .../modules/image/oci/storage/storage.c | 4 +- + src/utils/cutils/map/rb_tree.c | 1 + + test/image/oci/registry/CMakeLists.txt | 2 + + test/image/oci/storage/CMakeLists.txt | 3 + + test/image/oci/storage/images/CMakeLists.txt | 2 + + test/image/oci/storage/layers/CMakeLists.txt | 4 + + .../remote_layer_support/CMakeLists.txt | 44 ++++ + .../remote_layer_support/remote_layer_ut.cc | 93 +++++++ + test/image/oci/storage/rootfs/CMakeLists.txt | 2 + + test/mocks/remote_store_mock.cc | 68 ++++++ + test/mocks/remote_store_mock.h | 40 +++ + 27 files changed, 683 insertions(+), 469 deletions(-) + rename src/daemon/modules/image/oci/storage/{image_store => remote_layer_support}/image_remote_impl.c (71%) + rename src/daemon/modules/image/oci/storage/{layer_store => remote_layer_support}/layer_remote_impl.c (76%) + rename src/daemon/modules/image/oci/storage/{layer_store/graphdriver/overlay2 => remote_layer_support}/overlay_remote_impl.c (72%) + create mode 100644 test/image/oci/storage/remote_layer_support/CMakeLists.txt + create mode 100644 test/image/oci/storage/remote_layer_support/remote_layer_ut.cc + create mode 100644 test/mocks/remote_store_mock.cc + create mode 100644 test/mocks/remote_store_mock.h + +diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh +index 7c194826..e714d206 100755 +--- a/CI/make-and-install.sh ++++ b/CI/make-and-install.sh +@@ -106,7 +106,7 @@ rm -rf build + mkdir build + cd build + if [[ ${enable_gcov} -ne 0 ]]; then +- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON .. ++ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + else + cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON .. + fi +diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +index 7d4fb77c..ecf21caa 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c") +-ENDIF() + + set(IMAGE_STORE_SRCS + ${local_image_store_srcs} +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c +index a44bf362..7837f9db 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c +@@ -133,7 +133,7 @@ static void free_image_store(image_store_t *store) + (void)map_free(store->bydigest); + store->bydigest = NULL; + +- linked_list_for_each_safe(item, &(store->images_list), next) { ++ linked_list_for_each_safe (item, &(store->images_list), next) { + linked_list_del(item); + image_ref_dec((image_t *)item->elem); + free(item); +@@ -165,7 +165,7 @@ static void image_store_digest_field_kvfree(void *key, void *value) + + free(key); + if (val != NULL) { +- linked_list_for_each_safe(item, &(val->images_list), next) { ++ linked_list_for_each_safe (item, &(val->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -501,7 +501,7 @@ static void digest_image_slice_without_value(digest_image_t *digest_filter_image + return; + } + +- linked_list_for_each_safe(item, &(digest_filter_images->images_list), next) { ++ linked_list_for_each_safe (item, &(digest_filter_images->images_list), next) { + tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, img->simage->id) == 0) { + linked_list_del(item); +@@ -582,7 +582,7 @@ static int remove_image_from_memory(const char *id) + goto out; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *tmp = (image_t *)item->elem; + if (strcmp(tmp->simage->id, id) != 0) { + continue; +@@ -681,7 +681,7 @@ static void free_digest_image(digest_image_t *ptr) + return; + } + +- linked_list_for_each_safe(item, &(ptr->images_list), next) { ++ linked_list_for_each_safe (item, &(ptr->images_list), next) { + linked_list_del(item); + free(item); + item = NULL; +@@ -2679,7 +2679,7 @@ int image_store_get_all_images(imagetool_images_list *images_list) + goto unlock; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + imagetool_image_summary *imginfo = NULL; + image_t *img = (image_t *)item->elem; + imginfo = get_image_summary(img); +@@ -3102,7 +3102,7 @@ out: + return ret; + } + +-int validate_manifest_schema_version_1(const char *path, bool *valid) ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) + { + int ret = 0; + int nret; +@@ -3509,7 +3509,7 @@ static int get_images_from_json() + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } +@@ -3546,7 +3546,7 @@ static void image_store_check_all_images() + return; + } + +- linked_list_for_each_safe(item, &(g_image_store->images_list), next) { ++ linked_list_for_each_safe (item, &(g_image_store->images_list), next) { + image_t *img = (image_t *)item->elem; + if (img->spec == NULL) { + ERROR("Failed to check spec info of image: %s, try to delete", img->simage->id); +@@ -3660,18 +3660,23 @@ out: + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int append_image_by_directory_with_lock(const char *id) ++int remote_append_image_by_directory_with_lock(const char *id) + { + int ret = 0; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; + ++ if (id == NULL) { ++ ERROR("can't add NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; + } + +- if (map_search(g_image_store->byid, (void *)id) != NULL ) { ++ if (map_search(g_image_store->byid, (void *)id) != NULL) { + DEBUG("remote image already exist, not added: %s", id); + goto out; + } +@@ -3690,10 +3695,15 @@ out: + return ret; + } + +-int remove_image_from_memory_with_lock(const char *id) ++int remote_remove_image_from_memory_with_lock(const char *id) + { + int ret = 0; + ++ if (id == NULL) { ++ ERROR("can't remove NULL remote image"); ++ return -1; ++ } ++ + if (!image_store_lock(EXCLUSIVE)) { + ERROR("Failed to lock remote image store when handle: %s", id); + return -1; +@@ -3712,9 +3722,8 @@ out: + return ret; + } + +-char *get_top_layer_from_json(const char *img_id) ++char *remote_image_get_top_layer_from_json(const char *img_id) + { +- + char *ret = NULL; + int nret = 0; + char image_path[PATH_MAX] = { 0x00 }; +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h +index c3cb50e3..5164cc73 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h ++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h +@@ -28,9 +28,6 @@ + #include "isula_libutils/imagetool_image.h" + #include "isula_libutils/imagetool_images_list.h" + #include "isula_libutils/imagetool_image_summary.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct storage_module_init_options; + +@@ -112,11 +109,11 @@ void image_store_free(); + imagetool_image_summary *image_store_get_image_summary(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *image_store_impl_remote_support(); +-int validate_manifest_schema_version_1(const char *path, bool *valid); +-int append_image_by_directory_with_lock(const char *image_dir); +-int remove_image_from_memory_with_lock(const char *id); +-char *get_top_layer_from_json(const char *img_id); /* return top layer id */ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid); ++int remote_append_image_by_directory_with_lock(const char *image_dir); ++int remote_remove_image_from_memory_with_lock(const char *id); ++// return top layer id ++char *remote_image_get_top_layer_from_json(const char *img_id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +index e04b4ad7..c218a7c0 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c") +-ENDIF() + add_subdirectory(graphdriver) + + set(LAYER_STORE_SRCS +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +index dd4e82aa..ceed16b7 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt +@@ -1,8 +1,5 @@ + # get current directory sources files + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs) +-IF (NOT ENABLE_REMOTE_LAYER_STORE) +-list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c") +-ENDIF() + + set(OVERLAY2_SRCS + ${local_overlay2_srcs} +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +index 5c1d93fb..e14271b1 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h +@@ -22,9 +22,6 @@ + #include + + #include "driver.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct driver_create_opts; + struct driver_mount_opts; +@@ -71,11 +68,6 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap + + int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info); + +-#ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *overlay_driver_impl_remote_support(void); +-bool overlay_remote_layer_valid(const char *layer_id); +-#endif +- + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index 680b35a2..12fca5ff 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1792,119 +1792,6 @@ out: + return ret; + } + +-static layer_t *load_one_layer_from_json(const char *id) +-{ +- int nret = 0; +- char *mount_point_path = NULL; +- char tmpdir[PATH_MAX] = { 0 }; +- char *rpath = NULL; +- layer_t *l = NULL; +- bool layer_valid = false; +- +- nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); +- if (nret < 0 || nret >= PATH_MAX) { +- ERROR("Sprintf: %s failed", id); +- goto free_out; +- } +- +- mount_point_path = mountpoint_json_path(id); +- if (mount_point_path == NULL) { +- ERROR("Out of Memory"); +- goto free_out; +- } +- +- rpath = layer_json_path(id); +- if (rpath == NULL) { +- ERROR("%s is invalid layer", id); +- goto free_out; +- } +- +- l = load_layer(rpath, mount_point_path); +- if (l == NULL) { +- ERROR("load layer: %s failed, remove it", id); +- goto free_out; +- } +- +- if (do_validate_image_layer(tmpdir, l) != 0) { +- ERROR("%s is invalid image layer", id); +- goto free_out; +- } +- +- if (do_validate_rootfs_layer(l) != 0) { +- ERROR("%s is invalid rootfs layer", id); +- goto free_out; +- } +- +- layer_valid = true; +- +-free_out: +- free(rpath); +- free(mount_point_path); +- if (!layer_valid) { +- free_layer_t(l); +- l = NULL; +- } +- // always return true; +- // if load layer failed, just remove it +- return l; +-} +- +-int load_one_layer(const char *id) +-{ +- int ret = 0; +- layer_t *tl = NULL; +- int i = 0; +- +- if (!layer_store_lock(true)) { +- return -1; +- } +- +- if (map_search(g_metadata.by_id, (void *)id) != NULL) { +- DEBUG("remote layer already exist, not added: %s", id); +- goto unlock_out; +- } +- +- tl = load_one_layer_from_json(id); +- if (tl == NULL) { +- ret = -1; +- goto unlock_out; +- } +- +- if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { +- ERROR("Insert id: %s for layer failed", tl->slayer->id); +- ret = -1; +- goto unlock_out; +- } +- +- for (; i < tl->slayer->names_len; i++) { +- // this should be done by master isulad +- // if (remove_name(tl->slayer->names[i])) { +- // should_save = true; +- // } +- if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { +- ret = -1; +- ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); +- goto unlock_out; +- } +- } +- ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s compress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); +- if (ret != 0) { +- ERROR("update layer: %s uncompress failed", tl->slayer->id); +- goto unlock_out; +- } +- +- ret = 0; +-unlock_out: +- layer_store_unlock(); +- return ret; +-} +- + static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context) + { + #define LAYER_NAME_LEN 64 +@@ -2486,7 +2373,7 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +-int remove_memory_stores_with_lock(const char *id) ++int remote_layer_remove_memory_stores_with_lock(const char *id) + { + int ret = 0; + +@@ -2499,6 +2386,11 @@ int remove_memory_stores_with_lock(const char *id) + goto unlock_out; + } + ++ if (map_search(g_metadata.by_id, (void *)id) == NULL) { ++ DEBUG("remote layer already removed, don't delete: %s", id); ++ goto unlock_out; ++ } ++ + ret = remove_memory_stores(id); + + unlock_out: +@@ -2506,4 +2398,114 @@ unlock_out: + + return ret; + } ++ ++static layer_t *load_one_layer_from_json(const char *id) ++{ ++ int nret = 0; ++ char *mount_point_path = NULL; ++ char tmpdir[PATH_MAX] = { 0 }; ++ char *rpath = NULL; ++ layer_t *l = NULL; ++ bool layer_valid = false; ++ ++ nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id); ++ if (nret < 0 || nret >= PATH_MAX) { ++ ERROR("Sprintf: %s failed", id); ++ goto free_out; ++ } ++ ++ mount_point_path = mountpoint_json_path(id); ++ if (mount_point_path == NULL) { ++ ERROR("Out of Memory"); ++ goto free_out; ++ } ++ ++ rpath = layer_json_path(id); ++ if (rpath == NULL) { ++ ERROR("%s is invalid layer", id); ++ goto free_out; ++ } ++ ++ l = load_layer(rpath, mount_point_path); ++ if (l == NULL) { ++ ERROR("load layer: %s failed, remove it", id); ++ goto free_out; ++ } ++ ++ if (do_validate_image_layer(tmpdir, l) != 0) { ++ ERROR("%s is invalid image layer", id); ++ goto free_out; ++ } ++ ++ if (do_validate_rootfs_layer(l) != 0) { ++ ERROR("%s is invalid rootfs layer", id); ++ goto free_out; ++ } ++ ++ layer_valid = true; ++ ++free_out: ++ free(rpath); ++ free(mount_point_path); ++ if (!layer_valid) { ++ free_layer_t(l); ++ l = NULL; ++ } ++ // always return true; ++ // if load layer failed, just remove it ++ return l; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ int ret = 0; ++ layer_t *tl = NULL; ++ int i = 0; ++ ++ if (!layer_store_lock(true)) { ++ return -1; ++ } ++ ++ if (map_search(g_metadata.by_id, (void *)id) != NULL) { ++ DEBUG("remote layer already exist, not added: %s", id); ++ goto unlock_out; ++ } ++ ++ tl = load_one_layer_from_json(id); ++ if (tl == NULL) { ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) { ++ ERROR("Insert id: %s for layer failed", tl->slayer->id); ++ ret = -1; ++ goto unlock_out; ++ } ++ ++ for (; i < tl->slayer->names_len; i++) { ++ // this should be done by master isulad ++ if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) { ++ ret = -1; ++ ERROR("Insert name: %s for layer failed", tl->slayer->names[i]); ++ goto unlock_out; ++ } ++ } ++ ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s compress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id); ++ if (ret != 0) { ++ ERROR("update layer: %s uncompress failed", tl->slayer->id); ++ goto unlock_out; ++ } ++ ++ ret = 0; ++unlock_out: ++ layer_store_unlock(); ++ return ret; ++} + #endif +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 44bd297e..4677e5ee 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -24,9 +24,6 @@ + #include "storage.h" + #include "io_wrapper.h" + #include "map.h" +-#ifdef ENABLE_REMOTE_LAYER_STORE +-#include "remote_support.h" +-#endif + + struct io_read_wrapper; + struct layer_list; +@@ -84,10 +81,8 @@ int layer_store_check(const char *id); + container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id); + + #ifdef ENABLE_REMOTE_LAYER_STORE +-remote_support *layer_store_impl_remote_support(); +-bool layer_remote_layer_valid(const char *layer_id); +-int load_one_layer(const char *id); +-int remove_memory_stores_with_lock(const char *id); ++int remote_load_one_layer(const char *id); ++int remote_layer_remove_memory_stores_with_lock(const char *id); + #endif + + #ifdef __cplusplus +diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +similarity index 71% +rename from src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +index 20da8116..a822ea81 100644 +--- a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c +@@ -13,27 +13,23 @@ + * Description: provide remote image store functions + ******************************************************************************/ + #define _GNU_SOURCE +-#include "image_store.h" ++#include "remote_support.h" + + #include + #include + +-#include "remote_support.h" + #include "ro_symlink_maintain.h" + #include "map.h" + #include "utils_file.h" + #include "utils.h" + #include "layer_store.h" ++#include "image_store.h" + #include "utils_array.h" + +-struct remote_image_data { +- const char *image_home; +-}; +- + static map_t *image_byid_old = NULL; + static map_t *image_byid_new = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_image_data *remote_image_create(const char *remote_home, const char *remote_ro) + { + struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data)); + if (data == NULL) { +@@ -46,7 +42,7 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_image_destroy(struct remote_image_data *data) + { + if (data == NULL) { + return; +@@ -59,7 +55,7 @@ static void remote_support_destroy(void *data) + return; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(void *data) + { + int ret = 0; + int nret; +@@ -79,7 +75,7 @@ static int remote_support_scan(void *data) + image_dirs_num = util_array_len((const char **)image_dirs); + + for (i = 0; i < image_dirs_num; i++) { +- bool valid_v1_image = false; ++ bool is_v1_image = false; + + if (util_reg_match(id_patten, image_dirs[i]) != 0) { + DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s", +@@ -93,12 +89,14 @@ static int remote_support_scan(void *data) + continue; + } + +- if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) { ++ if (image_store_validate_manifest_schema_version_1(image_path, &is_v1_image) != 0) { + ERROR("Failed to validate manifest schema version 1 format"); + continue; + } + +- if (!valid_v1_image) { ++ // for refresh, we don't care v1 image, cause image should be handled by master isulad ++ // when master isulad pull images ++ if (!is_v1_image) { + map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist); + } + } +@@ -108,12 +106,13 @@ out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(void *data) + { + char **array_added = NULL; + char **array_deleted = NULL; + char *top_layer = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + int ret = 0; + +@@ -121,26 +120,28 @@ static int remote_support_add(void *data) + return -1; + } + +- array_added = added_layers(image_byid_old, image_byid_new); +- array_deleted = deleted_layers(image_byid_old, image_byid_new); ++ array_added = remote_added_layers(image_byid_old, image_byid_new); ++ array_deleted = remote_deleted_layers(image_byid_old, image_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- top_layer = get_top_layer_from_json(array_added[i]); +- if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) { +- ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]); ++ top_layer = remote_image_get_top_layer_from_json(array_added[i]); ++ if (top_layer != NULL && !remote_layer_layer_valid(top_layer)) { ++ WARN("Current not find valid under layer, remoet image:%s not added", array_added[i]); + map_remove(image_byid_new, (void *)array_added[i]); + continue; + } + +- if (append_image_by_directory_with_lock(array_added[i]) != 0) { ++ if (remote_append_image_by_directory_with_lock(array_added[i]) != 0) { + ERROR("Failed to load image into memrory: %s", array_added[i]); ++ map_remove(image_byid_new, (void *)array_added[i]); + ret = -1; + } + } + + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { +- if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) { ++ if (remote_remove_image_from_memory_with_lock(array_deleted[i]) != 0) { + ERROR("Failed to remove remote memory store"); ++ map_insert(image_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -148,7 +149,7 @@ static int remote_support_add(void *data) + tmp_map = image_byid_old; + image_byid_old = image_byid_new; + image_byid_new = tmp_map; +- empty_map(image_byid_new); ++ map_clear(image_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -157,17 +158,13 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *image_store_impl_remote_support(void) +-{ +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++void remote_image_refresh(struct remote_image_data *data) { ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +similarity index 76% +rename from src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +index d676458c..3e3afff6 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c +@@ -13,7 +13,7 @@ + * Description: remote layer store implementation + ******************************************************************************/ + #define _GNU_SOURCE +-#include "layer_store.h" ++#include "remote_support.h" + + #include + #include +@@ -21,20 +21,15 @@ + + #include "map.h" + #include "utils.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "layer_store.h" + #include "path.h" + #include "driver_overlay2.h" + +-struct remote_layer_data { +- const char *layer_home; +- const char *layer_ro; +-}; +- + static map_t *layer_byid_old = NULL; + static map_t *layer_byid_new = NULL; + +-static void *remote_support_create(const char *layer_home, const char *layer_ro) ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro) + { + struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data)); + if (data == NULL) { +@@ -49,7 +44,7 @@ static void *remote_support_create(const char *layer_home, const char *layer_ro) + return data; + }; + +-static void remote_support_destroy(void *data) ++void remote_layer_destroy(struct remote_layer_data *data) + { + if (data == NULL) { + return; +@@ -72,10 +67,9 @@ static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_di + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_layer_data *data) + { +- struct remote_layer_data *remote_data = data; +- return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data); ++ return util_scan_subdirs(data->layer_ro, layer_walk_dir_cb, data); + } + + static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -85,6 +79,11 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + int nret = 0; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't delete NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -98,11 +97,14 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + goto out; + } + ++ // return 0 if path already removed + if (util_path_remove(clean_path) != 0) { + SYSERROR("Failed to remove link path %s", clean_path); ++ ret = -1; ++ goto out; + } + +- if (remove_memory_stores_with_lock(layer_id) != 0) { ++ if (remote_layer_remove_memory_stores_with_lock(layer_id) != 0) { + ERROR("Failed to remove remote layer store memory"); + ret = -1; + } +@@ -110,7 +112,6 @@ static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_i + out: + free(ro_symlink); + return ret; +- + } + + static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) +@@ -119,6 +120,11 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + char *layer_dir = NULL; + int ret = 0; + ++ if (layer_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->layer_home, layer_id); + layer_dir = util_path_join(data->layer_ro, layer_id); + +@@ -140,7 +146,7 @@ static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id) + goto free_out; + } + // insert layer into memory +- if (load_one_layer(layer_id) != 0) { ++ if (remote_load_one_layer(layer_id) != 0) { + ERROR("Failed to load new layer: %s into memory", layer_id); + ret = -1; + } +@@ -152,30 +158,32 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_layer_add(struct remote_layer_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(layer_byid_old, layer_byid_new); +- array_deleted = deleted_layers(layer_byid_old, layer_byid_new); ++ array_added = remote_added_layers(layer_byid_old, layer_byid_new); ++ array_deleted = remote_deleted_layers(layer_byid_old, layer_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { +- if (!overlay_remote_layer_valid(array_added[i]) != 0) { ++ if (!remote_overlay_layer_valid(array_added[i]) != 0) { ++ WARN("remote overlay layer current not valid: %s", array_added[i]); + map_remove(layer_byid_new, (void *)array_added[i]); +- ERROR("remote overlay layer current not valid: %s", array_added[i]); + continue; + } + + if (add_one_remote_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote layer: %s", array_added[i]); ++ map_remove(layer_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -183,6 +191,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(layer_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -190,7 +199,7 @@ static int remote_support_add(void *data) + tmp_map = layer_byid_old; + layer_byid_old = layer_byid_new; + layer_byid_new = tmp_map; +- empty_map(layer_byid_new); ++ map_clear(layer_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -198,22 +207,20 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *layer_store_impl_remote_support() ++void remote_layer_refresh(struct remote_layer_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote layer failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_layer_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool layer_remote_layer_valid(const char *layer_id) ++ ++bool remote_layer_layer_valid(const char *layer_id) + { + return map_search(layer_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +similarity index 72% +rename from src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c +rename to src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +index a674a00f..de2e583c 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c +@@ -13,13 +13,13 @@ + * Description: provide remote implementation for driver overlay + ******************************************************************************/ + #define _GNU_SOURCE +-#include "driver_overlay2.h" ++#include "remote_support.h" + + #include + + #include "map.h" +-#include "remote_support.h" + #include "ro_symlink_maintain.h" ++#include "driver_overlay2.h" + #include "isula_libutils/log.h" + #include "utils.h" + #include "utils_array.h" +@@ -29,15 +29,13 @@ + #define OVERLAY_LINK_DIR "l" + #define OVERLAY_LAYER_LINK "link" + +-struct remote_overlay_data { +- const char *overlay_home; +- const char *overlay_ro; +-}; +- ++// key: id, value: short id in 'l' dir ++// store short id to delete symbol link in 'l' dir + static map_t *overlay_byid_old = NULL; + static map_t *overlay_byid_new = NULL; ++static map_t *overlay_id_link = NULL; + +-static void *remote_support_create(const char *remote_home, const char *remote_ro) ++struct remote_overlay_data *remote_overlay_create(const char *remote_home, const char *remote_ro) + { + struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data)); + if (data == NULL) { +@@ -48,11 +46,12 @@ static void *remote_support_create(const char *remote_home, const char *remote_r + data->overlay_ro = remote_ro; + overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ overlay_id_link = map_new(MAP_STR_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); + + return data; + } + +-static void remote_support_destroy(void *data) ++void remote_overlay_destroy(struct remote_overlay_data *data) + { + if (data == NULL) { + return; +@@ -60,6 +59,7 @@ static void remote_support_destroy(void *data) + + map_free(overlay_byid_old); + map_free(overlay_byid_new); ++ map_free(overlay_id_link); + free(data); + } + +@@ -74,10 +74,9 @@ static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_ + return true; + } + +-static int remote_support_scan(void *data) ++static int remote_dir_scan(struct remote_overlay_data *data) + { +- struct remote_overlay_data *remote_data = data; +- return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data); ++ return util_scan_subdirs(data->overlay_ro, overlay_walk_dir_cb, data); + } + + static int do_diff_symlink(const char *id, char *link_id, const char *driver_home) +@@ -128,10 +127,17 @@ out: + static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id) + { + char *ro_symlink = NULL; ++ char *link_path = NULL; ++ char *link_id = NULL; + char clean_path[PATH_MAX] = { 0 }; + int nret = 0; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't remove NULL remote layer"); ++ return -1; ++ } ++ + nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id); + if (nret < 0 || nret > PATH_MAX) { + SYSERROR("Create layer symbol link path failed"); +@@ -149,8 +155,40 @@ static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, con + SYSERROR("Failed to remove link path %s", clean_path); + } + ++ link_id = (char *)map_search(overlay_id_link, (void *)overlay_id); ++ ++ if (link_id == NULL) { ++ ERROR("Failed to find link id for overlay layer: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ ++ nret = asprintf(&link_path, "%s/%s/%s", data->overlay_home, OVERLAY_LINK_DIR, link_id); ++ if (nret < 0 || nret > PATH_MAX) { ++ SYSERROR("Create link path failed"); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) { ++ ERROR("Failed to clean path: %s", ro_symlink); ++ ret = -1; ++ goto out; ++ } ++ ++ if (util_path_remove(clean_path) != 0) { ++ SYSERROR("Failed to remove link path %s", clean_path); ++ } ++ ++ if (!map_remove(overlay_id_link, (void *)overlay_id)) { ++ ERROR("Failed to remove link id for overlay layers: %s", overlay_id); ++ ret = -1; ++ goto out; ++ } ++ + out: + free(ro_symlink); ++ free(link_path); + return ret; + } + +@@ -162,6 +200,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + char *diff_symlink = NULL; + int ret = 0; + ++ if (overlay_id == NULL) { ++ ERROR("can't add NULL remote layer"); ++ return -1; ++ } ++ + ro_symlink = util_path_join(data->overlay_home, overlay_id); + if (ro_symlink == NULL) { + ERROR("Failed to join ro symlink path: %s", overlay_id); +@@ -211,6 +254,11 @@ static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const + ret = -1; + } + ++ if (!map_insert(overlay_id_link, util_strdup_s(overlay_id), (void *)diff_symlink)) { ++ ERROR("can't insert remote layer into map"); ++ ret = -1; ++ } ++ + free_out: + free(ro_symlink); + free(layer_dir); +@@ -220,24 +268,26 @@ free_out: + return ret; + } + +-static int remote_support_add(void *data) ++static int remote_image_add(struct remote_overlay_data *data) + { + int ret = 0; + char **array_added = NULL; + char **array_deleted = NULL; + map_t *tmp_map = NULL; ++ bool exist = true; + int i = 0; + + if (data == NULL) { + return -1; + } + +- array_added = added_layers(overlay_byid_old, overlay_byid_new); +- array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new); ++ array_added = remote_added_layers(overlay_byid_old, overlay_byid_new); ++ array_deleted = remote_deleted_layers(overlay_byid_old, overlay_byid_new); + + for (i = 0; i < util_array_len((const char **)array_added); i++) { + if (add_one_remote_overlay_layer(data, array_added[i]) != 0) { + ERROR("Failed to add remote overlay layer: %s", array_added[i]); ++ map_remove(overlay_byid_new, (void *)array_added[i]); + ret = -1; + } + } +@@ -245,6 +295,7 @@ static int remote_support_add(void *data) + for (i = 0; i < util_array_len((const char **)array_deleted); i++) { + if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) { + ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]); ++ map_insert(overlay_byid_new, array_deleted[i], (void *)&exist); + ret = -1; + } + } +@@ -252,7 +303,7 @@ static int remote_support_add(void *data) + tmp_map = overlay_byid_old; + overlay_byid_old = overlay_byid_new; + overlay_byid_new = tmp_map; +- empty_map(overlay_byid_new); ++ map_clear(overlay_byid_new); + + util_free_array(array_added); + util_free_array(array_deleted); +@@ -260,23 +311,19 @@ static int remote_support_add(void *data) + return ret; + } + +-remote_support *overlay_driver_impl_remote_support(void) ++void remote_overlay_refresh(struct remote_overlay_data *data) + { +- remote_support *rs = util_common_calloc_s(sizeof(remote_support)); +- if (rs == NULL) { +- ERROR("Failed to calloc overlay supporter"); +- return NULL; ++ if (remote_dir_scan(data) != 0) { ++ ERROR("remote overlay failed to scan dir, skip refresh"); ++ return; + } + +- rs->create = remote_support_create; +- rs->destroy = remote_support_destroy; +- rs->scan_remote_dir = remote_support_scan; +- rs->load_item = remote_support_add; +- +- return rs; ++ if (remote_image_add(data) != 0) { ++ ERROR("refresh overlay failed"); ++ } + } + +-bool overlay_remote_layer_valid(const char *layer_id) ++bool remote_overlay_layer_valid(const char *layer_id) + { + return map_search(overlay_byid_old, (void *)layer_id) != NULL; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 9dc096f7..3c7d0f54 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -15,108 +15,107 @@ + + #include "remote_support.h" + +-#include "layer_store.h" +-#include "image_store.h" ++#include ++ + #include "isula_libutils/log.h" +-#include "driver_overlay2.h" + #include "utils.h" + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = layer_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++struct supporters { ++ struct remote_image_data *image_data; ++ struct remote_layer_data *layer_data; ++ struct remote_overlay_data *overlay_data; ++}; + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; +- } ++static struct supporters supporters; ++ ++static void *remote_refresh_ro_symbol_link(void *arg) ++{ ++ struct supporters *refresh_supporters = (struct supporters *)arg; ++ prctl(PR_SET_NAME, "RoLayerRefresh"); + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ while (true) { ++ util_usleep_nointerupt(5 * 1000 * 1000); ++ DEBUG("remote refresh start\n"); + +- return supporter; ++ remote_overlay_refresh(refresh_supporters->overlay_data); ++ remote_layer_refresh(refresh_supporters->layer_data); ++ remote_image_refresh(refresh_supporters->image_data); + +-err_out: +- free(handlers); +- free(supporter); ++ DEBUG("remote refresh end\n"); ++ } + return NULL; + } + +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro) ++int remote_start_refresh_thread(void) + { +- remote_support *handlers = image_store_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; +- } ++ int res = 0; ++ pthread_t a_thread; ++ maintain_context ctx = get_maintain_context(); + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ supporters.image_data = remote_image_create(ctx.image_home, NULL); ++ if (supporters.image_data == NULL) { ++ goto free_out; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); +- +- return supporter; +- +-err_out: +- free(handlers); +- free(supporter); +- return NULL; +-} ++ supporters.layer_data = remote_layer_create(ctx.layer_home, ctx.layer_ro_dir); ++ if (supporters.layer_data == NULL) { ++ goto free_out; ++ } + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro) +-{ +- remote_support *handlers = overlay_driver_impl_remote_support(); +- if (handlers == NULL || handlers->create == NULL) { +- return NULL; ++ supporters.overlay_data = remote_overlay_create(ctx.overlay_home, ctx.overlay_ro_dir); ++ if (supporters.overlay_data == NULL) { ++ goto free_out; + } + +- remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter)); +- if (supporter == NULL) { +- goto err_out; ++ res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); ++ if (res != 0) { ++ CRIT("Thread creation failed"); ++ return -1; + } + +- supporter->handlers = handlers; +- supporter->data = handlers->create(remote_home, remote_ro); ++ if (pthread_detach(a_thread) != 0) { ++ SYSERROR("Failed to detach 0x%lx", a_thread); ++ return -1; ++ } + +- return supporter; ++ return 0; + +-err_out: +- free(handlers); +- free(supporter); +- return NULL; ++free_out: ++ remote_image_destroy(supporters.image_data); ++ remote_layer_destroy(supporters.layer_data); ++ remote_overlay_destroy(supporters.overlay_data); + ++ return -1; + } + +-void destroy_suppoter(remote_supporter *supporter) ++// this function calculate map_a - map_b => diff_list ++// diff_list contains keys inside map_a but not inside map_b ++static char **map_diff(const map_t *map_a, const map_t *map_b) + { +- if (supporter->handlers->destroy == NULL) { +- ERROR("destroy_supporter operation not supported"); +- return; ++ char **array = NULL; ++ map_itor *itor = map_itor_new(map_a); ++ bool *found = NULL; ++ ++ // iter new_map, every item not in old, append them to new_layers ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ char *id = map_itor_key(itor); ++ found = map_search(map_b, id); ++ if (found == NULL) { ++ util_array_append(&array, util_strdup_s(id)); ++ } + } + +- supporter->handlers->destroy(supporter->data); +- free(supporter->handlers); +- free(supporter); ++ map_itor_free(itor); ++ ++ return array; + } + +-int scan_remote_dir(remote_supporter *supporter) ++char **remote_deleted_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("scan_remote_dir operation not supported"); +- return -1; +- } +- return supporter->handlers->scan_remote_dir(supporter->data); ++ return map_diff(old, new); + } + +-int load_item(remote_supporter *supporter) ++char **remote_added_layers(const map_t *old, const map_t *new) + { +- if (supporter->handlers->scan_remote_dir == NULL) { +- ERROR("load_item operation not supported"); +- return -1; +- } +- return supporter->handlers->load_item(supporter->data); ++ return map_diff(new, old); + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index d1f7af35..892a9155 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -17,39 +17,59 @@ + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + + #include "linked_list.h" +-#define REMOTE_RO_LAYER_DIR "RO" +-#define OVERLAY_RO_DIR "RO" ++#include "map.h" ++#include "ro_symlink_maintain.h" + + #ifdef __cplusplus + extern "C" { + #endif + +-typedef struct { +- void *(*create)(const char *remote_home, const char *remote_ro); +- void (*destroy)(void *data); +- // populate the list contains all dirs +- int (*scan_remote_dir)(void *data); +- // consume the list contains all dirs +- int (*load_item)(void *data); +-} remote_support; ++struct remote_overlay_data { ++ const char *overlay_home; ++ const char *overlay_ro; ++}; + +-typedef struct { +- void *data; +- remote_support *handlers; +-} remote_supporter; ++struct remote_layer_data { ++ const char *layer_home; ++ const char *layer_ro; ++}; + +-// RemoteSupport *impl_remote_support(); +-remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro); ++struct remote_image_data { ++ const char *image_home; ++}; + +-remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro); ++// image impl ++struct remote_image_data *remote_image_create(const char *image_home, const char *image_ro); + +-remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro); ++void remote_image_destroy(struct remote_image_data *data); + +-void destroy_suppoter(remote_supporter *supporter); ++void remote_image_refresh(struct remote_image_data *data); + +-int scan_remote_dir(remote_supporter *supporter); ++// layer impl ++struct remote_layer_data *remote_layer_create(const char *layer_home, const char *layer_ro); + +-int load_item(remote_supporter *supporter); ++void remote_layer_destroy(struct remote_layer_data *data); ++ ++void remote_layer_refresh(struct remote_layer_data *data); ++ ++bool remote_layer_layer_valid(const char *layer_id); ++ ++// overlay impl ++struct remote_overlay_data *remote_overlay_create(const char *overlay_home, const char *overlay_ro); ++ ++void remote_overlay_destroy(struct remote_overlay_data *data); ++ ++void remote_overlay_refresh(struct remote_overlay_data *data); ++ ++bool remote_overlay_layer_valid(const char *layer_id); ++ ++// start refresh remote ++int remote_start_refresh_thread(void); ++ ++// extra map utils ++char **remote_deleted_layers(const map_t *old, const map_t *new_l); ++ ++char **remote_added_layers(const map_t *old, const map_t *new_l); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +index 7df7a221..a3aa3aa4 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c +@@ -17,19 +17,15 @@ + + #include + #include ++#include + #include + #include +-#include "map.h" ++ + #include "path.h" + #include "linked_list.h" +-#include "layer_store.h" +-#include "layer.h" + #include "isula_libutils/log.h" +-#include "image_store.h" +-#include "remote_support.h" + #include "utils.h" + #include "utils_file.h" +-#include "stdlib.h" + + #define REMOTE_RO_LAYER_DIR "RO" + +@@ -43,14 +39,6 @@ static char *layer_home; + static char *overlay_ro_dir; + static char *overlay_home; + +-struct supporters { +- remote_supporter *image_supporter; +- remote_supporter *layer_supporter; +- remote_supporter *overlay_supporter; +-}; +- +-static struct supporters supporters; +- + int remote_image_init(const char *root_dir) + { + if (root_dir == NULL) { +@@ -134,75 +122,11 @@ void remote_maintain_cleanup(void) + overlay_ro_dir = NULL; + } + +-// to maintain the symbol links, add new symbol link and delete invalid symbol link +-// arg is const char *driver_home +-// scanning driver->home/RO/ directory, build symlink in driver->home +-static void *remote_refresh_ro_symbol_link(void *arg) +-{ +- struct supporters *supporters = (struct supporters *)arg; +- prctl(PR_SET_NAME, "RoLayerRefresh"); +- +- while (true) { +- util_usleep_nointerupt(5 * 1000 * 1000); +- DEBUG("remote refresh start\n"); +- scan_remote_dir(supporters->overlay_supporter); +- load_item(supporters->overlay_supporter); +- scan_remote_dir(supporters->layer_supporter); +- load_item(supporters->layer_supporter); +- scan_remote_dir(supporters->image_supporter); +- load_item(supporters->image_supporter); +- DEBUG("remote refresh end\n"); +- } +- return NULL; +-} +- +-int start_refresh_thread(void) +-{ +- int res = 0; +- pthread_t a_thread; +- +- supporters.image_supporter = create_image_supporter(image_home, NULL); +- if (supporters.image_supporter == NULL) { +- goto free_out; +- } +- +- supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir); +- if (supporters.layer_supporter == NULL) { +- goto free_out; +- } +- +- supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir); +- if (supporters.overlay_supporter == NULL) { +- goto free_out; +- } +- +- res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); +- if (res != 0) { +- CRIT("Thread creation failed"); +- return -1; +- } +- +- if (pthread_detach(a_thread) != 0) { +- SYSERROR("Failed to detach 0x%lx", a_thread); +- return -1; +- } +- +- return 0; +- +-free_out: +- destroy_suppoter(supporters.image_supporter); +- destroy_suppoter(supporters.layer_supporter); +- destroy_suppoter(supporters.overlay_supporter); +- +- return -1; +-} +- + static int do_build_ro_dir(const char *home, const char *id) + { + char *ro_symlink = NULL; + char *ro_layer_dir = NULL; + int nret = 0; +- // bool ret = true; + int ret = 0; + + nret = asprintf(&ro_symlink, "%s/%s", home, id); +@@ -305,43 +229,15 @@ int remote_overlay_remove_ro_dir(const char *id) + return do_remove_ro_dir(overlay_home, id); + } + +-static char **map_diff(map_t *map_a, map_t *map_b) +-{ +- char **array = NULL; +- map_itor *itor = map_itor_new(map_a); +- bool *found = NULL; +- +- // iter new_map, every item not in old, append them to new_layers +- for (; map_itor_valid(itor); map_itor_next(itor)) { +- char *id = map_itor_key(itor); +- found = map_search(map_b, id); +- if (found == NULL) { +- util_array_append(&array, util_strdup_s(id)); +- } +- } +- +- map_itor_free(itor); +- +- return array; +-} +- +-char **deleted_layers(map_t *old, map_t *new) +-{ +- return map_diff(old, new); +-} +- +-char **added_layers(map_t *old, map_t *new) ++maintain_context get_maintain_context(void) + { +- return map_diff(new, old); +-} ++ maintain_context ctx = {0x0}; + +-int empty_map(map_t *mp) +-{ +- if (mp == NULL) { +- return -1; +- } ++ ctx.image_home = image_home; ++ ctx.layer_ro_dir = layer_ro_dir; ++ ctx.layer_home = layer_home; ++ ctx.overlay_ro_dir = overlay_ro_dir; ++ ctx.overlay_home = overlay_home; + +- map_clear(mp); +- mp->store->root = mp->store->nil; +- return 0; ++ return ctx; + } +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +index 25712d40..aa2036ea 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h +@@ -15,12 +15,21 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H + +-#include "map.h" ++#define REMOTE_RO_LAYER_DIR "RO" ++#define OVERLAY_RO_DIR "RO" + + #ifdef __cplusplus + extern "C" { + #endif + ++typedef struct { ++ const char *image_home; ++ const char *layer_ro_dir; ++ const char *layer_home; ++ const char *overlay_ro_dir; ++ const char *overlay_home; ++} maintain_context; ++ + int remote_image_init(const char *root_dir); + + int remote_layer_init(const char *root_dir); +@@ -29,8 +38,6 @@ int remote_overlay_init(const char *driver_home); + + void remote_maintain_cleanup(void); + +-int start_refresh_thread(void); +- + int remote_layer_build_ro_dir(const char *id); + + int remote_overlay_build_ro_dir(const char *id); +@@ -39,11 +46,7 @@ int remote_layer_remove_ro_dir(const char *id); + + int remote_overlay_remove_ro_dir(const char *id); + +-char **deleted_layers(map_t *old, map_t *new); +- +-char **added_layers(map_t *old, map_t *new); +- +-int empty_map(map_t *mp); ++maintain_context get_maintain_context(void); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 31812a22..f9830ac3 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -43,7 +43,7 @@ + #include "utils_verify.h" + #include "sha256.h" + #ifdef ENABLE_REMOTE_LAYER_STORE +-#include "ro_symlink_maintain.h" ++#include "remote_support.h" + #endif + + static pthread_rwlock_t g_storage_rwlock; +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/utils/cutils/map/rb_tree.c b/src/utils/cutils/map/rb_tree.c +index e933003a..b85371e3 100644 +--- a/src/utils/cutils/map/rb_tree.c ++++ b/src/utils/cutils/map/rb_tree.c +@@ -134,6 +134,7 @@ void rbtree_clear(rb_tree_t *tree) + return; + } + rbtree_destroy_all(tree, tree->root); ++ tree->root = tree->nil; + } + + void rbtree_free(rb_tree_t *tree) +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 31240ae0..df184b09 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -26,6 +26,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry_apiv2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/http_request.c +@@ -56,6 +57,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks + ) +diff --git a/test/image/oci/storage/CMakeLists.txt b/test/image/oci/storage/CMakeLists.txt +index feb81b14..7a9d77fe 100644 +--- a/test/image/oci/storage/CMakeLists.txt ++++ b/test/image/oci/storage/CMakeLists.txt +@@ -3,3 +3,6 @@ project(iSulad_UT) + add_subdirectory(images) + add_subdirectory(rootfs) + add_subdirectory(layers) ++IF (ENABLE_REMOTE_LAYER_STORE) ++add_subdirectory(remote_layer_support) ++ENDIF() +diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt +index 3e6b69a4..8446ebba 100644 +--- a/test/image/oci/storage/images/CMakeLists.txt ++++ b/test/image/oci/storage/images/CMakeLists.txt +@@ -21,6 +21,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry_type.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_images_ut.cc) +@@ -39,6 +40,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/registry + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt +index 952e9483..ae0ac9c3 100644 +--- a/test/image/oci/storage/layers/CMakeLists.txt ++++ b/test/image/oci/storage/layers/CMakeLists.txt +@@ -30,6 +30,7 @@ add_executable(${DRIVER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_driver_ut.cc) +@@ -52,6 +53,7 @@ target_include_directories(${DRIVER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) +@@ -105,6 +107,7 @@ add_executable(${LAYER_EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/wrapper_devmapper.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota/project_quota.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/driver_quota_mock.cc + storage_layers_ut.cc) + +@@ -129,6 +132,7 @@ target_include_directories(${LAYER_EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/quota ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks + ) + +diff --git a/test/image/oci/storage/remote_layer_support/CMakeLists.txt b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +new file mode 100644 +index 00000000..c36d9049 +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/CMakeLists.txt +@@ -0,0 +1,44 @@ ++project(iSulad_UT) ++ ++SET(EXE remote_layer_support_ut) ++ ++add_executable(${EXE} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/remote_store_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/map/map.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/overlay_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/image_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/layer_remote_impl.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_file.c ++ remote_layer_ut.cc ++ ) ++ ++target_include_directories(${EXE} PUBLIC ++ ${GTEST_INCLUDE_DIR} ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../include ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/image_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2 ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/layer_store/graphdriver ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks ++ ) ++ ++target_link_libraries(${EXE} ++ ${GTEST_BOTH_LIBRARIES} ++ ${GMOCK_LIBRARY} ++ ${GMOCK_MAIN_LIBRARY} ++ ${CMAKE_THREAD_LIBS_INIT} ++ ${ISULA_LIBUTILS_LIBRARY} ++ -lgtest -lgtest_main libutils_ut -lcrypto -lyajl -lz) ++ ++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) ++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +diff --git a/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +new file mode 100644 +index 00000000..5f5e92fb +--- /dev/null ++++ b/test/image/oci/storage/remote_layer_support/remote_layer_ut.cc +@@ -0,0 +1,93 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-16 ++ * Description: provide remote layer support ut ++ ******************************************************************************/ ++#include ++ ++#include "remote_store_mock.h" ++#include "ro_symlink_maintain.h" ++#include "remote_support.h" ++#include "map.h" ++ ++using ::testing::Invoke; ++ ++bool invokeOverlayRemoteLayerValid(const char *id) ++{ ++ return true; /* currently always valid overlay layer */ ++} ++ ++bool invokeLayerRemoteLayerValid(const char *id) ++{ ++ return true; ++} ++ ++int invokeLayerLoadOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeLayerRemoveOneLayer(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageAppendOneImage(const char *id) ++{ ++ return 0; ++} ++ ++int invokeImageRemoveOneImage(const char *id) ++{ ++ return 0; ++} ++ ++char *invokeImageGetTopLayer(const char *id) ++{ ++ return NULL; ++} ++ ++int invokeImageValidSchemaV1(const char *path, bool *valid) ++{ ++ return 0; ++} ++ ++void mockCommonAll(MockRemoteStore *mock) ++{ ++ EXPECT_CALL(*mock, LayerLoadOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerLoadOneLayer)); ++ EXPECT_CALL(*mock, LayerRemoveOneLayer(::testing::_)).WillRepeatedly(Invoke(invokeLayerRemoveOneLayer)); ++ ++ EXPECT_CALL(*mock, ImageAppendOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageAppendOneImage)); ++ EXPECT_CALL(*mock, ImageRemoveOneImage(::testing::_)).WillRepeatedly(Invoke(invokeImageRemoveOneImage)); ++ EXPECT_CALL(*mock, ImageGetTopLayer(::testing::_)).WillRepeatedly(Invoke(invokeImageGetTopLayer)); ++ EXPECT_CALL(*mock, ImageValidSchemaV1(::testing::_, ::testing::_)).WillRepeatedly(Invoke(invokeImageValidSchemaV1)); ++} ++ ++TEST(remote_Layer_ut, test_map_diff) ++{ ++ // old: a b x ++ // new: x b c ++ map_t *old_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ map_t *new_one = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); ++ bool exist = true; ++ ++ map_insert(old_one, (void *)"a", (void *)&exist); ++ map_insert(old_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"b", (void *)&exist); ++ map_insert(new_one, (void *)"c", (void *)&exist); ++ ++ char **added = remote_added_layers(old_one, new_one); ++ char **deleted = remote_deleted_layers(old_one, new_one); ++ ++ ASSERT_EQ(added[0][0], 'c'); ++ ASSERT_EQ(deleted[0][0], 'a'); ++} +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 4d7d3533..5383fbac 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -22,6 +22,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/utils_images.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../mocks/isulad_config_mock.cc + storage_rootfs_ut.cc) +@@ -41,6 +42,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/rootfs_store ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image/oci/storage/remote_layer_support + ) + + target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) +diff --git a/test/mocks/remote_store_mock.cc b/test/mocks/remote_store_mock.cc +new file mode 100644 +index 00000000..c6428623 +--- /dev/null ++++ b/test/mocks/remote_store_mock.cc +@@ -0,0 +1,68 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#include "remote_store_mock.h" ++ ++namespace { ++MockRemoteStore *g_remote_store_mock = nullptr; ++} ++ ++int remote_load_one_layer(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerLoadOneLayer(id); ++ } ++ return -1; ++} ++ ++int remote_layer_remove_memory_stores_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->LayerRemoveOneLayer(id); ++ } ++ return -1; ++} ++ ++int image_store_validate_manifest_schema_version_1(const char *path, bool *valid) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageValidSchemaV1(path, valid); ++ } ++ return -1; ++} ++ ++int remote_append_image_by_directory_with_lock(const char *image_dir) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageAppendOneImage(image_dir); ++ } ++ return -1; ++} ++ ++int remote_remove_image_from_memory_with_lock(const char *id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageRemoveOneImage(id); ++ } ++ return -1; ++} ++ ++char *remote_image_get_top_layer_from_json(const char *img_id) ++{ ++ if (g_remote_store_mock != nullptr) { ++ return g_remote_store_mock->ImageGetTopLayer(img_id); ++ } ++ return nullptr; ++} +diff --git a/test/mocks/remote_store_mock.h b/test/mocks/remote_store_mock.h +new file mode 100644 +index 00000000..e8d73ef1 +--- /dev/null ++++ b/test/mocks/remote_store_mock.h +@@ -0,0 +1,40 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2023-03-09 ++ * Description: provide mock for image store, layer store and driver overlay ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++#define _ISULAD_TEST_MOCKS_REMOTE_STORE_MOCK_H ++ ++#include ++ ++#include "image_store.h" ++#include "layer_store.h" ++#include "driver_overlay2.h" ++ ++class MockRemoteStore { ++public: ++ virtual ~MockRemoteStore() = default; ++ // MOCK_METHOD1(OverlayRemoteLayerValid, bool(const char *)); ++ ++ // MOCK_METHOD1(LayerRemoteLayerValid, bool(const char *)); ++ MOCK_METHOD1(LayerLoadOneLayer, int(const char *)); ++ MOCK_METHOD1(LayerRemoveOneLayer, int(const char *)); ++ ++ MOCK_METHOD1(ImageAppendOneImage, int(const char *)); ++ MOCK_METHOD1(ImageRemoveOneImage, int(const char *)); ++ MOCK_METHOD1(ImageGetTopLayer, char *(const char *)); ++ MOCK_METHOD2(ImageValidSchemaV1, int(const char *, bool *)); ++}; ++ ++#endif // _ISULAD_TEST_MOCKS_IMAGE_MOCK_H +-- +2.25.1 + diff --git a/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch b/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch new file mode 100644 index 0000000..37a3bae --- /dev/null +++ b/0055-bugfix-when-refresh-can-t-load-or-pull-images.patch @@ -0,0 +1,319 @@ +From e15b302f7001507ebbf37d5485143f3124f8f54f Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Mon, 20 Mar 2023 23:47:25 -0700 +Subject: [PATCH 55/56] bugfix when refresh can't load or pull images + +Signed-off-by: Neil.wrz +--- + src/daemon/modules/image/oci/oci_image.c | 105 +++++++++++++++++- + .../remote_layer_support/remote_support.c | 34 +++++- + .../remote_layer_support/remote_support.h | 4 +- + .../modules/image/oci/storage/storage.c | 2 +- + .../modules/image/oci/storage/storage.h | 2 + + 5 files changed, 143 insertions(+), 4 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index e4442a79..06a0c6ac 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -41,6 +41,39 @@ + + struct oci_image_module_data g_oci_image_module_data = { 0 }; + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++// intend to make remote refresh and oci ops exlusive ++static bool g_enable_remote; ++static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER; ++ ++static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++#endif ++ + static void free_oci_image_data(void) + { + free(g_oci_image_module_data.root_dir); +@@ -216,6 +249,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args) + + #ifdef ENABLE_REMOTE_LAYER_STORE + storage_opts->enable_remote_layer = args->storage_enable_remote_layer; ++ storage_opts->remote_lock = &g_remote_lock; + #endif + + if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts, +@@ -299,6 +333,10 @@ int oci_init(const isulad_daemon_configs *args) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ g_enable_remote = args->storage_enable_remote_layer; ++#endif ++ + if (storage_module_init_helper(args) != 0) { + ret = -1; + goto out; +@@ -317,6 +355,7 @@ void oci_exit() + + int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + { ++ int ret = 0; + if (request == NULL || request->image == NULL || response == NULL) { + ERROR("Invalid NULL param"); + return -1; +@@ -327,8 +366,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response) + isulad_try_set_error_message("Invalid image name: %s", request->image); + return -1; + } ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because pull have exclusive access against remote refresh ++ // pull can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ ++ ret = oci_do_pull_image(request, response); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + +- return oci_do_pull_image(request, response); ++ return ret; + } + + int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs) +@@ -437,6 +492,15 @@ int oci_rmi(const im_rmi_request *request) + return -1; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ return -1; ++ } ++#endif ++ + if (!util_valid_image_name(request->image.image)) { + ERROR("Invalid image name: %s", request->image.image); + isulad_try_set_error_message("Invalid image name: %s", request->image.image); +@@ -498,6 +562,11 @@ int oci_rmi(const im_rmi_request *request) + } + + out: ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif + free(real_image_name); + free(image_ID); + util_free_array_by_len(image_names, image_names_len); +@@ -523,7 +592,24 @@ int oci_import(const im_import_request *request, char **id) + goto err_out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because import have exclusive access against remote refresh ++ // import can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto err_out; ++ } ++#endif ++ + ret = oci_do_import(request->file, dest_name, id); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + goto err_out; + } +@@ -673,7 +759,24 @@ int oci_load_image(const im_load_request *request) + goto out; + } + ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ // read lock here because load have exclusive access against remote refresh ++ // load can work concurrently with other oci operations. ++ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) { ++ ERROR("Failed to lock oci remote lock when load image"); ++ ret = -1; ++ goto out; ++ } ++#endif ++ + ret = oci_do_load(request); ++ ++#ifdef ENABLE_REMOTE_LAYER_STORE ++ if (g_enable_remote) { ++ oci_remote_unlock(&g_remote_lock); ++ } ++#endif ++ + if (ret != 0) { + ERROR("Failed to load image"); + goto out; +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +index 3c7d0f54..7d457755 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c +@@ -24,10 +24,38 @@ struct supporters { + struct remote_image_data *image_data; + struct remote_layer_data *layer_data; + struct remote_overlay_data *overlay_data; ++ pthread_rwlock_t *remote_lock; + }; + + static struct supporters supporters; + ++static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable) ++{ ++ int nret = 0; ++ ++ if (writable) { ++ nret = pthread_rwlock_wrlock(remote_lock); ++ } else { ++ nret = pthread_rwlock_rdlock(remote_lock); ++ } ++ if (nret != 0) { ++ ERROR("Lock memory store failed: %s", strerror(nret)); ++ return false; ++ } ++ ++ return true; ++} ++ ++static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock) ++{ ++ int nret = 0; ++ ++ nret = pthread_rwlock_unlock(remote_lock); ++ if (nret != 0) { ++ FATAL("Unlock memory store failed: %s", strerror(nret)); ++ } ++} ++ + static void *remote_refresh_ro_symbol_link(void *arg) + { + struct supporters *refresh_supporters = (struct supporters *)arg; +@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg) + util_usleep_nointerupt(5 * 1000 * 1000); + DEBUG("remote refresh start\n"); + ++ remote_refresh_lock(supporters.remote_lock, true); + remote_overlay_refresh(refresh_supporters->overlay_data); + remote_layer_refresh(refresh_supporters->layer_data); + remote_image_refresh(refresh_supporters->image_data); ++ remote_refresh_unlock(supporters.remote_lock); + + DEBUG("remote refresh end\n"); + } + return NULL; + } + +-int remote_start_refresh_thread(void) ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock) + { + int res = 0; + pthread_t a_thread; +@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void) + goto free_out; + } + ++ supporters.remote_lock = remote_lock; ++ + res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters); + if (res != 0) { + CRIT("Thread creation failed"); +diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +index 892a9155..30e3ebb0 100644 +--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h ++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h +@@ -16,6 +16,8 @@ + #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H + ++#include ++ + #include "linked_list.h" + #include "map.h" + #include "ro_symlink_maintain.h" +@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data); + bool remote_overlay_layer_valid(const char *layer_id); + + // start refresh remote +-int remote_start_refresh_thread(void); ++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock); + + // extra map utils + char **remote_deleted_layers(const map_t *old, const map_t *new_l); +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index f9830ac3..836ccf4d 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts) + } + + #ifdef ENABLE_REMOTE_LAYER_STORE +- if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) { ++ if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) { + ERROR("Failed to start remote refresh thread"); + } + #endif +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 7404ee54..df9fd761 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -72,6 +73,7 @@ struct storage_module_init_options { + bool integration_check; + #ifdef ENABLE_REMOTE_LAYER_STORE + bool enable_remote_layer; ++ pthread_rwlock_t *remote_lock; + #endif + }; + +-- +2.25.1 + diff --git a/0056-remove-unused-headers.patch b/0056-remove-unused-headers.patch new file mode 100644 index 0000000..b37b9f4 --- /dev/null +++ b/0056-remove-unused-headers.patch @@ -0,0 +1,40 @@ +From 2b798cf4053298dc44304319073cda1a00a466f4 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Thu, 27 Apr 2023 00:26:15 -0700 +Subject: [PATCH 56/56] remove unused headers + +Signed-off-by: Neil.wrz +--- + src/daemon/modules/image/oci/storage/layer_store/layer_store.h | 1 - + src/daemon/modules/image/oci/storage/storage.h | 2 ++ + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +index 4677e5ee..be8c52dc 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h +@@ -23,7 +23,6 @@ + + #include "storage.h" + #include "io_wrapper.h" +-#include "map.h" + + struct io_read_wrapper; + struct layer_list; +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index df9fd761..a761938c 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -18,7 +18,9 @@ + #include + #include + #include ++#ifdef ENABLE_REMOTE_LAYER_STORE + #include ++#endif + #include + #include + +-- +2.25.1 + diff --git a/0057-change-isulad-shim-epoll-struct.patch b/0057-change-isulad-shim-epoll-struct.patch new file mode 100644 index 0000000..a270d9d --- /dev/null +++ b/0057-change-isulad-shim-epoll-struct.patch @@ -0,0 +1,1793 @@ +From 15275529aed5b4f2e317902dfd3390ee114b310f Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sun, 16 Apr 2023 13:56:18 +0800 +Subject: [PATCH 57/57] change isulad-shim epoll struct + +Signed-off-by: zhongtao +--- + src/cmd/isulad-shim/common.c | 9 + + src/cmd/isulad-shim/common.h | 2 + + src/cmd/isulad-shim/linked_list.h | 134 ++++ + src/cmd/isulad-shim/main.c | 13 +- + src/cmd/isulad-shim/mainloop.c | 168 +++++ + src/cmd/isulad-shim/mainloop.h | 53 ++ + src/cmd/isulad-shim/process.c | 941 ++++++++++++------------- + src/cmd/isulad-shim/process.h | 47 +- + test/cmd/isulad-shim/CMakeLists.txt | 1 + + test/cmd/isulad-shim/isulad-shim_ut.cc | 1 + + 10 files changed, 825 insertions(+), 544 deletions(-) + create mode 100644 src/cmd/isulad-shim/linked_list.h + create mode 100644 src/cmd/isulad-shim/mainloop.c + create mode 100644 src/cmd/isulad-shim/mainloop.h + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index e1ca96e1..18443385 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -524,4 +524,13 @@ err_out: + util_free_array(res_array); + errno = tmp_errno; + return NULL; ++} ++ ++void *util_common_calloc_s(size_t size) ++{ ++ if (size == 0 || size > MAX_MEMORY_SIZE) { ++ return NULL; ++ } ++ ++ return calloc((size_t)1, size); + } +\ No newline at end of file +diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h +index 3de16ace..d733823b 100644 +--- a/src/cmd/isulad-shim/common.h ++++ b/src/cmd/isulad-shim/common.h +@@ -128,6 +128,8 @@ char *util_strdup_s(const char *src); + + char **util_string_split_multi(const char *src_str, char delim); + ++void *util_common_calloc_s(size_t size); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/cmd/isulad-shim/linked_list.h b/src/cmd/isulad-shim/linked_list.h +new file mode 100644 +index 00000000..3a87990a +--- /dev/null ++++ b/src/cmd/isulad-shim/linked_list.h +@@ -0,0 +1,134 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: tanyifeng ++ * Create: 2018-11-08 ++ * Description: provide container linked list function definition ++ ******************************************************************************/ ++#ifndef __LINKED_LIST_H ++#define __LINKED_LIST_H ++ ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct linked_list { ++ void *elem; ++ struct linked_list *next; ++ struct linked_list *prev; ++}; ++ ++/* Iterate through an linked list. */ ++#define linked_list_for_each(__iterator, __list) \ ++ for ((__iterator) = (__list)->next; \ ++ (__iterator) != (__list); \ ++ (__iterator) = (__iterator)->next) ++ ++/* Iterate safely through an linked list. */ ++#define linked_list_for_each_safe(__iterator, __list, __next) \ ++ for ((__iterator) = (__list)->next, (__next) = (__iterator)->next; \ ++ (__iterator) != (__list); \ ++ (__iterator) = (__next), (__next) = (__next)->next) ++ ++/* Initialize list. */ ++static inline void linked_list_init(struct linked_list *list) ++{ ++ list->elem = NULL; ++ list->next = list->prev = list; ++} ++ ++/* Add an element to a list. See linked_list_add() and linked_list_add_tail() for an ++ * idiom. */ ++static inline void linked_list_add_elem(struct linked_list *list, void *elem) ++{ ++ list->elem = elem; ++} ++ ++/* Retrieve first element of list. */ ++static inline void *linked_list_first_elem(struct linked_list *list) ++{ ++ return list->next->elem; ++} ++ ++/* Retrieve last element of list. */ ++static inline void *linked_list_last_elem(struct linked_list *list) ++{ ++ return list->prev->elem; ++} ++ ++/* Retrieve first node of list. */ ++static inline void *linked_list_first_node(struct linked_list *list) ++{ ++ return list->next; ++} ++ ++/* Determine if list is empty. */ ++static inline int linked_list_empty(struct linked_list *list) ++{ ++ return list == list->next; ++} ++ ++/* Workhorse to be called from linked_list_add() and linked_list_add_tail(). */ ++static inline void __linked_list_add(struct linked_list *newlist, ++ struct linked_list *prev, ++ struct linked_list *next) ++{ ++ next->prev = newlist; ++ newlist->next = next; ++ newlist->prev = prev; ++ prev->next = newlist; ++} ++ ++/* Idiom to add an element to the beginning of an linked list */ ++static inline void linked_list_add(struct linked_list *head, ++ struct linked_list *list) ++{ ++ __linked_list_add(list, head, head->next); ++} ++ ++/* Idiom to add an element to the end of an linked list */ ++static inline void linked_list_add_tail(struct linked_list *head, ++ struct linked_list *list) ++{ ++ __linked_list_add(list, head->prev, head); ++} ++ ++/* Idiom to free an linked list */ ++static inline void linked_list_del(const struct linked_list *list) ++{ ++ struct linked_list *next = NULL; ++ struct linked_list *prev = NULL; ++ ++ next = list->next; ++ prev = list->prev; ++ next->prev = prev; ++ prev->next = next; ++} ++ ++/* Return length of the list. */ ++static inline size_t linked_list_len(struct linked_list *list) ++{ ++ size_t i = 0; ++ struct linked_list *iter = NULL; ++ linked_list_for_each(iter, list) { ++ i++; ++ } ++ ++ return i; ++} ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif ++ +diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c +index 68e99e53..0cccdf06 100644 +--- a/src/cmd/isulad-shim/main.c ++++ b/src/cmd/isulad-shim/main.c +@@ -105,9 +105,9 @@ int main(int argc, char **argv) + int ret = SHIM_ERR; + int efd = -1; + process_t *p = NULL; +- pthread_t tid_accept; + // execSync timeout + uint64_t timeout = 0; ++ pthread_t tid_epoll; + + g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); + if (g_log_fd < 0) { +@@ -154,18 +154,13 @@ int main(int argc, char **argv) + } + } + +- /* create main loop and start epoll for io copy */ +- ret = process_io_init(p); ++ /* start epoll for io copy */ ++ ret = process_io_start(p, &tid_epoll); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret); + exit(EXIT_FAILURE); + } + +- ret = open_io(p, &tid_accept); +- if (ret != SHIM_OK) { +- exit(EXIT_FAILURE); +- } +- + ret = create_process(p); + if (ret != SHIM_OK) { + if (p->console_sock_path != NULL) { +@@ -176,5 +171,5 @@ int main(int argc, char **argv) + + released_timeout_exit(); + +- return process_signal_handle_routine(p, tid_accept, timeout); ++ return process_signal_handle_routine(p, tid_epoll, timeout); + } +diff --git a/src/cmd/isulad-shim/mainloop.c b/src/cmd/isulad-shim/mainloop.c +new file mode 100644 +index 00000000..53c4d856 +--- /dev/null ++++ b/src/cmd/isulad-shim/mainloop.c +@@ -0,0 +1,168 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: tanyifeng ++ * Create: 2018-11-08 ++ * Description: provide container mainloop functions ++ ******************************************************************************/ ++#include "mainloop.h" ++#include ++#include ++#include ++#include ++#include ++ ++#include "common.h" ++ ++struct epoll_loop_handler { ++ epoll_loop_callback_t cb; ++ int cbfd; ++ void *cbdata; ++}; ++ ++#define MAX_EVENTS 100 ++ ++/* epoll loop */ ++int epoll_loop(struct epoll_descr *descr, int t) ++{ ++ int i; ++ int ret = 0; ++ struct epoll_loop_handler *epoll_handler = NULL; ++ struct epoll_event evs[MAX_EVENTS]; ++ ++ while (1) { ++ int ep_fds = epoll_wait(descr->fd, evs, MAX_EVENTS, t); ++ if (ep_fds < 0) { ++ if (errno == EINTR) { ++ continue; ++ } ++ ret = -1; ++ goto out; ++ } ++ ++ for (i = 0; i < ep_fds; i++) { ++ epoll_handler = (struct epoll_loop_handler *)(evs[i].data.ptr); ++ if (epoll_handler->cb(epoll_handler->cbfd, evs[i].events, epoll_handler->cbdata, descr) != ++ EPOLL_LOOP_HANDLE_CONTINUE) { ++ goto out; ++ } ++ } ++ ++ if (ep_fds == 0 && t != 0) { ++ if (descr->timeout_cb != NULL) { ++ descr->timeout_cb(descr->timeout_cbdata); ++ } ++ goto out; ++ } ++ ++ if (linked_list_empty(&descr->handler_list)) { ++ goto out; ++ } ++ } ++out: ++ return ret; ++} ++ ++/* epoll loop add handler */ ++int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data) ++{ ++ struct epoll_event ev = { 0 }; ++ struct epoll_loop_handler *epoll_handler = NULL; ++ struct linked_list *node = NULL; ++ ++ // if fd == -1; dose not add handler for it. ++ if (fd < 0) { ++ return 0; ++ } ++ ++ epoll_handler = util_common_calloc_s(sizeof(*epoll_handler)); ++ if (epoll_handler == NULL) { ++ goto fail_out; ++ } ++ ++ epoll_handler->cbfd = fd; ++ epoll_handler->cb = callback; ++ epoll_handler->cbdata = data; ++ ++ ev.events = EPOLLIN; ++ ev.data.ptr = epoll_handler; ++ ++ if (epoll_ctl(descr->fd, EPOLL_CTL_ADD, fd, &ev) < 0) { ++ goto fail_out; ++ } ++ ++ node = util_common_calloc_s(sizeof(struct linked_list)); ++ if (node == NULL) { ++ goto fail_out; ++ } ++ ++ node->elem = epoll_handler; ++ linked_list_add(&descr->handler_list, node); ++ return 0; ++ ++fail_out: ++ (void)epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, &ev); ++ free(epoll_handler); ++ return -1; ++} ++ ++/* epoll loop del handler */ ++int epoll_loop_del_handler(struct epoll_descr *descr, int fd) ++{ ++ struct epoll_loop_handler *epoll_handler = NULL; ++ struct linked_list *index = NULL; ++ ++ linked_list_for_each(index, &descr->handler_list) { ++ epoll_handler = index->elem; ++ ++ if (fd == epoll_handler->cbfd) { ++ if (epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, NULL)) { ++ goto fail_out; ++ } ++ ++ linked_list_del(index); ++ free(index->elem); ++ free(index); ++ return 0; ++ } ++ } ++ ++fail_out: ++ return -1; ++} ++ ++/* epoll loop open */ ++int epoll_loop_open(struct epoll_descr *descr) ++{ ++ descr->fd = epoll_create1(EPOLL_CLOEXEC); ++ if (descr->fd < 0) { ++ return -1; ++ } ++ ++ linked_list_init(&(descr->handler_list)); ++ descr->timeout_cb = NULL; ++ descr->timeout_cbdata = NULL; ++ return 0; ++} ++ ++/* epoll loop close */ ++int epoll_loop_close(struct epoll_descr *descr) ++{ ++ struct linked_list *index = NULL; ++ struct linked_list *next = NULL; ++ ++ linked_list_for_each_safe(index, &(descr->handler_list), next) { ++ linked_list_del(index); ++ free(index->elem); ++ free(index); ++ } ++ ++ return close(descr->fd); ++} +diff --git a/src/cmd/isulad-shim/mainloop.h b/src/cmd/isulad-shim/mainloop.h +new file mode 100644 +index 00000000..7a4f1cfd +--- /dev/null ++++ b/src/cmd/isulad-shim/mainloop.h +@@ -0,0 +1,53 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: tanyifeng ++ * Create: 2018-11-08 ++ * Description: provide container mainloop definition ++ ******************************************************************************/ ++#ifndef UTILS_CUTILS_MAINLOOP_H ++#define UTILS_CUTILS_MAINLOOP_H ++ ++#include ++#include "linked_list.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef void (*epoll_timeout_callback_t)(void *data); ++ ++struct epoll_descr { ++ int fd; ++ struct linked_list handler_list; ++ epoll_timeout_callback_t timeout_cb; ++ void *timeout_cbdata; ++}; ++ ++#define EPOLL_LOOP_HANDLE_CONTINUE 0 ++#define EPOLL_LOOP_HANDLE_CLOSE 1 ++ ++typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data, struct epoll_descr *descr); ++ ++extern int epoll_loop(struct epoll_descr *descr, int t); ++ ++extern int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data); ++ ++extern int epoll_loop_del_handler(struct epoll_descr *descr, int fd); ++ ++extern int epoll_loop_open(struct epoll_descr *descr); ++ ++extern int epoll_loop_close(struct epoll_descr *descr); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index a5e0bd39..c31f8af6 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -28,6 +28,7 @@ + #include + #include + #include ++#include + #include // IWYU pragma: keep + #include // IWYU pragma: keep + #include +@@ -37,6 +38,7 @@ + + #include "common.h" + #include "terminal.h" ++#include "mainloop.h" + + #define MAX_EVENTS 100 + #define DEFAULT_IO_COPY_BUF (16 * 1024) +@@ -78,7 +80,7 @@ static int receive_fd(int sock) + u_char *pfd = NULL; + int fd = -1; + int cmsgsize = CMSG_LEN(sizeof(int)); +- struct cmsghdr *cmptr = (struct cmsghdr *)calloc(1, cmsgsize); ++ struct cmsghdr *cmptr = (struct cmsghdr *)util_common_calloc_s(cmsgsize); + if (cmptr == NULL) { + return -1; + } +@@ -128,107 +130,6 @@ static bool check_fd(int fd) + return true; + } + +-static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to) +-{ +- int ret = SHIM_ERR; +- +- if (io_thd == NULL || io_thd->ioc == NULL) { +- return SHIM_ERR; +- } +- +- io_copy_t *ioc = io_thd->ioc; +- +- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { +- return SHIM_ERR; +- } +- /* add src fd */ +- if (from != -1 && ioc->fd_from == -1) { +- ioc->fd_from = from; +- struct epoll_event ev; +- ev.events = EPOLLIN; +- ev.data.ptr = io_thd; +- +- ret = epoll_ctl(epfd, EPOLL_CTL_ADD, from, &ev); +- if (ret != SHIM_OK) { +- write_message(g_log_fd, ERR_MSG, "add fd %d to epoll loop failed:%d", from, SHIM_SYS_ERR(errno)); +- pthread_mutex_unlock(&(ioc->mutex)); +- return SHIM_ERR; +- } +- } +- +- /* add dest fd */ +- if (to != -1) { +- /* new fd_node_t for dest fd */ +- fd_node_t *fn = (fd_node_t *)calloc(1, sizeof(fd_node_t)); +- if (fn == NULL) { +- pthread_mutex_unlock(&(ioc->mutex)); +- return SHIM_ERR; +- } +- fn->fd = to; +- fn->is_log = false; +- if (io_thd->terminal != NULL && to == io_thd->terminal->fd) { +- fn->is_log = true; +- } +- fn->next = NULL; +- +- if (ioc->fd_to == NULL) { +- ioc->fd_to = fn; +- } else { +- fd_node_t *tmp = ioc->fd_to; +- while (tmp->next != NULL) { +- tmp = tmp->next; +- } +- tmp->next = fn; +- } +- } +- pthread_mutex_unlock(&(ioc->mutex)); +- +- return SHIM_OK; +-} +- +-static void remove_io_dispatch(io_thread_t *io_thd, int from, int to) +-{ +- if (io_thd == NULL || io_thd->ioc == NULL) { +- return; +- } +- io_copy_t *ioc = io_thd->ioc; +- +- fd_node_t *tmp = NULL; +- do { +- /* remove src fd */ +- if (from != -1 && from == ioc->fd_from) { +- struct epoll_event ev; +- ev.events = EPOLLIN; +- ev.data.fd = ioc->fd_from; +- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); +- } +- +- /* remove dest fd */ +- if (ioc->fd_to == NULL) { +- break; +- } +- if (ioc->fd_to->fd == to) { +- /* remove the first fd node */ +- tmp = ioc->fd_to; +- ioc->fd_to = ioc->fd_to->next; +- break; +- } +- fd_node_t *pre = ioc->fd_to; +- tmp = ioc->fd_to->next; +- while (tmp != NULL && tmp->fd != to) { +- pre = tmp; +- tmp = tmp->next; +- } +- if (tmp != NULL) { +- pre->next = tmp->next; +- } +- } while (0); +- if (tmp != NULL) { +- free(tmp); +- tmp = NULL; +- } +-} +- + static int get_exec_winsize(const char *buf, struct winsize *wsize) + { + char **array = NULL; +@@ -261,347 +162,287 @@ out: + return ret; + } + +-static void *do_io_copy(void *data) ++static int sync_exit_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { +- io_thread_t *io_thd = (io_thread_t *)data; +- if (io_thd == NULL || io_thd->ioc == NULL) { +- return NULL; +- } +- io_copy_t *ioc = io_thd->ioc; +- char *buf = calloc(1, DEFAULT_IO_COPY_BUF + 1); +- if (buf == NULL) { +- return NULL; +- } ++ return EPOLL_LOOP_HANDLE_CLOSE; ++} + +- for (;;) { +- memset(buf, 0, DEFAULT_IO_COPY_BUF); +- (void)sem_wait(&(io_thd->sem_thd)); +- if (io_thd->is_stdin && io_thd->shutdown) { +- break; +- } ++static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) ++{ ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ int w_count = 0; ++ int *fd_to = NULL; + +- int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); +- if (r_count == -1) { +- if (errno == EAGAIN || errno == EINTR) { +- continue; +- } +- break; +- } else if (r_count == 0) { +- /* End of file. The remote has closed the connection */ +- break; +- } else if (ioc->id != EXEC_RESIZE) { +- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { +- continue; +- } ++ if (events & EPOLLHUP) { ++ return EPOLL_LOOP_HANDLE_CLOSE; ++ } + +- fd_node_t *fn = ioc->fd_to; +- fd_node_t *next = NULL; +- for (; fn != NULL; fn = next) { +- next = fn->next; +- if (fn->is_log) { +- shim_write_container_log_file(io_thd->terminal, ioc->id, buf, r_count); +- } else { +- int w_count = write_nointr_in_total(fn->fd, buf, r_count); +- if (w_count < 0) { +- /* When any error occurs, remove the write fd */ +- remove_io_dispatch(io_thd, -1, fn->fd); +- } +- } +- } +- pthread_mutex_unlock(&(ioc->mutex)); +- } else { +- if (pthread_mutex_lock(&(ioc->mutex)) != 0) { +- continue; +- } ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } + +- int resize_fd = ioc->fd_to->fd; +- struct winsize wsize = { 0x00 }; +- if (get_exec_winsize(buf, &wsize) < 0) { +- break; +- } +- if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { +- break; +- } +- pthread_mutex_unlock(&(ioc->mutex)); +- } ++ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + +- /* +- In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe. +- So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs. +- */ +- if (io_thd->shutdown && r_count <= 0) { +- break; +- } ++ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ if (r_count <= 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } +- struct epoll_event ev; +- ev.events = EPOLLIN; +- ev.data.fd = ioc->fd_from; +- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); + +- free(buf); ++ if (p->state->terminal) { ++ fd_to = &(p->recv_fd); ++ } else { ++ fd_to = &(p->shim_io->in); ++ } ++ w_count = write_nointr_in_total(*fd_to, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); ++ close(*fd_to); ++ *fd_to = -1; ++ } + +- return NULL; ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + +-static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data) ++static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { +- io_thread_t *thd = (io_thread_t *)data; +- if (thd->ioc == NULL || fd != thd->ioc->fd_from) { +- return; ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ int w_count = 0; ++ ++ if (events & EPOLLHUP) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- if (event & EPOLLIN) { +- (void)sem_post(&thd->sem_thd); +- } else if (event & EPOLLHUP) { +- thd->shutdown = true; +- (void)sem_post(&thd->sem_thd); ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } +-} + +-static int create_io_copy_thread(process_t *p, int std_id) +-{ +- int ret = SHIM_ERR; +- io_thread_t *io_thd = NULL; +- io_copy_t *ioc = NULL; ++ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + +- ioc = (io_copy_t *)calloc(1, sizeof(io_copy_t)); +- if (ioc == NULL) { +- goto failure; ++ if (p->block_read) { ++ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ } else { ++ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); + } +- ioc->id = std_id; +- ioc->fd_from = -1; +- ioc->fd_to = NULL; +- if (pthread_mutex_init(&(ioc->mutex), NULL) != 0) { +- goto failure; ++ if (r_count <= 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- io_thd = (io_thread_t *)calloc(1, sizeof(io_thread_t)); +- if (io_thd == NULL) { +- goto failure; ++ shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); ++ ++ if (p->isulad_io->out == -1) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } +- if (sem_init(&io_thd->sem_thd, 0, 0) != 0) { +- write_message(g_log_fd, ERR_MSG, "sem init failed:%d", SHIM_SYS_ERR(errno)); +- goto failure; ++ ++ w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); ++ close(p->isulad_io->out); ++ p->isulad_io->out = -1; + } +- io_thd->epfd = p->io_loop_fd; +- io_thd->ioc = ioc; +- io_thd->shutdown = false; +- io_thd->is_stdin = std_id == STDID_IN ? true : false; +- io_thd->terminal = std_id != STDID_IN ? p->terminal : NULL; + +- p->io_threads[std_id] = io_thd; ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} + +- ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd); +- if (ret != SHIM_OK) { +- write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno)); +- goto failure; ++static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) ++{ ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ int w_count = 0; ++ ++ if (events & EPOLLHUP) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- ret = SHIM_OK; ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++ } + +- return ret; ++ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + +-failure: +- if (ioc != NULL) { +- pthread_mutex_destroy(&(ioc->mutex)); +- free(ioc); ++ if (p->block_read) { ++ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ } else { ++ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); + } +- if (io_thd != NULL) { +- free(io_thd); ++ if (r_count <= 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- return SHIM_ERR; +-} +- +-static int start_io_copy_threads(process_t *p) +-{ +- int ret = SHIM_ERR; +- int i; +- +- /* 4 threads for stdin, stdout, stderr and exec resize */ +- for (i = 0; i < 4; i++) { +- /* +- * if the terminal is used, we do not need to active the io copy of stderr pipe, +- * for stderr and stdout are mixed together +- */ +- if (i == STDID_ERR && p->state->terminal) { +- continue; +- } ++ shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); + +- ret = create_io_copy_thread(p, i); +- if (ret != SHIM_OK) { +- return SHIM_ERR; +- } ++ if (p->isulad_io->err == -1) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } +- return SHIM_OK; +-} + +-static void destroy_io_thread(process_t *p, int std_id) +-{ +- io_thread_t *io_thd = p->io_threads[std_id]; +- if (io_thd == NULL) { +- return; ++ w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count); ++ if (w_count < 0) { ++ /* When any error occurs, set the write fd -1 */ ++ write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); ++ close(p->isulad_io->err); ++ p->isulad_io->err = -1; + } + +- io_thd->shutdown = true; +- (void)sem_post(&io_thd->sem_thd); +- pthread_join(io_thd->tid, NULL); +- if (io_thd->ioc != NULL) { +- free(io_thd->ioc); +- } +- free(io_thd); +- p->io_threads[std_id] = NULL; ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + +-/* +- std_id: channel type +- isulad_stdio: one side of the isulad fifo file +- fd: one side of the shim io pipe +- --------------------------------------------------------------- +- | CHANNEL | iSulad Fifo Side | Flow Direction | fd | +- --------------------------------------------------------------- +- | STDIN | READ | --> | WRITE | +- --------------------------------------------------------------- +- | STDOUT | WRITE | <-- | READ | +- --------------------------------------------------------------- +- | STDERR | WRITE | <-- | READ | +- --------------------------------------------------------------- +- | RESIZE | READ | --> | WRITE | +- --------------------------------------------------------------- +-*/ +-static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd) ++static int resize_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { +- mode_t mode; +- int fd_isulad = -1; +- int *fd_from = NULL; +- int *fd_to = NULL; ++ process_t *p = (process_t *)cbdata; ++ int r_count = 0; ++ int resize_fd = -1; + +- if (std_id == STDID_IN || std_id == EXEC_RESIZE) { +- mode = O_RDONLY; +- fd_from = &fd_isulad; +- fd_to = &fd; +- } else { +- mode = O_WRONLY; +- fd_from = &fd; +- fd_to = &fd_isulad; ++ if (events & EPOLLHUP) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- if (isulad_stdio != NULL && file_exists(isulad_stdio)) { +- fd_isulad = open_fifo_noblock(isulad_stdio, mode); +- if (fd_isulad < 0) { +- return SHIM_ERR; +- } +- /* open dummy fd to avoid resize epoll hub */ +- if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { +- return SHIM_ERR; +- } ++ if (!(events & EPOLLIN)) { ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + +- if (*fd_from != -1) { +- if (std_id != STDID_IN && std_id != EXEC_RESIZE && p->io_threads[std_id]->terminal != NULL) { +- (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd); +- } +- return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to); ++ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); ++ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); ++ if (r_count <= 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; + } + +- /* if no I/O source is available, the I/O thread nead to be destroyed */ +- destroy_io_thread(p, std_id); ++ resize_fd = p->recv_fd; ++ struct winsize wsize = { 0x00 }; ++ if (get_exec_winsize(p->buf, &wsize) < 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; ++ } ++ if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { ++ return EPOLL_LOOP_HANDLE_CLOSE; ++ } + +- return SHIM_OK; ++ return EPOLL_LOOP_HANDLE_CONTINUE; + } + +-static void *task_console_accept(void *data) ++static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) + { ++ process_t *p = (process_t *)cbdata; + int conn_fd = -1; +- int recv_fd = -1; + int ret = SHIM_ERR; +- console_accept_t *ac = (console_accept_t *)data; + +- conn_fd = accept(ac->listen_fd, NULL, NULL); ++ conn_fd = accept(p->listen_fd, NULL, NULL); + if (conn_fd < 0) { +- write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno)); ++ write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); + goto out; + } + +- recv_fd = receive_fd(conn_fd); +- if (check_fd(recv_fd) != true) { ++ p->recv_fd = receive_fd(conn_fd); ++ if (check_fd(p->recv_fd) != true) { + write_message(g_log_fd, ERR_MSG, "check console fd failed"); + goto out; + } + + /* do console io copy */ + +- /* p.state.stdin---->runtime.console */ +- ret = connect_to_isulad(ac->p, STDID_IN, ac->p->state->isulad_stdin, recv_fd); ++ // p->isulad_io->in ----> p->recv_fd ++ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); + if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); + goto out; + } +- +- /* p.state.stdout<------runtime.console */ +- ret = connect_to_isulad(ac->p, STDID_OUT, ac->p->state->isulad_stdout, recv_fd); ++ // p->recv_fd ----> p->isulad_io->out ++ ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p); + if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); + goto out; + } +- +- /* p.state.resize_fifo------>runtime.console */ +- ret = connect_to_isulad(ac->p, EXEC_RESIZE, ac->p->state->resize_fifo, recv_fd); ++ // p->isulad_io->resize ----> p->recv_fd ++ ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p); + if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); + goto out; + } + + out: + /* release listen socket at the first time */ +- close_fd(&ac->listen_fd); +- if (ac->p->console_sock_path != NULL) { +- (void)unlink(ac->p->console_sock_path); +- free(ac->p->console_sock_path); +- ac->p->console_sock_path = NULL; +- } +- free(ac); +- if (ret != SHIM_OK) { +- /* +- * When an error occurs during the receiving of the fd , the process +- * exits directly. The files created in the working directory will be +- * deleted by its parent process isulad +- */ +- exit(EXIT_FAILURE); ++ close_fd(&p->listen_fd); ++ if (p->console_sock_path != NULL) { ++ (void)unlink(p->console_sock_path); ++ free(p->console_sock_path); ++ p->console_sock_path = NULL; + } +- return NULL; ++ return ret; + } + +-static void *io_epoll_loop(void *data) ++static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) + { +- process_t *p = (process_t *)data; +- int wait_fds = 0; +- struct epoll_event evs[MAX_EVENTS]; +- int i; ++ int i, j; + +- if ((pthread_detach(pthread_self())) != 0) { +- write_message(g_log_fd, ERR_MSG, "detach thread failed"); +- return NULL; ++ for (i = 0; i < 3; i++) { ++ for (j = 0; j < 2; j++) { ++ int ret = fchown(stdio_fd[i][j], uid, gid); ++ if (ret != SHIM_OK) { ++ return SHIM_ERR; ++ } ++ } + } ++ return SHIM_OK; ++} + +- p->io_loop_fd = epoll_create1(EPOLL_CLOEXEC); +- if (p->io_loop_fd < 0) { +- write_message(g_log_fd, ERR_MSG, "epoll create failed:%d", SHIM_SYS_ERR(errno)); +- exit(EXIT_FAILURE); +- } +- (void)sem_post(&p->sem_mainloop); ++static void stdio_release(int (*stdio_fd)[2]) ++{ ++ int i, j; + +- for (;;) { +- wait_fds = epoll_wait(p->io_loop_fd, evs, MAX_EVENTS, -1); +- if (wait_fds < 0) { +- if (errno == EINTR) { +- continue; ++ for (i = 0; i < 3; i++) { ++ for (j = 0; j < 2; j++) { ++ if (stdio_fd[i][j] > 0) { ++ close(stdio_fd[i][j]); + } +- _exit(EXIT_FAILURE); + } ++ } ++} + +- for (i = 0; i < wait_fds; i++) { +- io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr; +- sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io); +- } ++static stdio_t *initialize_io(process_t *p) ++{ ++ int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; ++ ++ stdio_t *stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); ++ p->stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); ++ if (p->stdio == NULL || stdio == NULL) { ++ goto failure; ++ } ++ ++ /* don't open resize pipe */ ++ if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || ++ (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { ++ write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); ++ goto failure; + } ++ ++ p->stdio->in = stdio_fd[0][0]; // r ++ stdio->in = stdio_fd[0][1]; // w ++ p->stdio->out = stdio_fd[1][1]; // w ++ stdio->out = stdio_fd[1][0]; // r ++ p->stdio->err = stdio_fd[2][1]; // w ++ stdio->err = stdio_fd[2][0]; // r ++ p->stdio->resize = stdio_fd[3][0]; // r ++ stdio->resize = stdio_fd[3][1]; // w ++ ++ if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { ++ goto failure; ++ } ++ ++ return stdio; ++ ++failure: ++ if (stdio != NULL) { ++ free(stdio); ++ stdio = NULL; ++ } ++ if (p->stdio != NULL) { ++ free(p->stdio); ++ p->stdio = NULL; ++ } ++ stdio_release(stdio_fd); ++ ++ return NULL; + } + + static int new_temp_console_path(process_t *p) +@@ -614,7 +455,7 @@ static int new_temp_console_path(process_t *p) + if (ret != SHIM_OK) { + return SHIM_ERR; + } +- p->console_sock_path = (char *)calloc(1, MAX_CONSOLE_SOCK_LEN + 1); ++ p->console_sock_path = (char *)util_common_calloc_s(MAX_CONSOLE_SOCK_LEN + 1); + if (p->console_sock_path == NULL) { + return SHIM_ERR; + } +@@ -628,12 +469,11 @@ static int new_temp_console_path(process_t *p) + return SHIM_OK; + } + +-static int console_init(process_t *p, pthread_t *tid_accept) ++static int console_init(process_t *p, struct epoll_descr *descr) + { + int ret = SHIM_ERR; + int fd = -1; + struct sockaddr_un addr; +- console_accept_t *ac = NULL; + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { +@@ -657,145 +497,182 @@ static int console_init(process_t *p, pthread_t *tid_accept) + goto failure; + } + +- ac = (console_accept_t *)calloc(1, sizeof(console_accept_t)); +- if (ac == NULL) { +- goto failure; +- } +- ac->p = p; +- ac->listen_fd = fd; ++ p->listen_fd = fd; + +- ret = pthread_create(tid_accept, NULL, task_console_accept, ac); ++ ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p); + if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); + goto failure; + } + + return SHIM_OK; + failure: + close_fd(&fd); +- if (ac != NULL) { +- free(ac); +- ac = NULL; +- } + (void)unlink(p->console_sock_path); + + return SHIM_ERR; + } + +-static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) ++static int open_terminal_io(process_t *p, struct epoll_descr *descr) + { +- int i, j; ++ int ret = SHIM_ERR; + +- for (i = 0; i < 3; i++) { +- for (j = 0; j < 2; j++) { +- int ret = fchown(stdio_fd[i][j], uid, gid); +- if (ret != SHIM_OK) { +- return SHIM_ERR; +- } +- } ++ ret = new_temp_console_path(p); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); ++ return SHIM_ERR; + } +- return SHIM_OK; +-} +- +-static void stdio_release(int (*stdio_fd)[2]) +-{ +- int i, j; + +- for (i = 0; i < 3; i++) { +- for (j = 0; j < 2; j++) { +- if (stdio_fd[i][j] > 0) { +- close(stdio_fd[i][j]); +- } +- } +- } ++ /* begin listen from p->console_sock_path */ ++ return console_init(p, descr); + } + +-static stdio_t *initialize_io(process_t *p) ++static int open_generic_io(process_t *p, struct epoll_descr *descr) + { +- int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; +- +- stdio_t *stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); +- p->stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); +- if (p->stdio == NULL || stdio == NULL) { +- goto failure; +- } ++ int ret = SHIM_ERR; + +- /* don't open resize pipe */ +- if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || +- (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { +- write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); +- goto failure; ++ // io: in: w out/err: r ++ stdio_t *io = initialize_io(p); ++ if (io == NULL) { ++ return SHIM_ERR; + } ++ p->shim_io = io; + +- p->stdio->in = stdio_fd[0][0]; // r +- stdio->in = stdio_fd[0][1]; // w +- p->stdio->out = stdio_fd[1][1]; // w +- stdio->out = stdio_fd[1][0]; // r +- p->stdio->err = stdio_fd[2][1]; // w +- stdio->err = stdio_fd[2][0]; // r +- p->stdio->resize = stdio_fd[3][0]; // r +- stdio->resize = stdio_fd[3][1]; // w +- +- if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { +- goto failure; ++ // p->isulad_io->in ----> p->shim_io->in ++ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; + } +- +- return stdio; +- +-failure: +- if (stdio != NULL) { +- free(stdio); +- stdio = NULL; ++ // p->shim_io->out ----> p->isulad_io->out ++ ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; + } +- if (p->stdio != NULL) { +- free(p->stdio); +- p->stdio = NULL; ++ // p->shim_io->err ----> p->isulad_io->err ++ ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); ++ return SHIM_ERR; + } +- stdio_release(stdio_fd); + +- return NULL; ++ return SHIM_OK; + } + +-static int open_terminal_io(process_t *p, pthread_t *tid_accept) ++static int set_non_block(int fd) + { ++ int flag = -1; + int ret = SHIM_ERR; + +- ret = new_temp_console_path(p); +- if (ret != SHIM_OK) { +- write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); ++ flag = fcntl(fd, F_GETFL, 0); ++ if (flag < 0) { ++ return SHIM_ERR; ++ } ++ ++ ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK); ++ if (ret != 0) { + return SHIM_ERR; + } + +- /* begin listen and accept fd from p->console_sock_path */ +- return console_init(p, tid_accept); ++ return SHIM_OK; + } + +-static int open_generic_io(process_t *p) ++/* ++ std_id: channel type ++ isulad_stdio: one side of the isulad fifo file ++ fd: one side of the shim io pipe ++ --------------------------------------------------------------- ++ | CHANNEL | iSulad Fifo Side | Flow Direction | fd | ++ --------------------------------------------------------------- ++ | STDIN | READ | --> | WRITE | ++ --------------------------------------------------------------- ++ | STDOUT | WRITE | <-- | READ | ++ --------------------------------------------------------------- ++ | STDERR | WRITE | <-- | READ | ++ --------------------------------------------------------------- ++ | RESIZE | READ | --> | WRITE | ++ --------------------------------------------------------------- ++*/ ++static void *io_epoll_loop(void *data) + { +- int ret = SHIM_ERR; ++ int ret = 0; ++ int fd_out = -1; ++ int fd_err = -1; ++ process_t *p = (process_t *)data; ++ struct epoll_descr descr; + +- // io: in: w out/err: r +- stdio_t *io = initialize_io(p); +- if (io == NULL) { +- return SHIM_ERR; ++ ret = epoll_loop_open(&descr); ++ if (ret != 0) { ++ write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); + } +- p->shim_io = io; +- /* stdin */ +- ret = connect_to_isulad(p, STDID_IN, p->state->isulad_stdin, io->in); +- if (ret != SHIM_OK) { +- return SHIM_ERR; ++ ++ // sync fd: epoll loop will exit when recive sync fd event. ++ ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p); ++ if (ret != 0) { ++ write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); + } +- /* stdout */ +- ret = connect_to_isulad(p, STDID_OUT, p->state->isulad_stdout, io->out); +- if (ret != SHIM_OK) { +- return SHIM_ERR; ++ ++ if (p->state->terminal) { ++ ret = open_terminal_io(p, &descr); ++ } else { ++ ret = open_generic_io(p, &descr); + } +- /* stderr */ +- ret = connect_to_isulad(p, STDID_ERR, p->state->isulad_stderr, io->err); + if (ret != SHIM_OK) { +- return SHIM_ERR; ++ write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); + } + +- return SHIM_OK; ++ (void)sem_post(&p->sem_mainloop); ++ ++ ret = epoll_loop(&descr, -1); ++ if (ret != 0) { ++ write_message(g_log_fd, ERR_MSG, "epoll loop failed"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // in order to avoid data loss, set fd non-block and read it ++ p->block_read = false; ++ if (p->state->terminal) { ++ fd_out = p->recv_fd; ++ } else { ++ fd_out = p->shim_io->out; ++ fd_err = p->shim_io->err; ++ } ++ ++ if (fd_out > 0) { ++ ret = set_non_block(fd_out); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); ++ } ++ ++ for (;;) { ++ ret = stdout_cb(fd_out, EPOLLIN, p, &descr); ++ if (ret == EPOLL_LOOP_HANDLE_CLOSE) { ++ break; ++ } ++ } ++ } ++ ++ if (fd_err > 0) { ++ ret = set_non_block(fd_err); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); ++ exit(EXIT_FAILURE); ++ } ++ ++ for (;;) { ++ ret = stderr_cb(fd_err, EPOLLIN, p, &descr); ++ if (ret == EPOLL_LOOP_HANDLE_CLOSE) { ++ break; ++ } ++ } ++ } ++ ++ return NULL; + } + + static void adapt_for_isulad_stdin(process_t *p) +@@ -813,7 +690,7 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s + { + log_terminal *log_term = NULL; + +- log_term = calloc(1, sizeof(log_terminal)); ++ log_term = util_common_calloc_s(sizeof(log_terminal)); + if (log_term == NULL) { + write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal"); + goto clean_out; +@@ -858,11 +735,90 @@ clean_out: + return SHIM_ERR; + } + ++static int open_isulad_fd(int std_id, const char *isulad_stdio, int *fd) ++{ ++ mode_t mode = O_WRONLY; ++ ++ if (std_id == STDID_IN || std_id == EXEC_RESIZE) { ++ mode = O_RDONLY; ++ } ++ ++ if (isulad_stdio != NULL && file_exists(isulad_stdio)) { ++ *(fd) = open_fifo_noblock(isulad_stdio, mode); ++ if (*(fd) < 0) { ++ return -1; ++ } ++ /* open dummy fd to avoid resize epoll hub */ ++ if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ ++ ++static int init_isulad_stdio(process_t *p) ++{ ++ int ret = SHIM_OK; ++ p->isulad_io = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); ++ if (p->isulad_io == NULL) { ++ return SHIM_ERR; ++ } ++ ++ p->isulad_io->in = -1; ++ p->isulad_io->out = -1; ++ p->isulad_io->err = -1; ++ p->isulad_io->resize = -1; ++ ++ ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin); ++ goto failure; ++ } ++ ++ ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout); ++ goto failure; ++ } ++ ++ ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr); ++ goto failure; ++ } ++ ++ ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize); ++ if (ret != SHIM_OK) { ++ write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo); ++ goto failure; ++ } ++ return SHIM_OK; ++failure: ++ if (p->isulad_io != NULL) { ++ free(p->isulad_io); ++ p->isulad_io = NULL; ++ } ++ if (p->isulad_io->in > 0) { ++ close(p->isulad_io->in); ++ } ++ if (p->isulad_io->out > 0) { ++ close(p->isulad_io->out); ++ } ++ if (p->isulad_io->err > 0) { ++ close(p->isulad_io->err); ++ } ++ if (p->isulad_io->resize > 0) { ++ close(p->isulad_io->resize); ++ } ++ return SHIM_ERR; ++} ++ + process_t *new_process(char *id, char *bundle, char *runtime) + { + shim_client_process_state *p_state; + process_t *p = NULL; +- int i; + int ret; + + p_state = load_process(); +@@ -870,7 +826,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) + return NULL; + } + +- p = (process_t *)calloc(1, sizeof(process_t)); ++ p = (process_t *)util_common_calloc_s(sizeof(process_t)); + if (p == NULL) { + return NULL; + } +@@ -889,16 +845,31 @@ process_t *new_process(char *id, char *bundle, char *runtime) + p->bundle = bundle; + p->runtime = runtime; + p->state = p_state; +- ++ p->block_read = true; + p->console_sock_path = NULL; + p->exit_fd = -1; + p->io_loop_fd = -1; + p->ctr_pid = -1; ++ p->listen_fd = -1; ++ p->recv_fd = -1; + p->stdio = NULL; + p->shim_io = NULL; ++ p->isulad_io = NULL; + +- for (i = 0; i < 3; i++) { +- p->io_threads[i] = NULL; ++ p->sync_fd = eventfd(0, EFD_CLOEXEC); ++ if (p->sync_fd < 0) { ++ write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno)); ++ goto failure; ++ } ++ ++ ret = init_isulad_stdio(p); ++ if (ret != SHIM_OK) { ++ goto failure; ++ } ++ ++ p->buf = util_common_calloc_s(DEFAULT_IO_COPY_BUF + 1); ++ if (p->buf == NULL) { ++ goto failure; + } + + return p; +@@ -909,28 +880,11 @@ failure: + return NULL; + } + +-int open_io(process_t *p, pthread_t *tid_accept) ++int process_io_start(process_t *p, pthread_t *tid_epoll) + { + int ret = SHIM_ERR; + +- ret = start_io_copy_threads(p); +- if (ret != SHIM_OK) { +- return SHIM_ERR; +- } +- +- if (p->state->terminal) { +- return open_terminal_io(p, tid_accept); +- } +- +- return open_generic_io(p); +-} +- +-int process_io_init(process_t *p) +-{ +- int ret = SHIM_ERR; +- +- pthread_t tid_loop; +- ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p); ++ ret = pthread_create(tid_epoll, NULL, io_epoll_loop, p); + if (ret != SHIM_OK) { + return SHIM_SYS_ERR(errno); + } +@@ -1091,8 +1045,8 @@ static void exec_runtime_process(process_t *p, int exec_fd) + } + + char *cwd = getcwd(NULL, 0); +- char *log_path = (char *)calloc(1, PATH_MAX); +- char *pid_path = (char *)calloc(1, PATH_MAX); ++ char *log_path = (char *)util_common_calloc_s(PATH_MAX); ++ char *pid_path = (char *)util_common_calloc_s(PATH_MAX); + if (cwd == NULL || log_path == NULL || pid_path == NULL) { + (void)dprintf(exec_fd, "memory error: %s", strerror(errno)); + _exit(EXIT_FAILURE); +@@ -1294,13 +1248,11 @@ static int wait_container_process_with_timeout(process_t *p, const unsigned int + + } + +-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout) ++int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout) + { +- int i; + int nret = 0; + int ret = 0; + int status = 0; +- struct timespec ts; + + ret = wait_container_process_with_timeout(p, timeout, &status); + if (ret == SHIM_ERR_TIMEOUT) { +@@ -1324,27 +1276,20 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons + if (p->exit_fd > 0) { + (void)write_nointr(p->exit_fd, &status, sizeof(int)); + } +- // wait for task_console_accept thread termination. In order to make sure that +- // the io_copy connection is established and io_thread is not used by multiple threads. +- if (p->state->terminal) { +- if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { +- write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); +- nret = pthread_join(tid_accept, NULL); +- } else { +- // Set the maximum waiting time to 60s to prevent stuck. +- ts.tv_sec += 60; +- nret = pthread_timedjoin_np(tid_accept, NULL, &ts); +- } + +- if (nret != 0) { +- write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); ++ if (p->sync_fd > 0) { ++ if (eventfd_write(p->sync_fd, 1)) { ++ write_message(g_log_fd, ERR_MSG, "Failed to write sync fd"); + } + } + +- for (i = 0; i < 3; i++) { +- destroy_io_thread(p, i); ++ nret = pthread_join(tid_epoll, NULL); ++ if (nret != 0) { ++ write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread"); + } + ++ close(p->sync_fd); ++ + if (!p->state->exec) { + // if log did not contain "/n", print remaind container log when exit isulad-shim + shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0); +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 7e3259e8..4cb6462c 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -40,64 +40,37 @@ typedef struct { + int resize; + } stdio_t; + +-typedef struct fd_node { +- int fd; +- bool is_log; +- struct fd_node *next; +-} fd_node_t; +- +-typedef struct { +- int fd_from; +- fd_node_t *fd_to; +- int id;// 0,1,2,3 +- pthread_mutex_t mutex; +-} io_copy_t; +- +-typedef struct { +- int epfd; +- pthread_t tid; +- pthread_attr_t attr; +- sem_t sem_thd; +- io_copy_t *ioc; +- bool shutdown; +- bool is_stdin; +- log_terminal *terminal;// just used by stdout and stderr +-} io_thread_t; +- + typedef struct process { + char *id; + char *bundle; + char *runtime; +- char *console_sock_path;// pty socket path ++ char *console_sock_path; // pty socket path + int io_loop_fd; + int exit_fd; + int ctr_pid; ++ int sync_fd; ++ int listen_fd; ++ int recv_fd; ++ bool block_read; + log_terminal *terminal; +- stdio_t *stdio;// shim to on runtime side, in:r out/err: w ++ stdio_t *stdio; // shim to on runtime side, in:r out/err: w + stdio_t *shim_io; // shim io on isulad side, in: w out/err: r +- io_thread_t *io_threads[4];// stdin,stdout,stderr,exec_resize ++ stdio_t *isulad_io; // isulad io, in:r out/err: w + shim_client_process_state *state; + sem_t sem_mainloop; ++ char *buf; + } process_t; + +-typedef struct { +- int listen_fd; +- process_t *p; +-} console_accept_t; +- + typedef struct { + int pid; + int status; + } process_exit_t; + +- +- + process_t* new_process(char *id, char *bundle, char *runtime); + +-int open_io(process_t *p, pthread_t *tid_accept); +-int process_io_init(process_t *p); ++int process_io_start(process_t *p, pthread_t *tid_epoll); + int create_process(process_t *p); +-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout); ++int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout); + + #ifdef __cplusplus + } +diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt +index e5c1cd6e..122538ff 100644 +--- a/test/cmd/isulad-shim/CMakeLists.txt ++++ b/test/cmd/isulad-shim/CMakeLists.txt +@@ -6,6 +6,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/mainloop.c + isulad-shim_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc +index 34ecd452..8c116ac8 100644 +--- a/test/cmd/isulad-shim/isulad-shim_ut.cc ++++ b/test/cmd/isulad-shim/isulad-shim_ut.cc +@@ -11,6 +11,7 @@ + #include + #include + ++#include "mainloop.h" + #include "process.h" + #include "common.h" + +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec index 436fa22..bfa5fe2 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.0.18 -%global _release 5 +%global _release 6 %global is_systemd 1 %global enable_shimv2 1 %global is_embedded 1 @@ -59,6 +59,17 @@ Patch0043: 0043-isulad-shim-fix-log-loss-bug.patch Patch0044: 0044-remove-unused-func.patch Patch0045: 0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch Patch0046: 0046-free-timeout-when-shim_create-finished.patch +Patch0047: 0047-clean-isulad-shim-compile-relies.patch +Patch0048: 0048-remote-layer-store-demo.patch +Patch0049: 0049-add-ci-for-remote-ro.patch +Patch0050: 0050-fix-compile-error-when-not-enable-remote-ro.patch +Patch0051: 0051-CI-not-enable-remote-ro-for-ut.patch +Patch0052: 0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch +Patch0053: 0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch +Patch0054: 0054-refactor-remote-ro-code.patch +Patch0055: 0055-bugfix-when-refresh-can-t-load-or-pull-images.patch +Patch0056: 0056-remove-unused-headers.patch +Patch0057: 0057-change-isulad-shim-epoll-struct.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -303,6 +314,12 @@ fi %endif %changelog +* Thu May 04 2023 zhangxiaoyu - 2.0.18-6 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + * Mon Apr 24 2023 zhangxiaoyu - 2.0.18-5 - Type: bugfix - ID: NA