!563 [sync] PR-561: upgrade from upstream

Merge pull request !563 from openeuler-sync-bot/sync-pr561-openEuler-22.03-LTS-to-openEuler-22.03-LTS-Next

0047-clean-isulad-shim-compile-relies.patch

@@ -0,0 +1,347 @@
+From fde1c406a837b849f1182d8943f1f942088b608d Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Sun, 23 Apr 2023 15:01:58 +0800
+Subject: [PATCH 47/56] clean isulad shim compile relies
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/CMakeLists.txt                  |  12 +-
+ src/cmd/isulad-shim/common.c        | 179 ++++++++++++++++++++++++++++
+ src/cmd/isulad-shim/common.h        |  49 ++++++++
+ src/cmd/isulad-shim/process.c       |   3 -
+ test/cmd/isulad-shim/CMakeLists.txt |  13 --
+ 5 files changed, 238 insertions(+), 18 deletions(-)
+
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index f3dd3c19..02d7b13f 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -113,8 +113,16 @@ endif()
+ add_executable(isulad-shim
+     ${ISULAD_SHIM_SRCS}
+     )
+-target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS})
+-target_link_libraries(isulad-shim libisulad_tools)
++target_include_directories(isulad-shim PUBLIC 
++    ${ISULAD_SHIM_INCS} 
++    ${CMAKE_CURRENT_SOURCE_DIR}
++    ${COMMON_INCS}
++    ${CMAKE_BINARY_DIR}/conf
++    ${CHECKED_INCLUDE_DIRS}
++    ${SHARED_INCS}
++    ${ISULA_LIBUTILS_INCLUDE_DIR}
++    )
++target_link_libraries(isulad-shim ${ISULA_LIBUTILS_LIBRARY})
+ if (ANDROID OR MUSL)
+     target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY})
+ else()
+diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
+index f188da1e..e1ca96e1 100644
+--- a/src/cmd/isulad-shim/common.c
++++ b/src/cmd/isulad-shim/common.c
+@@ -27,6 +27,7 @@
+ #include <stdbool.h>
+ #include <stdarg.h>
+ #include <limits.h>
++#include <time.h>
+ 
+ int set_fd_no_inherited(int fd)
+ {
+@@ -346,3 +347,181 @@ int shim_util_safe_uint64(const char *numstr, uint64_t *converted)
+     return 0;
+ }
+ 
++void util_usleep_nointerupt(unsigned long usec)
++{
++#define SECOND_TO_USECOND_MUTIPLE 1000000
++    int ret = 0;
++    struct timespec request = { 0 };
++    struct timespec remain = { 0 };
++    if (usec == 0) {
++        return;
++    }
++
++    request.tv_sec = (time_t)(usec / SECOND_TO_USECOND_MUTIPLE);
++    request.tv_nsec = (long)((usec % SECOND_TO_USECOND_MUTIPLE) * 1000);
++
++    do {
++        ret = nanosleep(&request, &remain);
++        request = remain;
++    } while (ret == -1 && errno == EINTR);
++}
++
++void *util_smart_calloc_s(size_t unit_size, size_t count)
++{
++    if (unit_size == 0) {
++        return NULL;
++    }
++
++    if (count > (MAX_MEMORY_SIZE / unit_size)) {
++        return NULL;
++    }
++
++    return calloc(count, unit_size);
++}
++
++size_t util_array_len(const char **array)
++{
++    const char **pos;
++    size_t len = 0;
++
++    for (pos = array; pos != NULL && *pos != NULL; pos++) {
++        len++;
++    }
++
++    return len;
++}
++
++void util_free_array(char **array)
++{
++    char **p;
++
++    for (p = array; p != NULL && *p != NULL; p++) {
++        UTIL_FREE_AND_SET_NULL(*p);
++    }
++    free(array);
++}
++
++int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment)
++{
++    size_t add_capacity;
++    char **add_array = NULL;
++
++    if (orig_array == NULL || orig_capacity == NULL || increment == 0) {
++        return -1;
++    }
++
++    if (((*orig_array) == NULL) || ((*orig_capacity) == 0)) {
++        UTIL_FREE_AND_SET_NULL(*orig_array);
++        *orig_capacity = 0;
++    }
++
++    add_capacity = *orig_capacity;
++    while (size + 1 > add_capacity) {
++        add_capacity += increment;
++    }
++    if (add_capacity != *orig_capacity) {
++        add_array = util_smart_calloc_s(sizeof(void *), add_capacity);
++        if (add_array == NULL) {
++            return -1;
++        }
++        if (*orig_array != NULL) {
++            (void)memcpy(add_array, *orig_array, *orig_capacity * sizeof(void *));
++            UTIL_FREE_AND_SET_NULL(*orig_array);
++        }
++
++        *orig_array = add_array;
++        *orig_capacity = add_capacity;
++    }
++
++    return 0;
++}
++
++char *util_strdup_s(const char *src)
++{
++    char *dst = NULL;
++
++    if (src == NULL) {
++        return NULL;
++    }
++
++    dst = strdup(src);
++    if (dst == NULL) {
++        abort();
++    }
++
++    return dst;
++}
++
++static char **make_empty_array()
++{
++    char **res_array = NULL;
++
++    res_array = calloc(2, sizeof(char *));
++    if (res_array == NULL) {
++        return NULL;
++    }
++    res_array[0] = util_strdup_s("");
++    return res_array;
++}
++
++static char **util_shrink_array(char **orig_array, size_t new_size)
++{
++    char **new_array = NULL;
++    size_t i = 0;
++
++    if (new_size == 0) {
++        return orig_array;
++    }
++    new_array = util_smart_calloc_s(sizeof(char *), new_size);
++    if (new_array == NULL) {
++        return orig_array;
++    }
++
++    for (i = 0; i < new_size; i++) {
++        new_array[i] = orig_array[i];
++    }
++    free(orig_array);
++    return new_array;
++}
++
++char **util_string_split_multi(const char *src_str, char delim)
++{
++    int ret, tmp_errno;
++    char *token = NULL;
++    char *cur = NULL;
++    char **res_array = NULL;
++    char deli[2] = { delim, '\0' };
++    size_t count = 0;
++    size_t capacity = 0;
++    char *tmpstr = NULL;
++
++    if (src_str == NULL) {
++        return NULL;
++    }
++
++    if (src_str[0] == '\0') {
++        return make_empty_array();
++    }
++
++    tmpstr = util_strdup_s(src_str);
++    cur = tmpstr;
++    token = strsep(&cur, deli);
++    while (token != NULL) {
++        ret = util_grow_array(&res_array, &capacity, count + 1, 16);
++        if (ret < 0) {
++            goto err_out;
++        }
++        res_array[count] = util_strdup_s(token);
++        count++;
++        token = strsep(&cur, deli);
++    }
++    free(tmpstr);
++    return util_shrink_array(res_array, count + 1);
++
++err_out:
++    tmp_errno = errno;
++    free(tmpstr);
++    util_free_array(res_array);
++    errno = tmp_errno;
++    return NULL;
++}
+\ No newline at end of file
+diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
+index 91808295..3de16ace 100644
+--- a/src/cmd/isulad-shim/common.h
++++ b/src/cmd/isulad-shim/common.h
+@@ -58,6 +58,43 @@ extern "C" {
+ #define CONTAINER_ACTION_REBOOT 129
+ #define CONTAINER_ACTION_SHUTDOWN 130
+ 
++
++void util_usleep_nointerupt(unsigned long usec);
++/**
++ * retry_cnt: max count of call cb;
++ * interval_us: how many us to sleep, after call cb;
++ * cb: retry call function;
++ * return:
++ *  0 is cb successful at least once;
++ *  1 is all cb are failure;
++*/
++#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do {    \
++        size_t i = 0;                                               \
++        for(; i < retry_cnt; i++) {                                 \
++            ret = cb(##__VA_ARGS__);                                  \
++            if (ret == 0) {                                         \
++                break;                                              \
++            }                                                       \
++            util_usleep_nointerupt(interval_us);                    \
++        }                                                           \
++    } while(0)
++
++#define UTIL_FREE_AND_SET_NULL(p) \
++    do {                          \
++        if ((p) != NULL) {        \
++            free((void *)(p));    \
++            (p) = NULL;           \
++        }                         \
++    } while (0)
++
++#if __WORDSIZE == 64
++// current max user memory for 64-machine is 2^47 B
++#define MAX_MEMORY_SIZE ((size_t)1 << 47)
++#else
++// current max user memory for 32-machine is 2^31 B
++#define MAX_MEMORY_SIZE ((size_t)1 << 31)
++#endif
++
+ ssize_t read_nointr(int fd, void *buf, size_t count);
+ ssize_t write_nointr(int fd, const void *buf, size_t count);
+ 
+@@ -79,6 +116,18 @@ int open_no_inherit(const char *path, int flag, mode_t mode);
+ 
+ int shim_util_safe_uint64(const char *numstr, uint64_t *converted);
+ 
++void *util_smart_calloc_s(size_t unit_size, size_t count);
++
++size_t util_array_len(const char **array);
++
++void util_free_array(char **array);
++
++int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment);
++
++char *util_strdup_s(const char *src);
++
++char **util_string_split_multi(const char *src_str, char delim);
++
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index a676e7ce..a5e0bd39 100644
+--- a/src/cmd/isulad-shim/process.c
++++ b/src/cmd/isulad-shim/process.c
+@@ -37,9 +37,6 @@
+ 
+ #include "common.h"
+ #include "terminal.h"
+-#include "utils_array.h"
+-#include "utils_string.h"
+-#include "utils.h"
+ 
+ #define MAX_EVENTS 100
+ #define DEFAULT_IO_COPY_BUF (16 * 1024)
+diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt
+index dc293f6d..e5c1cd6e 100644
+--- a/test/cmd/isulad-shim/CMakeLists.txt
++++ b/test/cmd/isulad-shim/CMakeLists.txt
+@@ -6,26 +6,13 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c
+     isulad-shim_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+     ${GTEST_INCLUDE_DIR}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils
+     ${CMAKE_BINARY_DIR}/conf
+     )
+ target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
+-- 
+2.25.1
+

0049-add-ci-for-remote-ro.patch

@@ -0,0 +1,119 @@
+From 1b1900c183886e13edd86c2e2cb6e5c42cfebd3d Mon Sep 17 00:00:00 2001
+From: Neil <wrz750726@gmail.com>
+Date: Sun, 5 Mar 2023 12:23:29 +0000
+Subject: [PATCH 49/56] add ci for remote ro
+
+Signed-off-by: Neil <wangrunze13@huawei.com>
+---
+ CI/make-and-install.sh                        |  4 +-
+ .../container_cases/test_data/daemon.json     |  1 +
+ CI/test_cases/image_cases/ro_separate.sh      | 69 +++++++++++++++++++
+ 3 files changed, 72 insertions(+), 2 deletions(-)
+ create mode 100644 CI/test_cases/image_cases/ro_separate.sh
+
+diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
+index 81022d75..e714d206 100755
+--- a/CI/make-and-install.sh
++++ b/CI/make-and-install.sh
+@@ -106,9 +106,9 @@ rm -rf build
+ mkdir build
+ cd build
+ if [[ ${enable_gcov} -ne 0 ]]; then
+-  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON ..
++  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
+ else
+-  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON ..
++  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
+ fi
+ make -j $(nproc)
+ make install
+diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
+index aa88c9da..2664c6b2 100644
+--- a/CI/test_cases/container_cases/test_data/daemon.json
++++ b/CI/test_cases/container_cases/test_data/daemon.json
+@@ -19,6 +19,7 @@
+     "hook-spec": "/etc/default/isulad/hooks/default.json",
+     "start-timeout": "2m",
+     "storage-driver": "overlay2",
++    "storage-enable-remote-layer": false,
+     "storage-opts": [
+         "overlay2.override_kernel_check=true"
+     ],
+diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
+new file mode 100644
+index 00000000..47e04abb
+--- /dev/null
++++ b/CI/test_cases/image_cases/ro_separate.sh
+@@ -0,0 +1,69 @@
++#!/bin/bash
++#
++# attributes: isulad basic image
++# concurrent: NA
++# spend time: 22
++
++#######################################################################
++##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
++# - iSulad licensed under the Mulan PSL v2.
++# - You can use this software according to the terms and conditions of the Mulan PSL v2.
++# - You may obtain a copy of Mulan PSL v2 at:
++# -     http://license.coscl.org.cn/MulanPSL2
++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++# - PURPOSE.
++# - See the Mulan PSL v2 for more details.
++##- @Description:CI
++##- @Author: wangrunze
++##- @Create: 2023-03-03
++#######################################################################
++
++declare -r curr_path=$(dirname $(readlink -f "$0"))
++source ../helpers.sh
++single_image="${curr_path}/busybox.tar"
++
++function test_separate_ro()
++{
++  local ret=0
++  local test="isula separate ro test => (${FUNCNAME[@]})"
++
++  msg_info "${test} starting..."
++
++  sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json
++  start_isulad_with_valgrind
++  wait_isulad_running
++
++  isula rmi busybox
++
++  isula pull busybox
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++))
++
++  isula run -tid --name test_separate busybox /bin/sh
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++))
++
++  isula stop test_separate
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
++
++  isula rmi busybox
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
++
++  isula load -i $single_image
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++))
++
++  check_valgrind_log
++  [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++))
++
++  sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json
++  start_isulad_with_valgrind
++  wait_isulad_running
++
++  msg_info "${test} finished with return ${ret}..."
++  return ${ret}
++}
++
++declare -i ans=0
++
++test_separate_ro || ((ans++))
++
++show_result ${ans} "${curr_path}/${0}"
+-- 
+2.25.1
+

0050-fix-compile-error-when-not-enable-remote-ro.patch

@@ -0,0 +1,62 @@
+From e4f309f61b169529c263b7a83a0eda16ebe132f5 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Sun, 5 Mar 2023 18:55:40 -0800
+Subject: [PATCH 50/56] fix compile error when not enable remote ro
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ .../modules/image/oci/storage/image_store/CMakeLists.txt      | 3 +++
+ .../modules/image/oci/storage/layer_store/CMakeLists.txt      | 4 ++++
+ .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt   | 3 +++
+ 3 files changed, 10 insertions(+)
+
+diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
+index ecf21caa..7d4fb77c 100644
+--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
++++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
+@@ -1,5 +1,8 @@
+ # get current directory sources files
+ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs)
++IF (NOT ENABLE_REMOTE_LAYER_STORE)
++list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c")
++ENDIF()
+ 
+ set(IMAGE_STORE_SRCS
+     ${local_image_store_srcs}
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
+index f964f709..e04b4ad7 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
++++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
+@@ -1,5 +1,8 @@
+ # get current directory sources files
+ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs)
++IF (NOT ENABLE_REMOTE_LAYER_STORE)
++list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c")
++ENDIF()
+ add_subdirectory(graphdriver)
+ 
+ set(LAYER_STORE_SRCS
+@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS
+     ${GRAPHDRIVER_SRCS}
+     PARENT_SCOPE
+     )
++
+ set(LAYER_STORE_INCS
+     ${CMAKE_CURRENT_SOURCE_DIR}
+     ${GRAPHDRIVER_INCS}
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
+index ceed16b7..dd4e82aa 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
+@@ -1,5 +1,8 @@
+ # get current directory sources files
+ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs)
++IF (NOT ENABLE_REMOTE_LAYER_STORE)
++list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c")
++ENDIF()
+ 
+ set(OVERLAY2_SRCS
+     ${local_overlay2_srcs}
+-- 
+2.25.1
+

0051-CI-not-enable-remote-ro-for-ut.patch

@@ -0,0 +1,41 @@
+From 1d5aa99d97df3f4ddec4ae436cb0ccbbba3e863a Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Mon, 6 Mar 2023 18:59:43 -0800
+Subject: [PATCH 51/56] CI not enable remote ro for ut
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ CI/make-and-install.sh                   | 2 +-
+ CI/test_cases/image_cases/ro_separate.sh | 3 +++
+ 2 files changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
+index e714d206..7c194826 100755
+--- a/CI/make-and-install.sh
++++ b/CI/make-and-install.sh
+@@ -106,7 +106,7 @@ rm -rf build
+ mkdir build
+ cd build
+ if [[ ${enable_gcov} -ne 0 ]]; then
+-  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
++  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON ..
+ else
+   cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
+ fi
+diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
+index 47e04abb..df45e120 100644
+--- a/CI/test_cases/image_cases/ro_separate.sh
++++ b/CI/test_cases/image_cases/ro_separate.sh
+@@ -45,6 +45,9 @@ function test_separate_ro()
+   isula stop test_separate
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
+ 
++  isula rm test_separate
++  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++))
++
+   isula rmi busybox
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
+ 
+-- 
+2.25.1
+

0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch

@@ -0,0 +1,99 @@
+From 569e2d07cc153f2918868ba58bc7da9a626e4db0 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Tue, 7 Mar 2023 23:59:56 -0800
+Subject: [PATCH 52/56] bugfix remote ro try add or remove image/layer twice
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ .../image/oci/storage/image_store/image_store.c    | 14 ++++++++++++++
+ .../oci/storage/layer_store/layer_remote_impl.c    |  2 +-
+ .../image/oci/storage/layer_store/layer_store.c    | 11 +++++++++++
+ 3 files changed, 26 insertions(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
+index c6836e46..a44bf362 100644
+--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
++++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
+@@ -3671,6 +3671,11 @@ int append_image_by_directory_with_lock(const char *id)
+         return -1;
+     }
+ 
++    if (map_search(g_image_store->byid, (void *)id) != NULL ) {
++        DEBUG("remote image already exist, not added: %s", id);
++        goto out;
++    }
++
+     nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id);
+     if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
+         ERROR("Failed to get image path");
+@@ -3678,6 +3683,8 @@ int append_image_by_directory_with_lock(const char *id)
+     }
+ 
+     ret = append_image_by_directory(image_path);
++
++out:
+     image_store_unlock();
+ 
+     return ret;
+@@ -3692,7 +3699,14 @@ int remove_image_from_memory_with_lock(const char *id)
+         return -1;
+     }
+ 
++    if (map_search(g_image_store->byid, (void *)id) == NULL) {
++        DEBUG("remote image already remvoed, don't delete twice: %s", id);
++        goto out;
++    }
++
+     ret = remove_image_from_memory(id);
++
++out:
+     image_store_unlock();
+ 
+     return ret;
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
+index d03fc20b..d676458c 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
+@@ -175,7 +175,7 @@ static int remote_support_add(void *data)
+         }
+ 
+         if (add_one_remote_layer(data, array_added[i]) != 0) {
+-            ERROR("Failed to add remote overlay layer: %s", array_added[i]);
++            ERROR("Failed to add remote layer: %s", array_added[i]);
+             ret = -1;
+         }
+     }
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+index 8b8f5f1e..29ead711 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+@@ -1855,6 +1855,11 @@ int load_one_layer(const char *id)
+         return -1;
+     }
+ 
++    if (map_search(g_metadata.by_id, (void *)id) != NULL) {
++        DEBUG("remote layer already exist, not added: %s", id);
++        goto unlock_out;
++    }
++
+     tl = load_one_layer_from_json(id);
+     if (tl == NULL) {
+         ret = -1;
+@@ -2485,8 +2490,14 @@ int remove_memory_stores_with_lock(const char *id)
+         ERROR("Failed to lock layer store when handle: %s", id);
+         return -1;
+     }
++    if (map_search(g_metadata.by_id, (void *)id) == NULL) {
++        DEBUG("remote layer already removed, don't delete: %s", id);
++        goto unlock_out;
++    }
+ 
+     ret = remove_memory_stores(id);
++
++unlock_out:
+     layer_store_unlock();
+ 
+     return ret;
+-- 
+2.25.1
+

0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch

@@ -0,0 +1,34 @@
+From bbcc4c58d67aeceee55fcc1126deedafe0e43c20 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Tue, 14 Mar 2023 20:32:23 -0700
+Subject: [PATCH 53/56] bugfix can't delete layers under dir overlay-layers
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ .../modules/image/oci/storage/layer_store/layer_store.c   | 8 ++++++--
+ 1 file changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+index 29ead711..680b35a2 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+@@ -1412,10 +1412,14 @@ static int do_delete_layer(const char *id)
+     }
+ 
+ #ifdef ENABLE_REMOTE_LAYER_STORE
+-    if (l->slayer->writable) {
++    if (!g_enable_remote_layer) {
+         ret = layer_store_remove_layer(l->slayer->id);
+     } else {
+-        ret = remote_layer_remove_ro_dir(l->slayer->id);
++        if (l->slayer->writable) {
++            ret = layer_store_remove_layer(l->slayer->id);
++        } else {
++            ret = remote_layer_remove_ro_dir(l->slayer->id);
++        }
+     }
+ #else
+     ret = layer_store_remove_layer(l->slayer->id);
+-- 
+2.25.1
+

0055-bugfix-when-refresh-can-t-load-or-pull-images.patch

@@ -0,0 +1,319 @@
+From e15b302f7001507ebbf37d5485143f3124f8f54f Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Mon, 20 Mar 2023 23:47:25 -0700
+Subject: [PATCH 55/56] bugfix when refresh can't load or pull images
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ src/daemon/modules/image/oci/oci_image.c      | 105 +++++++++++++++++-
+ .../remote_layer_support/remote_support.c     |  34 +++++-
+ .../remote_layer_support/remote_support.h     |   4 +-
+ .../modules/image/oci/storage/storage.c       |   2 +-
+ .../modules/image/oci/storage/storage.h       |   2 +
+ 5 files changed, 143 insertions(+), 4 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
+index e4442a79..06a0c6ac 100644
+--- a/src/daemon/modules/image/oci/oci_image.c
++++ b/src/daemon/modules/image/oci/oci_image.c
+@@ -41,6 +41,39 @@
+ 
+ struct oci_image_module_data g_oci_image_module_data = { 0 };
+ 
++#ifdef ENABLE_REMOTE_LAYER_STORE
++// intend to make remote refresh and oci ops exlusive
++static bool g_enable_remote;
++static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER;
++
++static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable)
++{
++    int nret = 0;
++
++    if (writable) {
++        nret = pthread_rwlock_wrlock(remote_lock);
++    } else {
++        nret = pthread_rwlock_rdlock(remote_lock);
++    }
++    if (nret != 0) {
++        ERROR("Lock memory store failed: %s", strerror(nret));
++        return false;
++    }
++
++    return true;
++}
++
++static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock)
++{
++    int nret = 0;
++
++    nret = pthread_rwlock_unlock(remote_lock);
++    if (nret != 0) {
++        FATAL("Unlock memory store failed: %s", strerror(nret));
++    }
++}
++#endif
++
+ static void free_oci_image_data(void)
+ {
+     free(g_oci_image_module_data.root_dir);
+@@ -216,6 +249,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args)
+ 
+ #ifdef ENABLE_REMOTE_LAYER_STORE
+     storage_opts->enable_remote_layer = args->storage_enable_remote_layer;
++    storage_opts->remote_lock = &g_remote_lock;
+ #endif
+ 
+     if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts,
+@@ -299,6 +333,10 @@ int oci_init(const isulad_daemon_configs *args)
+         goto out;
+     }
+ 
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    g_enable_remote = args->storage_enable_remote_layer;
++#endif
++
+     if (storage_module_init_helper(args) != 0) {
+         ret = -1;
+         goto out;
+@@ -317,6 +355,7 @@ void oci_exit()
+ 
+ int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
+ {
++    int ret = 0;
+     if (request == NULL || request->image == NULL || response == NULL) {
+         ERROR("Invalid NULL param");
+         return -1;
+@@ -327,8 +366,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
+         isulad_try_set_error_message("Invalid image name: %s", request->image);
+         return -1;
+     }
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    // read lock here because pull have exclusive access against remote refresh
++    // pull can work concurrently with other oci operations.
++    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
++        ERROR("Failed to lock oci remote lock when load image");
++        return -1;
++    }
++#endif
++
++    ret = oci_do_pull_image(request, response);
++
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    if (g_enable_remote) {
++        oci_remote_unlock(&g_remote_lock);
++    }
++#endif
+ 
+-    return oci_do_pull_image(request, response);
++    return ret;
+ }
+ 
+ int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs)
+@@ -437,6 +492,15 @@ int oci_rmi(const im_rmi_request *request)
+         return -1;
+     }
+ 
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    // read lock here because load have exclusive access against remote refresh
++    // load can work concurrently with other oci operations.
++    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
++        ERROR("Failed to lock oci remote lock when load image");
++        return -1;
++    }
++#endif
++
+     if (!util_valid_image_name(request->image.image)) {
+         ERROR("Invalid image name: %s", request->image.image);
+         isulad_try_set_error_message("Invalid image name: %s", request->image.image);
+@@ -498,6 +562,11 @@ int oci_rmi(const im_rmi_request *request)
+     }
+ 
+ out:
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    if (g_enable_remote) {
++        oci_remote_unlock(&g_remote_lock);
++    }
++#endif
+     free(real_image_name);
+     free(image_ID);
+     util_free_array_by_len(image_names, image_names_len);
+@@ -523,7 +592,24 @@ int oci_import(const im_import_request *request, char **id)
+         goto err_out;
+     }
+ 
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    // read lock here because import have exclusive access against remote refresh
++    // import can work concurrently with other oci operations.
++    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
++        ERROR("Failed to lock oci remote lock when load image");
++        ret = -1;
++        goto err_out;
++    }
++#endif
++
+     ret = oci_do_import(request->file, dest_name, id);
++
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    if (g_enable_remote) {
++        oci_remote_unlock(&g_remote_lock);
++    }
++#endif
++
+     if (ret != 0) {
+         goto err_out;
+     }
+@@ -673,7 +759,24 @@ int oci_load_image(const im_load_request *request)
+         goto out;
+     }
+ 
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    // read lock here because load have exclusive access against remote refresh
++    // load can work concurrently with other oci operations.
++    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
++        ERROR("Failed to lock oci remote lock when load image");
++        ret = -1;
++        goto out;
++    }
++#endif
++
+     ret = oci_do_load(request);
++
++#ifdef ENABLE_REMOTE_LAYER_STORE
++    if (g_enable_remote) {
++        oci_remote_unlock(&g_remote_lock);
++    }
++#endif
++
+     if (ret != 0) {
+         ERROR("Failed to load image");
+         goto out;
+diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
+index 3c7d0f54..7d457755 100644
+--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
+@@ -24,10 +24,38 @@ struct supporters {
+     struct remote_image_data *image_data;
+     struct remote_layer_data *layer_data;
+     struct remote_overlay_data *overlay_data;
++    pthread_rwlock_t *remote_lock;
+ };
+ 
+ static struct supporters supporters;
+ 
++static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable)
++{
++    int nret = 0;
++
++    if (writable) {
++        nret = pthread_rwlock_wrlock(remote_lock);
++    } else {
++        nret = pthread_rwlock_rdlock(remote_lock);
++    }
++    if (nret != 0) {
++        ERROR("Lock memory store failed: %s", strerror(nret));
++        return false;
++    }
++
++    return true;
++}
++
++static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock)
++{
++    int nret = 0;
++
++    nret = pthread_rwlock_unlock(remote_lock);
++    if (nret != 0) {
++        FATAL("Unlock memory store failed: %s", strerror(nret));
++    }
++}
++
+ static void *remote_refresh_ro_symbol_link(void *arg)
+ {
+     struct supporters *refresh_supporters = (struct supporters *)arg;
+@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg)
+         util_usleep_nointerupt(5 * 1000 * 1000);
+         DEBUG("remote refresh start\n");
+ 
++        remote_refresh_lock(supporters.remote_lock, true);
+         remote_overlay_refresh(refresh_supporters->overlay_data);
+         remote_layer_refresh(refresh_supporters->layer_data);
+         remote_image_refresh(refresh_supporters->image_data);
++        remote_refresh_unlock(supporters.remote_lock);
+ 
+         DEBUG("remote refresh end\n");
+     }
+     return NULL;
+ }
+ 
+-int remote_start_refresh_thread(void)
++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock)
+ {
+     int res = 0;
+     pthread_t a_thread;
+@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void)
+         goto free_out;
+     }
+ 
++    supporters.remote_lock = remote_lock;
++
+     res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters);
+     if (res != 0) {
+         CRIT("Thread creation failed");
+diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
+index 892a9155..30e3ebb0 100644
+--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
++++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
+@@ -16,6 +16,8 @@
+ #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
+ #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
+ 
++#include <pthread.h>
++
+ #include "linked_list.h"
+ #include "map.h"
+ #include "ro_symlink_maintain.h"
+@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data);
+ bool remote_overlay_layer_valid(const char *layer_id);
+ 
+ // start refresh remote
+-int remote_start_refresh_thread(void);
++int remote_start_refresh_thread(pthread_rwlock_t *remote_lock);
+ 
+ // extra map utils
+ char **remote_deleted_layers(const map_t *old, const map_t *new_l);
+diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
+index f9830ac3..836ccf4d 100644
+--- a/src/daemon/modules/image/oci/storage/storage.c
++++ b/src/daemon/modules/image/oci/storage/storage.c
+@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts)
+     }
+ 
+ #ifdef ENABLE_REMOTE_LAYER_STORE
+-    if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) {
++    if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) {
+         ERROR("Failed to start remote refresh thread");
+     }
+ #endif
+diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
+index 7404ee54..df9fd761 100644
+--- a/src/daemon/modules/image/oci/storage/storage.h
++++ b/src/daemon/modules/image/oci/storage/storage.h
+@@ -18,6 +18,7 @@
+ #include <stdint.h>
+ #include <stdbool.h>
+ #include <stddef.h>
++#include <pthread.h>
+ #include <isula_libutils/imagetool_image.h>
+ #include <isula_libutils/json_common.h>
+ 
+@@ -72,6 +73,7 @@ struct storage_module_init_options {
+     bool integration_check;
+ #ifdef ENABLE_REMOTE_LAYER_STORE
+     bool enable_remote_layer;
++    pthread_rwlock_t *remote_lock;
+ #endif
+ };
+ 
+-- 
+2.25.1
+

0056-remove-unused-headers.patch

@@ -0,0 +1,40 @@
+From 2b798cf4053298dc44304319073cda1a00a466f4 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Thu, 27 Apr 2023 00:26:15 -0700
+Subject: [PATCH 56/56] remove unused headers
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ src/daemon/modules/image/oci/storage/layer_store/layer_store.h | 1 -
+ src/daemon/modules/image/oci/storage/storage.h                 | 2 ++
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
+index 4677e5ee..be8c52dc 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
+@@ -23,7 +23,6 @@
+ 
+ #include "storage.h"
+ #include "io_wrapper.h"
+-#include "map.h"
+ 
+ struct io_read_wrapper;
+ struct layer_list;
+diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
+index df9fd761..a761938c 100644
+--- a/src/daemon/modules/image/oci/storage/storage.h
++++ b/src/daemon/modules/image/oci/storage/storage.h
+@@ -18,7 +18,9 @@
+ #include <stdint.h>
+ #include <stdbool.h>
+ #include <stddef.h>
++#ifdef ENABLE_REMOTE_LAYER_STORE
+ #include <pthread.h>
++#endif
+ #include <isula_libutils/imagetool_image.h>
+ #include <isula_libutils/json_common.h>
+ 
+-- 
+2.25.1
+

iSulad.spec

@@ -1,5 +1,5 @@
 %global _version 2.0.18
-%global _release 5
+%global _release 6
 %global is_systemd 1
 %global enable_shimv2 1
 %global is_embedded 1
@@ -59,6 +59,17 @@ Patch0043:	0043-isulad-shim-fix-log-loss-bug.patch
 Patch0044:	0044-remove-unused-func.patch
 Patch0045:	0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch
 Patch0046:	0046-free-timeout-when-shim_create-finished.patch
+Patch0047:	0047-clean-isulad-shim-compile-relies.patch
+Patch0048:	0048-remote-layer-store-demo.patch
+Patch0049:	0049-add-ci-for-remote-ro.patch
+Patch0050:	0050-fix-compile-error-when-not-enable-remote-ro.patch
+Patch0051:	0051-CI-not-enable-remote-ro-for-ut.patch
+Patch0052:	0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch
+Patch0053:	0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch
+Patch0054:	0054-refactor-remote-ro-code.patch
+Patch0055:	0055-bugfix-when-refresh-can-t-load-or-pull-images.patch
+Patch0056:	0056-remove-unused-headers.patch
+Patch0057:	0057-change-isulad-shim-epoll-struct.patch
 
 %ifarch x86_64 aarch64
 Provides:       libhttpclient.so()(64bit)
@@ -303,6 +314,12 @@ fi
 %endif
 
 %changelog
+* Thu May 04 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-6
+- Type: bugfix
+- ID: NA
+- SUG: NA
+- DESC: upgrade from upstream
+
 * Mon Apr 24 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-5
 - Type: bugfix
 - ID: NA