packages/iSulad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!563 [sync] PR-561: upgrade from upstream

Browse Source 
Merge pull request !563 from openeuler-sync-bot/sync-pr561-openEuler-22.03-LTS-to-openEuler-22.03-LTS-Next
This commit is contained in:
haozi007 2023-05-05 02:26:00 +00:00 committed by Gitee
commit e35ece16a0
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
12 changed files with 7251 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

347
0047-clean-isulad-shim-compile-relies.patch Normal file
View File

@ -0,0 +1,347 @@
From fde1c406a837b849f1182d8943f1f942088b608d Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Sun, 23 Apr 2023 15:01:58 +0800
Subject: [PATCH 47/56] clean isulad shim compile relies
Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
src/CMakeLists.txt | 12 +-
src/cmd/isulad-shim/common.c | 179 ++++++++++++++++++++++++++++
src/cmd/isulad-shim/common.h | 49 ++++++++
src/cmd/isulad-shim/process.c | 3 -
test/cmd/isulad-shim/CMakeLists.txt | 13 --
5 files changed, 238 insertions(+), 18 deletions(-)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f3dd3c19..02d7b13f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -113,8 +113,16 @@ endif()
add_executable(isulad-shim
${ISULAD_SHIM_SRCS}
)
-target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS})
-target_link_libraries(isulad-shim libisulad_tools)
+target_include_directories(isulad-shim PUBLIC
+ ${ISULAD_SHIM_INCS}
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ ${COMMON_INCS}
+ ${CMAKE_BINARY_DIR}/conf
+ ${CHECKED_INCLUDE_DIRS}
+ ${SHARED_INCS}
+ ${ISULA_LIBUTILS_INCLUDE_DIR}
+ )
+target_link_libraries(isulad-shim ${ISULA_LIBUTILS_LIBRARY})
if (ANDROID OR MUSL)
target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY})
else()
diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
index f188da1e..e1ca96e1 100644
--- a/src/cmd/isulad-shim/common.c
+++ b/src/cmd/isulad-shim/common.c
@@ -27,6 +27,7 @@
#include <stdbool.h>
#include <stdarg.h>
#include <limits.h>
+#include <time.h>
int set_fd_no_inherited(int fd)
{
@@ -346,3 +347,181 @@ int shim_util_safe_uint64(const char *numstr, uint64_t *converted)
return 0;
}
+void util_usleep_nointerupt(unsigned long usec)
+{
+#define SECOND_TO_USECOND_MUTIPLE 1000000
+ int ret = 0;
+ struct timespec request = { 0 };
+ struct timespec remain = { 0 };
+ if (usec == 0) {
+ return;
+ }
+
+ request.tv_sec = (time_t)(usec / SECOND_TO_USECOND_MUTIPLE);
+ request.tv_nsec = (long)((usec % SECOND_TO_USECOND_MUTIPLE) * 1000);
+
+ do {
+ ret = nanosleep(&request, &remain);
+ request = remain;
+ } while (ret == -1 && errno == EINTR);
+}
+
+void *util_smart_calloc_s(size_t unit_size, size_t count)
+{
+ if (unit_size == 0) {
+ return NULL;
+ }
+
+ if (count > (MAX_MEMORY_SIZE / unit_size)) {
+ return NULL;
+ }
+
+ return calloc(count, unit_size);
+}
+
+size_t util_array_len(const char **array)
+{
+ const char **pos;
+ size_t len = 0;
+
+ for (pos = array; pos != NULL && *pos != NULL; pos++) {
+ len++;
+ }
+
+ return len;
+}
+
+void util_free_array(char **array)
+{
+ char **p;
+
+ for (p = array; p != NULL && *p != NULL; p++) {
+ UTIL_FREE_AND_SET_NULL(*p);
+ }
+ free(array);
+}
+
+int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment)
+{
+ size_t add_capacity;
+ char **add_array = NULL;
+
+ if (orig_array == NULL || orig_capacity == NULL || increment == 0) {
+ return -1;
+ }
+
+ if (((*orig_array) == NULL) || ((*orig_capacity) == 0)) {
+ UTIL_FREE_AND_SET_NULL(*orig_array);
+ *orig_capacity = 0;
+ }
+
+ add_capacity = *orig_capacity;
+ while (size + 1 > add_capacity) {
+ add_capacity += increment;
+ }
+ if (add_capacity != *orig_capacity) {
+ add_array = util_smart_calloc_s(sizeof(void *), add_capacity);
+ if (add_array == NULL) {
+ return -1;
+ }
+ if (*orig_array != NULL) {
+ (void)memcpy(add_array, *orig_array, *orig_capacity * sizeof(void *));
+ UTIL_FREE_AND_SET_NULL(*orig_array);
+ }
+
+ *orig_array = add_array;
+ *orig_capacity = add_capacity;
+ }
+
+ return 0;
+}
+
+char *util_strdup_s(const char *src)
+{
+ char *dst = NULL;
+
+ if (src == NULL) {
+ return NULL;
+ }
+
+ dst = strdup(src);
+ if (dst == NULL) {
+ abort();
+ }
+
+ return dst;
+}
+
+static char **make_empty_array()
+{
+ char **res_array = NULL;
+
+ res_array = calloc(2, sizeof(char *));
+ if (res_array == NULL) {
+ return NULL;
+ }
+ res_array[0] = util_strdup_s("");
+ return res_array;
+}
+
+static char **util_shrink_array(char **orig_array, size_t new_size)
+{
+ char **new_array = NULL;
+ size_t i = 0;
+
+ if (new_size == 0) {
+ return orig_array;
+ }
+ new_array = util_smart_calloc_s(sizeof(char *), new_size);
+ if (new_array == NULL) {
+ return orig_array;
+ }
+
+ for (i = 0; i < new_size; i++) {
+ new_array[i] = orig_array[i];
+ }
+ free(orig_array);
+ return new_array;
+}
+
+char **util_string_split_multi(const char *src_str, char delim)
+{
+ int ret, tmp_errno;
+ char *token = NULL;
+ char *cur = NULL;
+ char **res_array = NULL;
+ char deli[2] = { delim, '\0' };
+ size_t count = 0;
+ size_t capacity = 0;
+ char *tmpstr = NULL;
+
+ if (src_str == NULL) {
+ return NULL;
+ }
+
+ if (src_str[0] == '\0') {
+ return make_empty_array();
+ }
+
+ tmpstr = util_strdup_s(src_str);
+ cur = tmpstr;
+ token = strsep(&cur, deli);
+ while (token != NULL) {
+ ret = util_grow_array(&res_array, &capacity, count + 1, 16);
+ if (ret < 0) {
+ goto err_out;
+ }
+ res_array[count] = util_strdup_s(token);
+ count++;
+ token = strsep(&cur, deli);
+ }
+ free(tmpstr);
+ return util_shrink_array(res_array, count + 1);
+
+err_out:
+ tmp_errno = errno;
+ free(tmpstr);
+ util_free_array(res_array);
+ errno = tmp_errno;
+ return NULL;
+}
\ No newline at end of file
diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
index 91808295..3de16ace 100644
--- a/src/cmd/isulad-shim/common.h
+++ b/src/cmd/isulad-shim/common.h
@@ -58,6 +58,43 @@ extern "C" {
#define CONTAINER_ACTION_REBOOT 129
#define CONTAINER_ACTION_SHUTDOWN 130
+
+void util_usleep_nointerupt(unsigned long usec);
+/**
+ * retry_cnt: max count of call cb;
+ * interval_us: how many us to sleep, after call cb;
+ * cb: retry call function;
+ * return:
+ * 0 is cb successful at least once;
+ * 1 is all cb are failure;
+*/
+#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do { \
+ size_t i = 0; \
+ for(; i < retry_cnt; i++) { \
+ ret = cb(##__VA_ARGS__); \
+ if (ret == 0) { \
+ break; \
+ } \
+ util_usleep_nointerupt(interval_us); \
+ } \
+ } while(0)
+
+#define UTIL_FREE_AND_SET_NULL(p) \
+ do { \
+ if ((p) != NULL) { \
+ free((void *)(p)); \
+ (p) = NULL; \
+ } \
+ } while (0)
+
+#if __WORDSIZE == 64
+// current max user memory for 64-machine is 2^47 B
+#define MAX_MEMORY_SIZE ((size_t)1 << 47)
+#else
+// current max user memory for 32-machine is 2^31 B
+#define MAX_MEMORY_SIZE ((size_t)1 << 31)
+#endif
+
ssize_t read_nointr(int fd, void *buf, size_t count);
ssize_t write_nointr(int fd, const void *buf, size_t count);
@@ -79,6 +116,18 @@ int open_no_inherit(const char *path, int flag, mode_t mode);
int shim_util_safe_uint64(const char *numstr, uint64_t *converted);
+void *util_smart_calloc_s(size_t unit_size, size_t count);
+
+size_t util_array_len(const char **array);
+
+void util_free_array(char **array);
+
+int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size, size_t increment);
+
+char *util_strdup_s(const char *src);
+
+char **util_string_split_multi(const char *src_str, char delim);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
index a676e7ce..a5e0bd39 100644
--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
@@ -37,9 +37,6 @@
#include "common.h"
#include "terminal.h"
-#include "utils_array.h"
-#include "utils_string.h"
-#include "utils.h"
#define MAX_EVENTS 100
#define DEFAULT_IO_COPY_BUF (16 * 1024)
diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt
index dc293f6d..e5c1cd6e 100644
--- a/test/cmd/isulad-shim/CMakeLists.txt
+++ b/test/cmd/isulad-shim/CMakeLists.txt
@@ -6,26 +6,13 @@ add_executable(${EXE}
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_file.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_convert.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_verify.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/map.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c
isulad-shim_ut.cc)
target_include_directories(${EXE} PUBLIC
${GTEST_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256
- ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils
${CMAKE_BINARY_DIR}/conf
)
target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
--
2.25.1

2290
0048-remote-layer-store-demo.patch Normal file
View File

File diff suppressed because it is too large Load Diff

119
0049-add-ci-for-remote-ro.patch Normal file
View File

@ -0,0 +1,119 @@
From 1b1900c183886e13edd86c2e2cb6e5c42cfebd3d Mon Sep 17 00:00:00 2001
From: Neil <wrz750726@gmail.com>
Date: Sun, 5 Mar 2023 12:23:29 +0000
Subject: [PATCH 49/56] add ci for remote ro
Signed-off-by: Neil <wangrunze13@huawei.com>
---
CI/make-and-install.sh | 4 +-
.../container_cases/test_data/daemon.json | 1 +
CI/test_cases/image_cases/ro_separate.sh | 69 +++++++++++++++++++
3 files changed, 72 insertions(+), 2 deletions(-)
create mode 100644 CI/test_cases/image_cases/ro_separate.sh
diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
index 81022d75..e714d206 100755
--- a/CI/make-and-install.sh
+++ b/CI/make-and-install.sh
@@ -106,9 +106,9 @@ rm -rf build
mkdir build
cd build
if [[ ${enable_gcov} -ne 0 ]]; then
- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON ..
+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
else
- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON ..
+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
fi
make -j $(nproc)
make install
diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
index aa88c9da..2664c6b2 100644
--- a/CI/test_cases/container_cases/test_data/daemon.json
+++ b/CI/test_cases/container_cases/test_data/daemon.json
@@ -19,6 +19,7 @@
"hook-spec": "/etc/default/isulad/hooks/default.json",
"start-timeout": "2m",
"storage-driver": "overlay2",
+ "storage-enable-remote-layer": false,
"storage-opts": [
"overlay2.override_kernel_check=true"
],
diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
new file mode 100644
index 00000000..47e04abb
--- /dev/null
+++ b/CI/test_cases/image_cases/ro_separate.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# attributes: isulad basic image
+# concurrent: NA
+# spend time: 22
+
+#######################################################################
+##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# - http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: wangrunze
+##- @Create: 2023-03-03
+#######################################################################
+
+declare -r curr_path=$(dirname $(readlink -f "$0"))
+source ../helpers.sh
+single_image="${curr_path}/busybox.tar"
+
+function test_separate_ro()
+{
+ local ret=0
+ local test="isula separate ro test => (${FUNCNAME[@]})"
+
+ msg_info "${test} starting..."
+
+ sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json
+ start_isulad_with_valgrind
+ wait_isulad_running
+
+ isula rmi busybox
+
+ isula pull busybox
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++))
+
+ isula run -tid --name test_separate busybox /bin/sh
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++))
+
+ isula stop test_separate
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
+
+ isula rmi busybox
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
+
+ isula load -i $single_image
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++))
+
+ check_valgrind_log
+ [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++))
+
+ sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json
+ start_isulad_with_valgrind
+ wait_isulad_running
+
+ msg_info "${test} finished with return ${ret}..."
+ return ${ret}
+}
+
+declare -i ans=0
+
+test_separate_ro || ((ans++))
+
+show_result ${ans} "${curr_path}/${0}"
--
2.25.1

62
0050-fix-compile-error-when-not-enable-remote-ro.patch Normal file
View File

@ -0,0 +1,62 @@
From e4f309f61b169529c263b7a83a0eda16ebe132f5 Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Sun, 5 Mar 2023 18:55:40 -0800
Subject: [PATCH 50/56] fix compile error when not enable remote ro
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
.../modules/image/oci/storage/image_store/CMakeLists.txt | 3 +++
.../modules/image/oci/storage/layer_store/CMakeLists.txt | 4 ++++
.../storage/layer_store/graphdriver/overlay2/CMakeLists.txt | 3 +++
3 files changed, 10 insertions(+)
diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
index ecf21caa..7d4fb77c 100644
--- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
+++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
@@ -1,5 +1,8 @@
# get current directory sources files
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs)
+IF (NOT ENABLE_REMOTE_LAYER_STORE)
+list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c")
+ENDIF()
set(IMAGE_STORE_SRCS
${local_image_store_srcs}
diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
index f964f709..e04b4ad7 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
+++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
@@ -1,5 +1,8 @@
# get current directory sources files
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs)
+IF (NOT ENABLE_REMOTE_LAYER_STORE)
+list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c")
+ENDIF()
add_subdirectory(graphdriver)
set(LAYER_STORE_SRCS
@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS
${GRAPHDRIVER_SRCS}
PARENT_SCOPE
)
+
set(LAYER_STORE_INCS
${CMAKE_CURRENT_SOURCE_DIR}
${GRAPHDRIVER_INCS}
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
index ceed16b7..dd4e82aa 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
@@ -1,5 +1,8 @@
# get current directory sources files
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs)
+IF (NOT ENABLE_REMOTE_LAYER_STORE)
+list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c")
+ENDIF()
set(OVERLAY2_SRCS
${local_overlay2_srcs}
--
2.25.1

41
0051-CI-not-enable-remote-ro-for-ut.patch Normal file
View File

@ -0,0 +1,41 @@
From 1d5aa99d97df3f4ddec4ae436cb0ccbbba3e863a Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Mon, 6 Mar 2023 18:59:43 -0800
Subject: [PATCH 51/56] CI not enable remote ro for ut
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
CI/make-and-install.sh | 2 +-
CI/test_cases/image_cases/ro_separate.sh | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
index e714d206..7c194826 100755
--- a/CI/make-and-install.sh
+++ b/CI/make-and-install.sh
@@ -106,7 +106,7 @@ rm -rf build
mkdir build
cd build
if [[ ${enable_gcov} -ne 0 ]]; then
- cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
+ cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON ..
else
cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
fi
diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
index 47e04abb..df45e120 100644
--- a/CI/test_cases/image_cases/ro_separate.sh
+++ b/CI/test_cases/image_cases/ro_separate.sh
@@ -45,6 +45,9 @@ function test_separate_ro()
isula stop test_separate
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
+ isula rm test_separate
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++))
+
isula rmi busybox
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
--
2.25.1

99
0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch Normal file
View File

@ -0,0 +1,99 @@
From 569e2d07cc153f2918868ba58bc7da9a626e4db0 Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Tue, 7 Mar 2023 23:59:56 -0800
Subject: [PATCH 52/56] bugfix remote ro try add or remove image/layer twice
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
.../image/oci/storage/image_store/image_store.c | 14 ++++++++++++++
.../oci/storage/layer_store/layer_remote_impl.c | 2 +-
.../image/oci/storage/layer_store/layer_store.c | 11 +++++++++++
3 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
index c6836e46..a44bf362 100644
--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
@@ -3671,6 +3671,11 @@ int append_image_by_directory_with_lock(const char *id)
return -1;
}
+ if (map_search(g_image_store->byid, (void *)id) != NULL ) {
+ DEBUG("remote image already exist, not added: %s", id);
+ goto out;
+ }
+
nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id);
if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
ERROR("Failed to get image path");
@@ -3678,6 +3683,8 @@ int append_image_by_directory_with_lock(const char *id)
}
ret = append_image_by_directory(image_path);
+
+out:
image_store_unlock();
return ret;
@@ -3692,7 +3699,14 @@ int remove_image_from_memory_with_lock(const char *id)
return -1;
}
+ if (map_search(g_image_store->byid, (void *)id) == NULL) {
+ DEBUG("remote image already remvoed, don't delete twice: %s", id);
+ goto out;
+ }
+
ret = remove_image_from_memory(id);
+
+out:
image_store_unlock();
return ret;
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
index d03fc20b..d676458c 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
@@ -175,7 +175,7 @@ static int remote_support_add(void *data)
}
if (add_one_remote_layer(data, array_added[i]) != 0) {
- ERROR("Failed to add remote overlay layer: %s", array_added[i]);
+ ERROR("Failed to add remote layer: %s", array_added[i]);
ret = -1;
}
}
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
index 8b8f5f1e..29ead711 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -1855,6 +1855,11 @@ int load_one_layer(const char *id)
return -1;
}
+ if (map_search(g_metadata.by_id, (void *)id) != NULL) {
+ DEBUG("remote layer already exist, not added: %s", id);
+ goto unlock_out;
+ }
+
tl = load_one_layer_from_json(id);
if (tl == NULL) {
ret = -1;
@@ -2485,8 +2490,14 @@ int remove_memory_stores_with_lock(const char *id)
ERROR("Failed to lock layer store when handle: %s", id);
return -1;
}
+ if (map_search(g_metadata.by_id, (void *)id) == NULL) {
+ DEBUG("remote layer already removed, don't delete: %s", id);
+ goto unlock_out;
+ }
ret = remove_memory_stores(id);
+
+unlock_out:
layer_store_unlock();
return ret;
--
2.25.1

34
0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch Normal file
View File

@ -0,0 +1,34 @@
From bbcc4c58d67aeceee55fcc1126deedafe0e43c20 Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Tue, 14 Mar 2023 20:32:23 -0700
Subject: [PATCH 53/56] bugfix can't delete layers under dir overlay-layers
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
.../modules/image/oci/storage/layer_store/layer_store.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
index 29ead711..680b35a2 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -1412,10 +1412,14 @@ static int do_delete_layer(const char *id)
}
#ifdef ENABLE_REMOTE_LAYER_STORE
- if (l->slayer->writable) {
+ if (!g_enable_remote_layer) {
ret = layer_store_remove_layer(l->slayer->id);
} else {
- ret = remote_layer_remove_ro_dir(l->slayer->id);
+ if (l->slayer->writable) {
+ ret = layer_store_remove_layer(l->slayer->id);
+ } else {
+ ret = remote_layer_remove_ro_dir(l->slayer->id);
+ }
}
#else
ret = layer_store_remove_layer(l->slayer->id);
--
2.25.1

2089
0054-refactor-remote-ro-code.patch Normal file
View File

File diff suppressed because it is too large Load Diff

319
0055-bugfix-when-refresh-can-t-load-or-pull-images.patch Normal file
View File

@ -0,0 +1,319 @@
From e15b302f7001507ebbf37d5485143f3124f8f54f Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Mon, 20 Mar 2023 23:47:25 -0700
Subject: [PATCH 55/56] bugfix when refresh can't load or pull images
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
src/daemon/modules/image/oci/oci_image.c | 105 +++++++++++++++++-
.../remote_layer_support/remote_support.c | 34 +++++-
.../remote_layer_support/remote_support.h | 4 +-
.../modules/image/oci/storage/storage.c | 2 +-
.../modules/image/oci/storage/storage.h | 2 +
5 files changed, 143 insertions(+), 4 deletions(-)
diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
index e4442a79..06a0c6ac 100644
--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
@@ -41,6 +41,39 @@
struct oci_image_module_data g_oci_image_module_data = { 0 };
+#ifdef ENABLE_REMOTE_LAYER_STORE
+// intend to make remote refresh and oci ops exlusive
+static bool g_enable_remote;
+static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER;
+
+static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable)
+{
+ int nret = 0;
+
+ if (writable) {
+ nret = pthread_rwlock_wrlock(remote_lock);
+ } else {
+ nret = pthread_rwlock_rdlock(remote_lock);
+ }
+ if (nret != 0) {
+ ERROR("Lock memory store failed: %s", strerror(nret));
+ return false;
+ }
+
+ return true;
+}
+
+static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock)
+{
+ int nret = 0;
+
+ nret = pthread_rwlock_unlock(remote_lock);
+ if (nret != 0) {
+ FATAL("Unlock memory store failed: %s", strerror(nret));
+ }
+}
+#endif
+
static void free_oci_image_data(void)
{
free(g_oci_image_module_data.root_dir);
@@ -216,6 +249,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args)
#ifdef ENABLE_REMOTE_LAYER_STORE
storage_opts->enable_remote_layer = args->storage_enable_remote_layer;
+ storage_opts->remote_lock = &g_remote_lock;
#endif
if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts,
@@ -299,6 +333,10 @@ int oci_init(const isulad_daemon_configs *args)
goto out;
}
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ g_enable_remote = args->storage_enable_remote_layer;
+#endif
+
if (storage_module_init_helper(args) != 0) {
ret = -1;
goto out;
@@ -317,6 +355,7 @@ void oci_exit()
int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
{
+ int ret = 0;
if (request == NULL || request->image == NULL || response == NULL) {
ERROR("Invalid NULL param");
return -1;
@@ -327,8 +366,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
isulad_try_set_error_message("Invalid image name: %s", request->image);
return -1;
}
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ // read lock here because pull have exclusive access against remote refresh
+ // pull can work concurrently with other oci operations.
+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
+ ERROR("Failed to lock oci remote lock when load image");
+ return -1;
+ }
+#endif
+
+ ret = oci_do_pull_image(request, response);
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ if (g_enable_remote) {
+ oci_remote_unlock(&g_remote_lock);
+ }
+#endif
- return oci_do_pull_image(request, response);
+ return ret;
}
int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs)
@@ -437,6 +492,15 @@ int oci_rmi(const im_rmi_request *request)
return -1;
}
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ // read lock here because load have exclusive access against remote refresh
+ // load can work concurrently with other oci operations.
+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
+ ERROR("Failed to lock oci remote lock when load image");
+ return -1;
+ }
+#endif
+
if (!util_valid_image_name(request->image.image)) {
ERROR("Invalid image name: %s", request->image.image);
isulad_try_set_error_message("Invalid image name: %s", request->image.image);
@@ -498,6 +562,11 @@ int oci_rmi(const im_rmi_request *request)
}
out:
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ if (g_enable_remote) {
+ oci_remote_unlock(&g_remote_lock);
+ }
+#endif
free(real_image_name);
free(image_ID);
util_free_array_by_len(image_names, image_names_len);
@@ -523,7 +592,24 @@ int oci_import(const im_import_request *request, char **id)
goto err_out;
}
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ // read lock here because import have exclusive access against remote refresh
+ // import can work concurrently with other oci operations.
+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
+ ERROR("Failed to lock oci remote lock when load image");
+ ret = -1;
+ goto err_out;
+ }
+#endif
+
ret = oci_do_import(request->file, dest_name, id);
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ if (g_enable_remote) {
+ oci_remote_unlock(&g_remote_lock);
+ }
+#endif
+
if (ret != 0) {
goto err_out;
}
@@ -673,7 +759,24 @@ int oci_load_image(const im_load_request *request)
goto out;
}
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ // read lock here because load have exclusive access against remote refresh
+ // load can work concurrently with other oci operations.
+ if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
+ ERROR("Failed to lock oci remote lock when load image");
+ ret = -1;
+ goto out;
+ }
+#endif
+
ret = oci_do_load(request);
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+ if (g_enable_remote) {
+ oci_remote_unlock(&g_remote_lock);
+ }
+#endif
+
if (ret != 0) {
ERROR("Failed to load image");
goto out;
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
index 3c7d0f54..7d457755 100644
--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
@@ -24,10 +24,38 @@ struct supporters {
struct remote_image_data *image_data;
struct remote_layer_data *layer_data;
struct remote_overlay_data *overlay_data;
+ pthread_rwlock_t *remote_lock;
};
static struct supporters supporters;
+static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable)
+{
+ int nret = 0;
+
+ if (writable) {
+ nret = pthread_rwlock_wrlock(remote_lock);
+ } else {
+ nret = pthread_rwlock_rdlock(remote_lock);
+ }
+ if (nret != 0) {
+ ERROR("Lock memory store failed: %s", strerror(nret));
+ return false;
+ }
+
+ return true;
+}
+
+static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock)
+{
+ int nret = 0;
+
+ nret = pthread_rwlock_unlock(remote_lock);
+ if (nret != 0) {
+ FATAL("Unlock memory store failed: %s", strerror(nret));
+ }
+}
+
static void *remote_refresh_ro_symbol_link(void *arg)
{
struct supporters *refresh_supporters = (struct supporters *)arg;
@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg)
util_usleep_nointerupt(5 * 1000 * 1000);
DEBUG("remote refresh start\n");
+ remote_refresh_lock(supporters.remote_lock, true);
remote_overlay_refresh(refresh_supporters->overlay_data);
remote_layer_refresh(refresh_supporters->layer_data);
remote_image_refresh(refresh_supporters->image_data);
+ remote_refresh_unlock(supporters.remote_lock);
DEBUG("remote refresh end\n");
}
return NULL;
}
-int remote_start_refresh_thread(void)
+int remote_start_refresh_thread(pthread_rwlock_t *remote_lock)
{
int res = 0;
pthread_t a_thread;
@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void)
goto free_out;
}
+ supporters.remote_lock = remote_lock;
+
res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters);
if (res != 0) {
CRIT("Thread creation failed");
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
index 892a9155..30e3ebb0 100644
--- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
@@ -16,6 +16,8 @@
#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
+#include <pthread.h>
+
#include "linked_list.h"
#include "map.h"
#include "ro_symlink_maintain.h"
@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data);
bool remote_overlay_layer_valid(const char *layer_id);
// start refresh remote
-int remote_start_refresh_thread(void);
+int remote_start_refresh_thread(pthread_rwlock_t *remote_lock);
// extra map utils
char **remote_deleted_layers(const map_t *old, const map_t *new_l);
diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
index f9830ac3..836ccf4d 100644
--- a/src/daemon/modules/image/oci/storage/storage.c
+++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts)
}
#ifdef ENABLE_REMOTE_LAYER_STORE
- if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) {
+ if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) {
ERROR("Failed to start remote refresh thread");
}
#endif
diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
index 7404ee54..df9fd761 100644
--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -18,6 +18,7 @@
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
+#include <pthread.h>
#include <isula_libutils/imagetool_image.h>
#include <isula_libutils/json_common.h>
@@ -72,6 +73,7 @@ struct storage_module_init_options {
bool integration_check;
#ifdef ENABLE_REMOTE_LAYER_STORE
bool enable_remote_layer;
+ pthread_rwlock_t *remote_lock;
#endif
};
--
2.25.1

40
0056-remove-unused-headers.patch Normal file
View File

@ -0,0 +1,40 @@
From 2b798cf4053298dc44304319073cda1a00a466f4 Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Thu, 27 Apr 2023 00:26:15 -0700
Subject: [PATCH 56/56] remove unused headers
Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
src/daemon/modules/image/oci/storage/layer_store/layer_store.h | 1 -
src/daemon/modules/image/oci/storage/storage.h | 2 ++
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
index 4677e5ee..be8c52dc 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
@@ -23,7 +23,6 @@
#include "storage.h"
#include "io_wrapper.h"
-#include "map.h"
struct io_read_wrapper;
struct layer_list;
diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
index df9fd761..a761938c 100644
--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -18,7 +18,9 @@
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
+#ifdef ENABLE_REMOTE_LAYER_STORE
#include <pthread.h>
+#endif
#include <isula_libutils/imagetool_image.h>
#include <isula_libutils/json_common.h>
--
2.25.1

1793
0057-change-isulad-shim-epoll-struct.patch Normal file
View File

File diff suppressed because it is too large Load Diff

19
iSulad.spec
View File

@ -1,5 +1,5 @@
%global _version 2.0.18 %global _version 2.0.18
%global _release 5 %global _release 6
%global is_systemd 1 %global is_systemd 1
%global enable_shimv2 1 %global enable_shimv2 1
%global is_embedded 1 %global is_embedded 1
@ -59,6 +59,17 @@ Patch0043: 0043-isulad-shim-fix-log-loss-bug.patch
Patch0044: 0044-remove-unused-func.patch Patch0044: 0044-remove-unused-func.patch
Patch0045: 0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch Patch0045: 0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch
Patch0046: 0046-free-timeout-when-shim_create-finished.patch Patch0046: 0046-free-timeout-when-shim_create-finished.patch
Patch0047: 0047-clean-isulad-shim-compile-relies.patch
Patch0048: 0048-remote-layer-store-demo.patch
Patch0049: 0049-add-ci-for-remote-ro.patch
Patch0050: 0050-fix-compile-error-when-not-enable-remote-ro.patch
Patch0051: 0051-CI-not-enable-remote-ro-for-ut.patch
Patch0052: 0052-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch
Patch0053: 0053-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch
Patch0054: 0054-refactor-remote-ro-code.patch
Patch0055: 0055-bugfix-when-refresh-can-t-load-or-pull-images.patch
Patch0056: 0056-remove-unused-headers.patch
Patch0057: 0057-change-isulad-shim-epoll-struct.patch
%ifarch x86_64 aarch64 %ifarch x86_64 aarch64
Provides: libhttpclient.so()(64bit) Provides: libhttpclient.so()(64bit)
@ -303,6 +314,12 @@ fi
%endif %endif
%changelog %changelog
* Thu May 04 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-6
- Type: bugfix
- ID: NA
- SUG: NA
- DESC: upgrade from upstream
* Mon Apr 24 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-5 * Mon Apr 24 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.18-5
- Type: bugfix - Type: bugfix
- ID: NA - ID: NA