diff --git a/0027-modifying-cpurt-file-permissions.patch b/0027-modifying-cpurt-file-permissions.patch new file mode 100644 index 0000000..52914f2 --- /dev/null +++ b/0027-modifying-cpurt-file-permissions.patch @@ -0,0 +1,72 @@ +From eb46344fd8b7d42e6268353bfc801a1a9c8cb9a3 Mon Sep 17 00:00:00 2001 +From: songbuhuang <544824346@qq.com> +Date: Thu, 9 Mar 2023 19:38:09 +0800 +Subject: [PATCH 27/46] modifying cpurt file permissions + +Signed-off-by: songbuhuang <544824346@qq.com> +--- + src/common/constants.h | 4 ++++ + src/daemon/common/sysinfo.c | 3 +-- + src/daemon/executor/container_cb/execution.c | 3 ++- + 3 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/src/common/constants.h b/src/common/constants.h +index 93a069de..1a4cb7c4 100644 +--- a/src/common/constants.h ++++ b/src/common/constants.h +@@ -22,6 +22,10 @@ extern "C" { + + /* mode of file and directory */ + ++#define DEFAULT_CGROUP_FILE_MODE 0644 ++ ++#define DEFAULT_CGROUP_DIR_MODE 0755 ++ + #define DEFAULT_SECURE_FILE_MODE 0640 + + #define DEFAULT_SECURE_DIRECTORY_MODE 0750 +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index 8ad92b2a..cb02bee3 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -24,6 +24,7 @@ + #include + #include + ++#include "constants.h" + #include "err_msg.h" + #include "isula_libutils/log.h" + #include "utils.h" +@@ -71,8 +72,6 @@ + + #define CGROUP_MOUNTPOINT "/sys/fs/cgroup" + #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad" +-#define DEFAULT_CGROUP_DIR_MODE 0755 +-#define DEFAULT_CGROUP_FILE_MODE 0644 + #define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers" + #define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control" + #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective" +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 92c34b09..130bdaa4 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -46,6 +46,7 @@ + #include "isulad_config.h" + #include "specs_api.h" + #include "container_api.h" ++#include "constants.h" + #include "execution_extend.h" + #include "execution_information.h" + #include "execution_stream.h" +@@ -318,7 +319,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const + return 0; + } + +- ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE); ++ ret = util_mkdir_p(path, DEFAULT_CGROUP_DIR_MODE); + if (ret != 0) { + ERROR("Failed to mkdir: %s", path); + return -1; +-- +2.25.1 + diff --git a/0028-add-design-docs-for-cri-manager.patch b/0028-add-design-docs-for-cri-manager.patch new file mode 100644 index 0000000..56c65d3 --- /dev/null +++ b/0028-add-design-docs-for-cri-manager.patch @@ -0,0 +1,492 @@ +From 5cab44aa7709c743a1968ea5ac44863e6cf5434c Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Thu, 16 Mar 2023 11:41:43 +0800 +Subject: [PATCH 28/46] add design docs for cri manager + +Signed-off-by: haozi007 +--- + docs/cri_pod_manager_design.md | 432 ++++++++++++++++++ + docs/design/isulad_cri_listcontainerstats.svg | 1 + + docs/design/isulad_cri_remove_pod.svg | 1 + + docs/design/isulad_cri_run_pod.svg | 1 + + docs/design/isulad_cri_stop_pod.svg | 1 + + 5 files changed, 436 insertions(+) + create mode 100644 docs/cri_pod_manager_design.md + create mode 100644 docs/design/isulad_cri_listcontainerstats.svg + create mode 100644 docs/design/isulad_cri_remove_pod.svg + create mode 100644 docs/design/isulad_cri_run_pod.svg + create mode 100644 docs/design/isulad_cri_stop_pod.svg + +diff --git a/docs/cri_pod_manager_design.md b/docs/cri_pod_manager_design.md +new file mode 100644 +index 00000000..aa7ac165 +--- /dev/null ++++ b/docs/cri_pod_manager_design.md +@@ -0,0 +1,432 @@ ++| Author | 刘昊 | ++| ------ | ------------------------------------------ | ++| Date | 2023-03-06 | ++| Email | [liuhao27@huawei.com](liuhao27@huawei.com) | ++ ++本文主要对 `iSulad` 的 `CRI` 实现部分进行源码级别的学习和研究，为后续想了解 `CRI` 或者 `iSulad` 的同学提供帮助。 ++ ++*注：本文以iSulad 2.0.18版本为分析对象。* ++ ++本文主要是针对 `CRI` 的 `RuntimeService` 部分，涉及如下接口： ++- `RunPodSandbox` ++- `StopPodSandbox` ++- `RemovePodSandbox` ++- `PodSandboxStatus` ++- `ListPodSandbox` ++- `UpdateRuntimeConfig` ++- `Status` ++- `ContainerStats` ++- `ListContainerStats` ++ ++## CRI 代码结构 ++ ++CRI 涉及代码文件如下： ++ ++```c ++// CRI proto文件 ++src/api/services/cri/api.proto ++// proto 文件解析生成入口 ++cmake/protoc.cmake ++ ++// grpc服务注册入口 ++src/daemon/entry/connect/grpc/grpc_service.cc ++// cri 镜像部分的封装类 ++src/daemon/entry/connect/grpc/runtime_image_service.h ++// cri runtime 部分的封装类 ++src/daemon/entry/connect/grpc/runtime_runtime_service.h ++ ++// cri 具体实现目录 ++src/daemon/entry/cri/ ++``` ++ ++### CRI gRPC 服务注册和封装 ++ ++iSulad `gRPC` 模式下，直接在 `gRPC` 服务初始化时，注册了 `CRI` 服务： ++ ++```c ++// src/daemon/entry/connect/grpc/grpc_service.cc ++int Init(const struct service_arguments *args) ++{ ++ // 注册CRI runtime服务 ++ m_builder.RegisterService(&m_runtimeRuntimeService); ++ // 注册CRI 镜像服务 ++ m_builder.RegisterService(&m_runtimeImageService); ++} ++``` ++ ++`CRI` 服务初始化： ++ ++```c ++// src/daemon/entry/connect/grpc/runtime_runtime_service.cc ++void RuntimeRuntimeServiceImpl::Init(Network::NetworkPluginConf mConf, isulad_daemon_configs *config, Errors &err) ++{ ++ ... ++ // 初始化 CNI 网络插件 ++ Network::ProbeNetworkPlugins(mConf.GetPluginConfDir(), mConf.GetPluginBinDir(), &plugins); ++ Network::InitNetworkPlugin(&plugins, mConf.GetPluginName(), mConf.GetHairpinMode(), mConf.GetNonMasqueradeCIDR(), mConf.GetMTU(), &chosen, err); ++ ++ // 初始化，CRI 服务具体实现的类 ++ RuntimeVersionerService *runtimeVersioner = new RuntimeVersionerServiceImpl(cb); ++    ContainerManagerService *containerManager = new ContainerManagerServiceImpl(cb); ++    PodSandboxManagerService *podSandboxManager = new PodSandboxManagerServiceImpl(podSandboxImage, cb, pluginManager); ++    RuntimeManagerService *runtimeManager = new RuntimeManagerServiceImpl(cb, pluginManager); ++    std::unique_ptr service( ++        new CRIRuntimeServiceImpl(runtimeVersioner, containerManager, podSandboxManager, runtimeManager)); ++    rService = std::move(service); ++    // 初始化websocket服务 ++    websocket_server_init(err); ++ ... ++} ++``` ++ ++由于 `CRI` 具体实现类，定义在 `src/daemon/entry/cri/` 目录中；iSulad 通过封装的方式，在 `grpc` 服务提供了统一的入口，包括镜像和runtime两个封装类： ++- src/daemon/entry/connect/grpc/runtime_image_service.cc ++- src/daemon/entry/connect/grpc/runtime_runtime_service.cc ++ ++## PodSandbox 相关接口 ++ ++第一部分，对 `CRI` 的新增的 `Pod` 相关的接口进行详细的探讨。 ++ ++### RunPodSandbox ++ ++`RunPodSandbox` 负责 `Pod` 的创建、启动，以及网络环境准备等操作。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call RunPod ++ CRI-isula ->> iSulad: create container ++ CRI-isula ->> iSulad: inspect container ++ CRI-isula ->> cni plugin: prepare network ++ cni plugin -->> CRI-isula: return ++ CRI-isula ->> iSulad: start container ++ iSulad -->> CRI-isula: return ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++// src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc ++auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &runtimeHandler, Errors &error) -> std::string ++{ ++ ... ++ // Step1：根据Pod镜像策略，如果镜像不存在，是否下载 ++ EnsureSandboxImageExists() ++ // Step2：根据Pod配置，生成Pod对应容器的配置； ++ // 根据runtime选择对应的容器运行时； ++ CreateSandboxContainer() ++ // Step3：标识Pod网络状态为false ++ SetNetworkReady() ++ // Step4：通过inspect获取Pod信息，用于后续流程； ++ // 以及判定Pod状态是否符合预期； ++ InspectContainer() ++ // Step5：判断网络模式为file，是则先创建网络命名空间； ++ // 并且调用CNI插件，设置网络设备； ++ util_mount_namespace() ++ SetupSandboxNetwork() ++ // Step6：启动Pod ++ StartSandboxContainer() ++ // Step7：非file网络模式，则使用容器启动创建的网络名空间； ++ // 然后调用CNI插件，设置网络设备； ++ SetupSandboxNetwork() ++ ... ++} ++``` ++ ++详细的代码走读，代码流程图如下： ++ ++runpod ++ ++### StopPodSandbox ++ ++`StopPodSandbox` 负责 `Pod` 的停止，以及网络资源的清理。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call StopPod ++ CRI-isula ->> iSulad: get container info ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: stop all containers in pod ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> cni plugin: clear cni networks ++ cni plugin -->> CRI-isula: return ++ CRI-isula ->> iSulad: stop pod ++ iSulad -->> CRI-isula: return ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++void PodSandboxManagerServiceImpl::StopPodSandbox(const std::string &podSandboxID, Errors &error) ++{ ++ // Step1: get real pod ID ++ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); ++ // Step2: stop all containers in pod ++ StopAllContainersInSandbox(realSandboxID, error); ++ // Step3: clear cni network ++ ClearCniNetwork(realSandboxID, hostNetwork, ns, name, errlist, stdAnnos, error); ++ // Step4: stop pod ++ StopContainerHelper(realSandboxID, error); ++} ++``` ++ ++详细的代码走读，代码流程图如下： ++ ++stoppod ++ ++### RemovePodSandbox ++ ++`RemovePodSandbox` 负责删除 `Pod` ，包括对应容器的删除和其他数据的清理。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call RemovePod ++ CRI-isula ->> iSulad: get container info ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: remove all containers in pod ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: remove pod ++ iSulad -->> CRI-isula: return ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++void PodSandboxManagerServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Errors &error) ++{ ++ // Step1: get real pod ID ++ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); ++ // Step2: remove all containers in pod ++ RemoveAllContainersInSandbox(realSandboxID, errors); ++ // Step3: remove pod ++ DoRemovePodSandbox(realSandboxID, errors); ++} ++``` ++ ++详细的代码走读，代码流程图如下： ++ ++removepod ++ ++### PodSandboxStatus ++ ++`PodSandboxStatus` 负责获取 `Pod` 的状态信息，包括网络信息、创建时间、state、meta-data等等数据。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call Pod status ++ CRI-isula ->> iSulad: get pod real ID ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: call Inspect Pod ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set data from inspect response ++ CRI-isula ->> CNI: call network status ++ CNI -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set network ip and so on ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++std::unique_ptr ++PodSandboxManagerServiceImpl::PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++{ ++ // Step1: get real pod ID ++ GetRealSandboxIDToStop(podSandboxID, hostNetwork, name, ns, realSandboxID, stdAnnos, error); ++ // Step2: inspect pod sandbox ++ InspectContainer(realSandboxID, error, true); ++ // Step3: set status from inspect reponse ++ PodSandboxStatusToGRPC(inspect, realSandboxID, podStatus, error); ++ // Step 3.1 set timestamp ++ podStatus->set_created_at(createdAt); ++ // Step 3.2 set state ++ podStatus->set_state(runtime::v1alpha2::SANDBOX_READY); ++ // Step 3.3 set labels and annotations ++ CRIHelpers::ExtractLabels(inspect->config->labels, *podStatus->mutable_labels()); ++ CRIHelpers::ExtractAnnotations(inspect->config->annotations, *podStatus->mutable_annotations()); ++ // Step 3.4 set options ++ options->set_network(SharesHostNetwork(inspect)); ++ options->set_pid(SharesHostPid(inspect)); ++ options->set_ipc(SharesHostIpc(inspect)); ++ // Step 3.5 set network status ++ SetSandboxStatusNetwork(inspect, podSandboxID, podStatus, error); ++} ++``` ++ ++详细的代码走读，代码流程图在 [StopPod代码走读中](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 ++ ++## 通用接口 ++ ++### UpdateRuntimeConfig ++ ++`UpdateRuntimeConfig` 负责对运行配置进行更新，当前只有 `CIDR` 的配置更新。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call update runtime config ++ CRI-isula ->> network plugin: update ++ network plugin -->> cni plugin: update ++ cni plugin ->> network plugin: return ++ network plugin -->> CRI-isula: return ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++// src/daemon/entry/cri/cri_runtime_manager_service_impl.cc ++void RuntimeManagerServiceImpl::UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors & /*error*/) ++{ ++ // 设置用户传入的新CIDR值 ++ events[NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = config.network_config().pod_cidr(); ++ // 调用网络插件 --> CNI 网络模块，刷新CIDR配置 ++ m_pluginManager->Event(NET_PLUGIN_EVENT_POD_CIDR_CHANGE, events); ++} ++``` ++ ++代码结构比较简单，只是更新了类的一个成员的值；因此，不做详细的分析。 ++ ++### Status ++ ++`Status` 接口，主要用于获取容器引擎的 和 网络模块的状态是否正常。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call Status ++ CRI-isula ->> iSulad: call version ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set state of runtime ++ CRI-isula ->> network plugin: Status ++ network plugin -->> cni plugin: Status ++ cni plugin ->> network plugin: return ++ network plugin -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set state of network ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++// src/daemon/entry/cri/cri_runtime_manager_service_impl.cc ++auto RuntimeManagerServiceImpl::Status(Errors &error) -> std::unique_ptr ++{ ++ // 获取isulad的版本信息 ++ m_cb->container.version(nullptr, &response); ++ // 设置对应的状态 ++ runtimeReady->set_status(true/false); ++ // 调用网络插件的Status ++ m_pluginManager->Status(error); ++ // 设置对应的状态信息 ++ networkReady->set_status(true/false); ++} ++``` ++ ++代码结构比较简单，只是获取两个模块的状态，然后设置返回值；因此，不做详细的分析。 ++ ++## 容器相关接口 ++ ++### ContainerStats ++ ++`ContainerStats` 负责获取容器的度量数据。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call ContainerStats ++ CRI-isula ->> iSulad: call container.stats ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: call inspect ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set meta-data ++ CRI-isula ->> CRI-isula: set labels and annotations ++ CRI-isula ->> CRI-isula: set file usage ++ CRI-isula ->> CRI-isula: set memory usage ++ CRI-isula ->> CRI-isula: set cpu usage ++ CRI-isula -->> kubelet: return ++``` ++ ++源码实现，概要逻辑如下： ++ ++```c ++// src/daemon/entry/cri/cri_container_manager_service_impl.cc ++auto ContainerManagerServiceImpl::ContainerStats(const std::string &containerID, Errors &error) ++-> std::unique_ptr ++{ ++ // 获取容器的stats信息 ++ m_cb->container.stats(request, &response); ++ // 转换返回信息为CRI 对应的结果 ++ ContainerStatsToGRPC(response, &contStatsVec, error); ++ // inspect容器，获取meta-data信息，设置到返回值中 ++ PackContainerStatsAttributes(response->container_stats[i]->id, container, error); ++ // 获取容器使用的文件系统大小等度量数据 ++ PackContainerStatsFilesystemUsage(response->container_stats[i]->id, response->container_stats[i]->image_type, timestamp, container); ++ // 设置内存度量数据 ++ container->mutable_memory()->mutable_working_set_bytes()->set_value(workingset); ++ container->mutable_memory()->set_timestamp(timestamp); ++ // 设置CPU度量数据 ++ container->mutable_cpu()->mutable_usage_core_nano_seconds()->set_value(response->container_stats[i]->cpu_use_nanos); ++ container->mutable_cpu()->set_timestamp(timestamp); ++} ++``` ++ ++ ++由于 `ContainerStats` 和 `ListContainerStats` 实现基本一致，因此，详细分析部分放到一起分析，具体见 `ListContainerStats` 章节。 ++ ++### ListContainerStats ++ ++`ListContainerStats` 负责获取多容器的度量数据。 ++ ++相关组件的序列图如下： ++ ++```mermaid ++sequenceDiagram ++ kubelet ->> CRI-isula: call ContainerStats ++ CRI-isula ->> iSulad: call container.stats with filter ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> iSulad: call inspect ++ iSulad -->> CRI-isula: return ++ CRI-isula ->> CRI-isula: set meta-data ++ CRI-isula ->> CRI-isula: set labels and annotations ++ CRI-isula ->> CRI-isula: set file usage ++ CRI-isula ->> CRI-isula: set memory usage ++ CRI-isula ->> CRI-isula: set cpu usage ++ CRI-isula -->> kubelet: return ++``` ++ ++ 跟 `ContainerStats` 唯一的区别是，调用 `container->stats()` 的参数不同，`ListContainerStats` 根据过滤器去获取需要采集度量数据的容器列表，而 `ContainerStats` 只采集确定容器ID 的度量数据。 ++ ++源码实现，概要逻辑如下： ++ ++```c ++// src/daemon/entry/cri/cri_container_manager_service_impl.cc ++void ContainerManagerServiceImpl::ListContainerStats(const runtime::v1alpha2::ContainerStatsFilter *filter, std::vector> *containerstats, Errors &error) ++{ ++ // 根据用户参数，设置stats的过滤条件 ++ PackContainerStatsFilter(filter, request, error); ++ // 获取容器的stats信息 ++ m_cb->container.stats(request, &response); ++ // 转换返回信息为CRI 对应的结果 --> 与ContainerStats逻辑相同 ++ ContainerStatsToGRPC(response, &contStatsVec, error); ++} ++``` ++ ++详细的代码走读，代码流程图如下： ++ ++listcontainerstats ++ ++ +diff --git a/docs/design/isulad_cri_listcontainerstats.svg b/docs/design/isulad_cri_listcontainerstats.svg +new file mode 100644 +index 00000000..8781c39a +--- /dev/null ++++ b/docs/design/isulad_cri_listcontainerstats.svg +@@ -0,0 +1 @@ ++
ListContainerStats
PackContainerStatsFilter
call container.stats()
error?
ContainerStatsToGRPC
No
return error
Yes
sandbox id
empty?
add sandbox id filter
Yes
add type container lable
foreach selectors
finish?
add selector label
No
error?
return error
Yes
No
Yes
Yes
foreach return stats list
finish?
PackContainerStatsAttributes
No
error?
PackContainerStatsFilesystemUsage
No
mem_used
!= 0
set memory stats
nano_cpu
!= 0
set cpu stats
Yes
Yes
No
No
ContainerStatus
GetRealContainerOrSandboxID
InspectContainer
return error
Yes
return
return null
?
Yes
error?
Yes
ContainerStatusToGRPC
No
Yes
call container.get_id()
error?
Yes
got real sandbox ID
No
GetContainerTimeStamps
error?
Yes
set container start/create/finish time
No
PackContainerImageToStatus
UpdateBaseStatusFromInspect
PackLabelsToStatus
ParseContainerName
return
null?
Yes
Yes
No
has
meta?
No
add meta-data
Yes
No
label size
> 0
add labels
Yes
annotation
size > 0
add annotations
BUG：应该直接检查error
ConvertMountsToStatus
Yes
No
error?
ContainerStats
set containerID filter
 +\ No newline at end of file +diff --git a/docs/design/isulad_cri_remove_pod.svg b/docs/design/isulad_cri_remove_pod.svg +new file mode 100644 +index 00000000..f470bd17 +--- /dev/null ++++ b/docs/design/isulad_cri_remove_pod.svg +@@ -0,0 +1 @@ ++
RemovePodSandbox
GetRealContainerOrSandboxID
error?
not found
error?
Yes
return error
No
RemoveAllContainersInSandbox
No
Yes
call  container.list()
foreach return container list
RemoveContainer
DoRemovePodSandbox
error?
add error message
GetRealContainerOrSandboxID
error?
Yes
RemoveContainerLogSymlink
No
GetContainerLogPath
InspectContainer
error?
remove log path
No
call container.remove()
Yes
Yes
error?
Yes
No
cal container.remove()
pod-container
error?
ClearNetworkReady
No
not found
error?
Yes
Yes
return error
No
return
 +\ No newline at end of file +diff --git a/docs/design/isulad_cri_run_pod.svg b/docs/design/isulad_cri_run_pod.svg +new file mode 100644 +index 00000000..87ffad73 +--- /dev/null ++++ b/docs/design/isulad_cri_run_pod.svg +@@ -0,0 +1 @@ ++
RunPodSandbox
EnsureSandboxImageExists
ImageStatus
exist?
PullImage
NO
CreateSandboxContainer
Yes
MakeSandboxIsuladConfig
GenerateSandboxCreateContainerRequest
add sandbox
add pod name
add container type
add sandbox namespace 
add sandbox name
add UID
add  ATTEMPT
labels
annotations
has metadata?
Yes
set network mode
No
has linux options?
ApplySandboxLinuxOptions
Yes
ApplySandboxResources
No
has security options?
add security_opt
into hostconfig
Yes
CreateCheckpoint
save into annotation of pod
PackCreateContainerRequest
MakeSandboxName
parse runtime handler
set image
set host config
set custom config
ApplySandboxSecurityContext
run_as_user?
Yes
set user
set privileged
set readonly rootfs
No
has capabilities?
has cap
add?
set cap_adds in hostconfig
Yes
has cap
drop?
No
set cap_drops in hostconfig
Yes
Yes
add no-new-privileged into security opts
no new
privs?
Yes
No
has supplemental
groups?
No
add group_add into hostconfig
Yes
error?
No
ModifySandboxNamespaceOptions
No
return error
Yes
set pid-mode
set ipc-mode
set uts-mode
set network-mode
set cgroup_parent
cgroup parent?
Yes
sysctl size
> 0
No
add sysctls into hostconfig
Yes
No
No
call executor->create()
error?
return error
Yes
SetNetworkReady false
No
InspectContainer
ns is
file?
create new netns
Yes
SetupSandboxNetwork
StartSandboxContainer
No
ns is
file?
return response
Yes
No
SetNetworkReady true
error?
No
Yes
SetupSandboxNetwork
error?
No
StopContainerHelper
Yes
has dns?
SetupSandboxFiles
Yes
is NODE
network?
return
Yes
No
ns is
file?
No
add sandbox_key into annotations
call NetworkPlugin
SetUpPod
No
Yes
Lock Pod
call CNI SetUpPod
error?
No
return error
UnLock pod
cni inited?
No
Yes
AddToNetwork loop network
Yes
error?
Yes
SetupMultNetworks
No
TearDownMultNetworks
error?
Yes
foreach
AddToNetwork
user defined network
AddToNetwork
default network
return
No
SetupSandboxNetwork
call cni_add_network_list
 +\ No newline at end of file +diff --git a/docs/design/isulad_cri_stop_pod.svg b/docs/design/isulad_cri_stop_pod.svg +new file mode 100644 +index 00000000..b0549958 +--- /dev/null ++++ b/docs/design/isulad_cri_stop_pod.svg +@@ -0,0 +1 @@ ++
 
StopPodSandbox
empty
podID?
return
GetRealSandboxIDToStop
Yes
No
PodSandboxStatus
GetRealContainerOrSandboxID
call cb->get_id
error?
return
Yes
check input id is prefix of return id
valid
No
No
InspectContainer
return podID


PodSandboxStatusToGRPC


fill podStatus from inspect response:

1. fill pod ID;
2. fill created time;
3. fill state;
4. fill labels;
5. fill annotations;
6. fill meta-data: name, namespace, uid, attempt;
7. fill options: network, pid, ipc;
SetSandboxStatusNetwork
GetIPs
size of ip
list > 0
StopAllContainersInSandbox
No
error?
Yes
No found
error?
set real PodID to input ID
return error
No
Yes
has options
?
No
return hostNetwork
has meta
data?
return ns and name
Yes
return annotations
No
FiltersAddLabel
call container.list()
foreach return containers
error?
return error
Yes
No
StopContainer
GetRealContainerOrSandboxID
error?
Yes
error?
Yes
call container.stop()
No
No
ClearCniNetwork
StopContainerHelper
host
network?
network
ready?
No
InspectContainer
Yes
TearDownPod
SetNetworkReady false
network
type is file?
clear network namespace
Yes
Lock()
failed?
CNI TearDownPod
Unlock()
No
Yes
return 0
Yes
No
ignore error,
success forever
inited?
No
GetNetNSPath
RLockNetworkMap
error?
Yes
Yes
TearDownMultNetworks
UnlockNetworkMap
No
get user defined network list
error?
foreach user network


DeleteFromNetwork


DeleteFromNetwork
default network
Yes
No
BuildCNIRuntimeConf
error?
Get CNI bin Paths
null?
return
No
Yes
Yes
cni_del_network_list
call CNI plugin
DeleteFromNetwork
call container.stop()
finfish
StopPodSandbox
add_additional_ips
Yes
 +\ No newline at end of file +-- +2.25.1 + diff --git a/0029-improve-check-of-process-failure.patch b/0029-improve-check-of-process-failure.patch new file mode 100644 index 0000000..05fc259 --- /dev/null +++ b/0029-improve-check-of-process-failure.patch @@ -0,0 +1,43 @@ +From 5a6112a35daa7229ffb03d0dbb1df0bd1dba3469 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Thu, 16 Mar 2023 19:16:30 +0800 +Subject: [PATCH 29/46] improve check of process failure + +1. fix docs error; +2. check error to decrease Unnecessary process; + +Signed-off-by: haozi007 +--- + docs/cri_pod_manager_design.md | 2 +- + src/daemon/entry/cri/cri_container_manager_service_impl.cc | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/docs/cri_pod_manager_design.md b/docs/cri_pod_manager_design.md +index aa7ac165..91d74f5a 100644 +--- a/docs/cri_pod_manager_design.md ++++ b/docs/cri_pod_manager_design.md +@@ -263,7 +263,7 @@ PodSandboxManagerServiceImpl::PodSandboxStatus(const std::string &podSandboxID, + } + ``` + +-详细的代码走读，代码流程图在 [StopPod代码走读中](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 ++详细的代码走读，代码流程图在 [StopPod代码走读](#stoppodsandbox) 中进行了详细的分析，可以参考对应的部分。 + + ## 通用接口 + +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +index 93b939c9..0fc1884f 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +@@ -730,7 +730,7 @@ void ContainerManagerServiceImpl::PackContainerStatsAttributes( + + container->mutable_attributes()->set_id(id); + auto status = ContainerStatus(std::string(id), error); +- if (status == nullptr) { ++ if (error.NotEmpty()) { + return; + } + +-- +2.25.1 + diff --git a/0030-support-isula-update-when-runtime-is-runc.patch b/0030-support-isula-update-when-runtime-is-runc.patch new file mode 100644 index 0000000..0be335e --- /dev/null +++ b/0030-support-isula-update-when-runtime-is-runc.patch @@ -0,0 +1,251 @@ +From 0752a4324e7a8f54e4ebe5efb403221388b483d3 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 6 Mar 2023 11:31:56 +0800 +Subject: [PATCH 30/46] support isula update when runtime is runc + +Signed-off-by: zhongtao +--- + .../executor/container_cb/execution_extend.c | 1 + + src/daemon/modules/api/runtime_api.h | 1 + + .../modules/runtime/isula/isula_rt_ops.c | 167 +++++++++++++++++- + 3 files changed, 163 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c +index b0da705e..58303f80 100644 +--- a/src/daemon/executor/container_cb/execution_extend.c ++++ b/src/daemon/executor/container_cb/execution_extend.c +@@ -1129,6 +1129,7 @@ static int do_update_resources(const container_update_request *request, containe + if (container_is_running(cont->state)) { + params.rootpath = cont->root_path; + params.hostconfig = hostconfig; ++ params.state = cont->state_path; + if (runtime_update(id, cont->runtime, ¶ms)) { + ERROR("Update container %s failed", id); + ret = -1; +diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h +index c5e05ca2..a8dfdeae 100644 +--- a/src/daemon/modules/api/runtime_api.h ++++ b/src/daemon/modules/api/runtime_api.h +@@ -164,6 +164,7 @@ typedef struct _rt_attach_params_t { + typedef struct _rt_update_params_t { + const char *rootpath; + const host_config *hostconfig; ++ const char *state; + } rt_update_params_t; + + typedef struct _rt_listpids_params_t { +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 51a72c4f..4553fa90 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -39,6 +39,7 @@ + #include "constants.h" + #include "isula_libutils/shim_client_process_state.h" + #include "isula_libutils/shim_client_runtime_stats.h" ++#include "isula_libutils/shim_client_cgroup_resources.h" + #include "isula_libutils/oci_runtime_state.h" + #include "isulad_config.h" + #include "utils_string.h" +@@ -54,6 +55,9 @@ + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 + ++// file name formats of cgroup resources json ++#define RESOURCE_FNAME_FORMATS "%s/resources.json" ++ + // handle string from stderr output. + typedef int(*handle_output_callback_t)(const char *output); + +@@ -725,18 +729,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + runtime_exec_param_dump(params); + + if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { +- ERROR("failed make shim-pid full path"); ++ ERROR("Failed make shim-pid full path"); + return -1; + } + + if (pipe2(exec_fd, O_CLOEXEC) != 0) { +- ERROR("failed to create pipe for shim create"); ++ ERROR("Failed to create pipe for shim create"); + return -1; + } + + pid = fork(); + if (pid < 0) { +- ERROR("failed fork for shim parent %s", strerror(errno)); ++ ERROR("Failed fork for shim parent %s", strerror(errno)); + close(exec_fd[0]); + close(exec_fd[1]); + return -1; +@@ -1256,13 +1260,164 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ + return -1; + } + +-int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++static int to_engine_resources(const host_config *hostconfig, shim_client_cgroup_resources *cr) ++{ ++ uint64_t period = 0; ++ int64_t quota = 0; ++ ++ if (hostconfig == NULL || cr == NULL) { ++ return -1; ++ } ++ ++ cr->block_io = util_common_calloc_s(sizeof(shim_client_cgroup_resources_block_io)); ++ if (cr->block_io == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->cpu = util_common_calloc_s(sizeof(shim_client_cgroup_resources_cpu)); ++ if (cr->cpu == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->memory = util_common_calloc_s(sizeof(shim_client_cgroup_resources_memory)); ++ if (cr->memory == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ ++ cr->block_io->weight = hostconfig->blkio_weight; ++ cr->cpu->shares = (uint64_t)hostconfig->cpu_shares; ++ cr->cpu->period = (uint64_t)hostconfig->cpu_period; ++ cr->cpu->quota = hostconfig->cpu_quota; ++ cr->cpu->cpus = util_strdup_s(hostconfig->cpuset_cpus); ++ cr->cpu->mems = util_strdup_s(hostconfig->cpuset_mems); ++ cr->memory->limit = (uint64_t)hostconfig->memory; ++ cr->memory->swap = (uint64_t)hostconfig->memory_swap; ++ cr->memory->reservation = (uint64_t)hostconfig->memory_reservation; ++ cr->memory->kernel = (uint64_t)hostconfig->kernel_memory; ++ cr->cpu->realtime_period = hostconfig->cpu_realtime_period; ++ cr->cpu->realtime_runtime = hostconfig->cpu_realtime_runtime; ++ ++ // when --cpus=n is set, nano_cpus = n * 1e9. ++ if (hostconfig->nano_cpus > 0) { ++ // in the case, period will be set to the default value of 100000(0.1s). ++ period = (uint64_t)(100 * Time_Milli / Time_Micro); ++ // set quota = period * n, in order to let container process fully occupy n cpus. ++ if ((hostconfig->nano_cpus / 1e9) > (INT64_MAX / (int64_t)period)) { ++ ERROR("Overflow of quota"); ++ return -1; ++ } ++ quota = hostconfig->nano_cpus / 1e9 * (int64_t)period; ++ cr->cpu->period = period; ++ cr->cpu->quota = quota; ++ } ++ ++ return 0; ++} ++ ++static int create_resources_json_file(const char *workdir, const shim_client_cgroup_resources *cr, char *fname, ++ size_t fname_size) + { +- ERROR("isula update not support on isulad-shim"); +- isulad_set_error_message("isula update not support on isulad-shim"); ++ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 }; ++ parser_error perr = NULL; ++ char *data = NULL; ++ int retcode = 0; ++ int nret = 0; ++ ++ nret = snprintf(fname, fname_size, RESOURCE_FNAME_FORMATS, workdir); ++ if (nret < 0 || (size_t)nret >= fname_size) { ++ ERROR("Failed make resources.json full path"); ++ return -1; ++ } ++ ++ data = shim_client_cgroup_resources_generate_json(cr, &ctx, &perr); ++ if (data == NULL) { ++ retcode = -1; ++ ERROR("Failed generate json for resources.json error=%s", perr); ++ goto out; ++ } ++ ++ if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) { ++ retcode = -1; ++ ERROR("Failed write resources.json"); ++ goto out; ++ } ++ ++out: ++ UTIL_FREE_AND_SET_NULL(perr); ++ UTIL_FREE_AND_SET_NULL(data); ++ ++ return retcode; ++} ++ ++// show std error msg, always return -1. ++static int show_stderr(const char *err) ++{ ++ isulad_set_error_message(err); + return -1; + } + ++int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params) ++{ ++ int ret = 0; ++ char workdir[PATH_MAX] = { 0 }; ++ char resources_fname[PATH_MAX] = { 0 }; ++ const char *opts[2] = { 0 }; ++ shim_client_cgroup_resources *cr = NULL; ++ ++ if (id == NULL || runtime == NULL || params == NULL || params->state == NULL || strlen(params->state) == 0) { ++ ERROR("Nullptr arguments not allowed"); ++ return -1; ++ } ++ ++ ret = snprintf(workdir, sizeof(workdir), "%s/%s/update", params->state, id); ++ if (ret < 0 || (size_t)ret >= sizeof(workdir)) { ++ ERROR("Failed join update full path"); ++ return -1; ++ } ++ ++ ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE); ++ if (ret < 0) { ++ ERROR("Failed mkdir update workdir %s", workdir); ++ return ret; ++ } ++ ++ cr = util_common_calloc_s(sizeof(shim_client_cgroup_resources)); ++ if (cr == NULL) { ++ ERROR("Out of memory"); ++ goto del_out; ++ } ++ ++ ret = to_engine_resources(params->hostconfig, cr); ++ if (ret < 0) { ++ ERROR("Failed to get resources for update"); ++ goto del_out; ++ } ++ ++ ret = create_resources_json_file(workdir, cr, resources_fname, sizeof(resources_fname)); ++ if (ret != 0) { ++ ERROR("%s: failed create update json file", id); ++ goto del_out; ++ } ++ ++ opts[0] = "--resources"; ++ opts[1] = resources_fname; ++ ++ if (runtime_call_simple(workdir, runtime, "update", opts, 2, id, show_stderr) != 0) { ++ ERROR("Call runtime update id failed"); ++ ret = -1; ++ } ++ ++del_out: ++ if (util_recursive_rmdir(workdir, 0)) { ++ ERROR("Rmdir %s failed", workdir); ++ } ++ free_shim_client_cgroup_resources(cr); ++ return ret; ++} ++ + int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params) + { + char workdir[PATH_MAX] = { 0 }; +-- +2.25.1 + diff --git a/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch b/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch new file mode 100644 index 0000000..bd5c270 --- /dev/null +++ b/0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch @@ -0,0 +1,80 @@ +From dd36a6031c3f25171d34c9f8cc483af01a4dace1 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 20 Mar 2023 14:49:07 +0800 +Subject: [PATCH 31/46] when calling runc start, unset NOTIFY_ SOCKET + +Signed-off-by: sailorvii +--- + src/daemon/modules/runtime/isula/isula_rt_ops.c | 14 ++++++++------ + 1 file changed, 8 insertions(+), 6 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 4553fa90..e974964a 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -469,6 +469,12 @@ static void runtime_exec_func(void *arg) + _exit(EXIT_FAILURE); + } + ++ // clear NOTIFY_SOCKET from the env to adapt runc start ++ if (strcmp(rei->subcmd, "start") == 0 && unsetenv("NOTIFY_SOCKET") != 0) { ++ dprintf(STDERR_FILENO, "unset env NOTIFY_SOCKET failed %s", strerror(errno)); ++ _exit(EXIT_FAILURE); ++ } ++ + execvp(rei->cmd, rei->params); + dprintf(STDERR_FILENO, "exec %s %s %s failed", rei->cmd, rei->subcmd, rei->id); + _exit(EXIT_FAILURE); +@@ -941,7 +947,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + char shim_pid_file_name[PATH_MAX] = { 0 }; + pid_t pid = 0; + pid_t shim_pid = -1; +- int ret = 0; ++ int ret = -1; + int splice_ret = 0; + proc_t *proc = NULL; + proc_t *p_proc = NULL; +@@ -963,28 +969,24 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + + pid = get_container_process_pid(workdir); + if (pid < 0) { +- ret = -1; + ERROR("%s: failed wait init pid", id); + goto out; + } + + file_read_int(shim_pid_file_name, &shim_pid); + if (shim_pid < 0) { +- ret = -1; + ERROR("%s: failed to read isulad shim pid", id); + goto out; + } + + proc = util_get_process_proc_info(pid); + if (proc == NULL) { +- ret = -1; + ERROR("%s: failed to read pidinfo", id); + goto out; + } + + p_proc = util_get_process_proc_info(shim_pid); + if (p_proc == NULL) { +- ret = -1; + ERROR("%s: failed to read isulad shim pidinfo", id); + goto out; + } +@@ -996,10 +998,10 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + + if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); +- ret = -1; + goto out; + } + ++ ret = 0; + out: + if (ret != 0) { + show_shim_runtime_errlog(workdir); +-- +2.25.1 + diff --git a/0032-add-CRI-container-design-doc.patch b/0032-add-CRI-container-design-doc.patch new file mode 100644 index 0000000..cc929c9 --- /dev/null +++ b/0032-add-CRI-container-design-doc.patch @@ -0,0 +1,850 @@ +From 5f75bec5b213eb85eda117ac912ddf4f94ff3277 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Mon, 20 Mar 2023 11:59:21 +0800 +Subject: [PATCH 32/46] add CRI container design doc + +Signed-off-by: zhangxiaoyu +--- + docs/cri_container_manager_design.md | 634 ++++++++++++++++++ + .../CRI/Attach_Flow_Chart.excalidraw.svg | 5 + + .../ContainerStatus_Flow_Chart.excalidraw.svg | 5 + + .../CreateContainer_Flow_Chart.excalidraw.svg | 5 + + .../CRI/ExecSync_Flow_Chart.excalidraw.svg | 5 + + .../design/CRI/Exec_Flow_Chart.excalidraw.svg | 5 + + .../ListContainers_Flow_Chart.excalidraw.svg | 5 + + .../RemoveContainer_Flow_Chart.excalidraw.svg | 5 + + .../StartContainer_Flow_Chart.excalidraw.svg | 5 + + .../StopContainer_Flow_Chart.excalidraw.svg | 5 + + ...ntainerResources_Flow_Chart.excalidraw.svg | 5 + + .../CRI/Websocket_Flow_Chart.excalidraw.svg | 5 + + ...Websocket_Server_Flow_Chart.excalidraw.svg | 5 + + ...Websocket_Stream_Flow_Chart.excalidraw.svg | 5 + + docs/design/CRI/Websocket_UML.excalidraw.svg | 5 + + 15 files changed, 704 insertions(+) + create mode 100644 docs/cri_container_manager_design.md + create mode 100644 docs/design/CRI/Attach_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/Exec_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg + create mode 100644 docs/design/CRI/Websocket_UML.excalidraw.svg + +diff --git a/docs/cri_container_manager_design.md b/docs/cri_container_manager_design.md +new file mode 100644 +index 00000000..5e8758b0 +--- /dev/null ++++ b/docs/cri_container_manager_design.md +@@ -0,0 +1,634 @@ ++| Author | 张晓雨 | ++| ------ | -----------------------------------------------------| ++| Date | 2023-03-20 | ++| Email | [zhangxiaoyu58@huawei.com](zhangxiaoyu58@huawei.com) | ++ ++本文主要对 `iSulad` 的 `CRI` 实现部分进行源码级别的学习和研究，为后续想了解 `CRI` 或者 `iSulad` 的同学提供帮助。 ++本文涉及`CRI`模块`Container`相关的接口。关于`Pod`相关接口的梳理请参考[cri_pod_manager_design.md](./cri_pod_manager_design.md)。 ++ ++*注：本文以iSulad 2.0.18版本为分析对象。* ++ ++本文主要是针对 `CRI` 的 `RuntimeService` 部分，涉及如下接口： ++- `CreateContainer` ++- `StartContainer` ++- `StopContainer` ++- `RemoveContainer` ++- `ListContainers` ++- `ContainerStatus` ++- `UpdateContainerResources` ++- `ExecSync` ++- `Exec` ++- `Attach` ++ ++## CreateContainer ++#### interface ++``` ++// CreateContainer creates a new container in specified PodSandbox ++rpc CreateContainer(CreateContainerRequest) returns (CreateContainerResponse) {} ++``` ++ ++#### Flow chart ++CreateContainer_Flow_Chart ++ ++#### Detail ++- GetRealContainerOrSandboxID ++get container from *g_containers_store* by (short) id, and filter container with label *cri.isulad.type=podSandbox* ++- GetContainerOrSandboxRuntime ++get podSandbox runtime ++- GenerateCreateContainerRequest ++set *container_create_request*, include ++ ```c ++ // name ++ std::string cname = CRINaming::MakeContainerName(podSandboxConfig, containerConfig) ++ // runtime ++ request->runtime = util_strdup_s(podSandboxRuntime.c_str()) ++ // image ++ request->image = util_strdup_s(containerConfig.image().image().c_str()) ++ // cgroup parent ++ hostconfig->cgroup_parent = util_strdup_s(podSandboxConfig.linux().cgroup_parent().c_str()) ++ // labels ++ custom_config->labels = CRIHelpers::MakeLabels(containerConfig.labels(), error) ++ // cri.isulad.type=container ++ append_json_map_string_string(custom_config->labels, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY.c_str(), CRIHelpers::Constants::CONTAINER_TYPE_LABEL_CONTAINER.c_str() ++ // log path ++ append_json_map_string_string(custom_config->labels, CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str(), real_logpath) ++ // annotations ++ CRIHelpers::MakeAnnotations(containerConfig.annotations(), error) ++ // mount ++ CRIHelpers::GenerateMountBindings(containerConfig.mounts(), hostconfig, error) ++ // device ++ PackCreateContainerHostConfigDevices(containerConfig, hostconfig, error) ++ // security context ++ PackCreateContainerHostConfigSecurityContext(containerConfig, hostconfig, error) ++ ... ++ ``` ++ ++## StartContainer ++#### interface ++``` ++// StartContainer starts the container. ++rpc StartContainer(StartContainerRequest) returns (StartContainerResponse) {} ++``` ++ ++#### Flow chart ++StartContainer_Flow_Chart ++ ++#### Detail ++- GetRealContainerOrSandboxID ++get container from *g_containers_store* by (short) id, and filter container with label *cri.isulad.type=container* ++- CreateContainerLogSymlink ++*inspect_container* , get *cri.container.logpath* annotation and container *log_path*, then create a symbolic from *log_path* to *cri.container.logpath* ++ ++## StopContainer ++#### interface ++``` ++// StopContainer stops a running container with a grace period (i.e., timeout). ++// This call is idempotent, and must not return an error if the container has ++// already been stopped. ++// The runtime must forcibly kill the container after the grace period is ++// reached. ++rpc StopContainer(StopContainerRequest) returns (StopContainerResponse) {} ++``` ++ ++#### Flow chart ++StopContainer_Flow_Chart ++ ++#### Detail ++- Timeout ++*timeout* in seconds to wait for the container to stop before forcibly terminating it, default value is 0 ++ ++## RemoveContainer ++#### interface ++``` ++// RemoveContainer removes the container. If the container is running, the ++// container must be forcibly removed. ++// This call is idempotent, and must not return an error if the container has ++// already been removed. ++rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse) {} ++``` ++ ++#### Flow chart ++RemoveContainer_Flow_Chart ++ ++#### Detail ++- RemoveContainerLogSymlink ++*inspect_container* , get *cri.container.logpath* annotation and container *log_path*, then delete *cri.container.logpath* ++ ++## ListContainers ++#### interface ++``` ++// ListContainers lists all containers by filters. ++rpc ListContainers(ListContainersRequest) returns (ListContainersResponse) {} ++``` ++ ++#### Flow chart ++ListContainers_Flow_Chart ++ ++#### Detail ++- ListContainersFromGRPC ++convert filter from GRPC request, and add *cri.isulad.type=container* to filter ++ ```c ++ // Add filter to get only non-sandbox containers ++ CRIHelpers::FiltersAddLabel((*request)->filters, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY, CRIHelpers::Constants::CONTAINER_TYPE_LABEL_CONTAINER) ++ // convert filter ++ // id ++ CRIHelpers::FiltersAdd((*request)->filters, "id", filter->id()) ++ // status ++ CRIHelpers::FiltersAdd((*request)->filters, "status", CRIHelpers::ToIsuladContainerStatus(filter->state())) ++ // sandbox id ++ CRIHelpers::FiltersAddLabel((*request)->filters, CRIHelpers::Constants::SANDBOX_ID_LABEL_KEY, filter->pod_sandbox_id()) ++ // label selector ++ for (auto &iter : filter->label_selector()) { ++ CRIHelpers::FiltersAddLabel((*request)->filters, iter.first, iter.second) ++ } ++ ``` ++- ListContainersToGRPC ++convert containers information to GRPC response, include ++ ```c ++ for (size_t i {}; i < response->containers_len; i++) { ++ // id ++ container->set_id(response->containers[i]->id) ++ // labels ++ CRIHelpers::ExtractLabels(response->containers[i]->labels, *container->mutable_labels()) ++ // annotations ++ CRIHelpers::ExtractAnnotations(response->containers[i]->annotations, *container->mutable_annotations()) ++ // name ++ CRINaming::ParseContainerName(container->annotations(), container->mutable_metadata(), error) ++ // image ++ image->set_image(response->containers[i]->image) ++ std::string imageID = CRIHelpers::ToPullableImageID(response->containers[i]->image, response->containers[i]->image_ref) ++ container->set_image_ref(imageID) ++ // runtime ++ CRIHelpers::ContainerStatusToRuntime(Container_Status(response->containers[i]->status)) ++ ... ++ } ++ ``` ++ ++## ContainerStatus ++#### interface ++``` ++// ContainerStatus returns status of the container. If the container is not ++// present, returns an error. ++rpc ContainerStatus(ContainerStatusRequest) returns (ContainerStatusResponse) {} ++``` ++ ++#### Flow chart ++ContainerStatus_Flow_Chart ++ ++#### Detail ++- ContainerStatusToGRPC ++add container information to GRPC response, include ++ ```c ++ // timestamps ++ CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error) ++ // image ++ PackContainerImageToStatus(inspect, contStatus, error) ++ // container status ++ UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus) ++ // labels ++ PackLabelsToStatus(inspect, contStatus) ++ // mount ++ ConvertMountsToStatus(inspect, contStatus); ++ ``` ++ ++## UpdateContainerResources ++#### interface ++``` ++// UpdateContainerResources updates ContainerConfig of the container synchronously. ++// If runtime fails to transactionally update the requested resources, an error is returned. ++rpc UpdateContainerResources(UpdateContainerResourcesRequest) returns (UpdateContainerResourcesResponse) {} ++``` ++ ++#### Flow chart ++UpdateContainerResources_Flow_Chart ++ ++#### Detail ++- Generate HostConfig ++set resources, include ++ ```c ++ // cpu ++ hostconfig->cpu_period = resources.cpu_period() ++ hostconfig->cpu_quota = resources.cpu_quota() ++ hostconfig->cpu_shares = resources.cpu_shares() ++ hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str()) ++ hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()) ++ // memory ++ hostconfig->memory_swap_limit_in_bytes = resources.memory_swap_limit_in_bytes() ++ hostconfig->memory = resources.memory_limit_in_bytes() ++ // cgroupv2 unified ++ for (auto &iter : resources.unified()) { ++ append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) ++ } ++ // huge page ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()) ++ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit() ++ hostconfig->hugetlbs_len++ ++ } ++ ``` ++ ++## ExecSync ++#### interface ++``` ++// ExecSync runs a command in a container synchronously. ++rpc ExecSync(ExecSyncRequest) returns (ExecSyncResponse) {} ++``` ++ ++#### Flow chart ++ExecSync_Flow_Chart ++ ++ ++#### Detail ++- ExecSyncFromGRPC ++set *container_exec_request*, include ++ ```c ++ (*request)->tty = false ++ (*request)->attach_stdin = false ++ (*request)->attach_stdout = true ++ (*request)->attach_stderr = true ++ (*request)->timeout = timeout ++ (*request)->container_id = util_strdup_s(containerID.c_str()) ++ // execution command ++ for (int i = 0; i < cmd.size(); i++) { ++ (*request)->argv[i] = util_strdup_s(cmd[i].c_str()) ++ (*request)->argv_len++ ++ } ++ // generate random suffix ++ (*request)->suffix = CRIHelpers::GenerateExecSuffix() ++ ``` ++- StringWriter ++set stdoutWriter and stderrWriter, write string to reply *stdout/stderr* ++ ```c ++ StdoutstringWriter.context = (void *)reply->mutable_stdout() ++ StdoutstringWriter.write_func = WriteToString ++ ++ StderrstringWriter.context = (void *)reply->mutable_stderr() ++ StderrstringWriter.write_func = WriteToString ++ ++ static auto WriteToString(void *context, const void *data, size_t len) -> ssize_t ++ { ++ if (len == 0) { ++ return 0; ++ } ++ ++ std::string *str = reinterpret_cast(context); ++ ++ str->append(reinterpret_cast(data), len); ++ return (ssize_t)len; ++ } ++ ``` ++ ++## Exec ++#### interface ++``` ++// Exec prepares a streaming endpoint to execute a command in the container. ++rpc Exec(ExecRequest) returns (ExecResponse) {} ++``` ++ ++#### Flow chart ++Exec_Flow_Chart ++ ++#### Detail ++- ValidateExecRequest ++1. check container is running ++2. check container is not paused ++3. check tty and stderr not both true ++4. check one of stdin, stdout, stderr must be true ++- GarbageCollection ++traverse from back of the *CacheEntry list* and remove the *expired requests*, also delete them in *token hash map* ++ ```c ++ void RequestCache::GarbageCollection() ++ { ++ auto now = std::chrono::system_clock::now(); ++ while (!m_ll.empty()) { ++ auto oldest = m_ll.back(); ++ if (now < oldest.expireTime) { ++ return; ++ } ++ if (oldest.req != nullptr) { ++ delete oldest.req; ++ oldest.req = nullptr; ++ } ++ m_ll.pop_back(); ++ m_tokens.erase(oldest.token); ++ } ++ } ++ ``` ++- UniqueToken ++`token=base64(random string))`. If token is exist in *token hash map*, retry generate token ++- Save ++save containerID, request, token in *CacheEntry list* and *token hash map* ++ ++## Attach ++#### interface ++``` ++// Attach prepares a streaming endpoint to attach to a running container. ++rpc Attach(AttachRequest) returns (AttachResponse) {} ++``` ++ ++#### Flow chart ++Attach_Flow_Chart ++ ++#### Detail ++- ValidateAttachRequest ++1. check tty and stderr not both true ++2. check one of stdin, stdout, stderr must be true ++ ++## Websocket ++ ++#### UML diagram ++Websocket_UML ++ ++### Start/Stop Websocket Server ++#### Flow chart ++Websocket_Server_Flow_Chart ++ ++#### Detail ++- CreateContext ++create websocket server context ++ ```c ++ /* ++ context->lws_lookup is allocated ( sizeof(struct lws *) * max_fds ) spaces, ++ In general, max_fds should be the process maximum number of open file descriptor. ++ If WS_ULIMIT_FDS set too large, context->lws_lookup will cost too much memory. ++ If WS_ULIMIT_FDS set too small, maybe fd > max_fds and context->lws_lookup[fd] will overflow. ++ */ ++ const size_t WS_ULIMIT_FDS { 10240 } ++ ++ info.port = m_listenPort ++ info.iface = "127.0.0.1" ++ info.protocols = m_protocols ++ info.ssl_cert_filepath = nullptr ++ info.ssl_private_key_filepath = nullptr ++ info.gid = -1 ++ info.uid = -1 ++ info.options = LWS_SERVER_OPTION_VALIDATE_UTF8 | LWS_SERVER_OPTION_DISABLE_IPV6 ++ info.max_http_header_pool = MAX_HTTP_HEADER_POOL ++ info.extensions = nullptr ++ ++ m_context = lws_create_context(&info) ++ ``` ++ ++### Websocket Main Callback ++#### Flow chart ++Websocket_Flow_Chart ++ ++#### Detail ++- LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION ++called when the handshake has been received and parsed from the client, but the response is not sent yet. Return non-zero to disallow the connection. ++ ```c ++ // Write Lock ++ WriteGuard lock(m_mutex) ++ ++ // get TOKEN URL ++ lws_hdr_copy(wsi, buf, sizeof(buf), WSI_TOKEN_GET_URI) ++ ++ // Validate URL ++ // URL format: "/cri/" + method + "/" + token + "/" + arg(container=cmd) ++ auto vec = CXXUtils::Split(buf + 1, '/'); ++ // method: exec and attach ++ // token is exist in token hash map ++ if vec.size() >= 3 && m_handler.IsValidMethod(vec.at(1)) && cache->IsValidToken(vec.at(2)) ++ ++ // socketID already exist ? ++ int socketID = lws_get_socket_fd(wsi) ++ if m_wsis.count(socketID) == 0 ++ ++ // too many connection sessions ? ++ if m_wsis.size() <= MAX_SESSION_NUM(128) ++ ++ // generate SessionData ++ GenerateSessionData(session, containerID) ++ ++ // save session ++ m_wsis.insert(std::make_pair(socketID, session)) ++ ++ // create StreamTask ++ std::thread streamTh([ = ]() { ++ StreamTask(&m_handler, session, vec.at(1), vec.at(2)).Run(); ++ }) ++ streamTh.detach() ++ ++ // Trigger polling in LWS_CALLBACK_SERVER_WRITEABLE ++ lws_callback_on_writable(wsi) ++ ``` ++- LWS_CALLBACK_SERVER_WRITEABLE ++If you call lws_callback_on_writable() on a connection, you will get one of these callbacks coming when the connection socket is able to accept another write packet without blocking. ++If it already was able to take another packet without blocking, you'll get this callback at the next call to the service loop function. ++ ```c ++ // Read Lock ++ ReadGuard lock(m_mutex) ++ ++ // get SessionData ++ int socketID = lws_get_socket_fd(wsi) ++ auto it = m_wsis.find(socketID) ++ if it != m_wsis.end() ++ ++ // record session closed ++ auto sessionClosed = it->second->IsClosed() ++ ++ // write message ++ while (!it->second->buffer.empty()) { ++ auto *message = it->second->FrontMessage() ++ // send success! free it and erase for list ++ WebsocketServer::GetInstance()->Wswrite(wsi, const_cast(message)) ++ free(message) ++ it->second->PopMessage() ++ } ++ ++ // return non-zero if session closed ++ if (sessionClosed) { ++ DEBUG("websocket session disconnected") ++ return -1 ++ } ++ ++ // return zero and callback with the reason of LWS_CALLBACK_SERVER_WRITEABLE again ++ lws_callback_on_writable(wsi) ++ ``` ++- LWS_CALLBACK_RECEIVE ++data has appeared for this server endpoint from a remote client, it can be found at in and is len bytes long ++ ```c ++ // Read Lock ++ ReadGuard lock(m_mutex) ++ ++ // get SessionData ++ auto it = m_wsis.find(socketID) ++ if it != m_wsis.end() ++ ++ // Last Stdin Uncomplete ++ if (!it->second->IsStdinComplete()) { ++ util_write_nointr_in_total(m_wsis[socketID]->pipes.at(1), (char *)in, len) ++ } ++ ++ // RESIZECHANNEL ++ if (*static_cast(in) == WebsocketChannel::RESIZECHANNEL) { ++ ParseTerminalSize(jsonData, len, width, height) ++ int ret = cb->container.resize(req, &res) ++ } ++ ++ // STDINCHANNEL ++ if (*static_cast(in) == WebsocketChannel::STDINCHANNEL) { ++ util_write_nointr_in_total(m_wsis[socketID]->pipes.at(1), (char *)in + 1, len - 1) ++ } ++ ++ // SetStdinComplete ++ size_t bytesLen = lws_remaining_packet_payload(wsi) ++ it->second->SetStdinComplete(bytesLen == 0) ++ ``` ++- LWS_CALLBACK_CLOSED ++when the websocket session ends ++ ```c ++ void WebsocketServer::CloseWsSession(int socketID) ++ { ++ // write lock ++ m_mutex.wrlock(); ++ auto it = m_wsis.find(socketID); ++ if (it == m_wsis.end()) { ++ m_mutex.unlock(); ++ return; ++ } ++ ++ auto session = it->second; ++ it->second = nullptr; ++ // delete session in map ++ m_wsis.erase(it); ++ m_mutex.unlock(); ++ ++ std::thread([session]() { ++ prctl(PR_SET_NAME, "WSSessionGC"); ++ session->CloseSession(); ++ session->EraseAllMessage(); ++ // close the pipe write endpoint first, make sure io copy thread exit, ++ // otherwise epoll will trigger EOF ++ if (session->pipes.at(1) >= 0) { ++ close(session->pipes.at(1)); ++ session->pipes.at(1) = -1; ++ } ++ (void)sem_wait(session->syncCloseSem); ++ (void)sem_destroy(session->syncCloseSem); ++ delete session->syncCloseSem; ++ session->syncCloseSem = nullptr; ++ close(session->pipes.at(0)); ++ delete session->sessionMutex; ++ session->sessionMutex = nullptr; ++ delete session; ++ }).detach(); ++ } ++ ``` ++ ++### Websocket Stream ++#### Flow chart ++Websocket_Stream_Flow_Chart ++ ++#### Detail ++- ConsumeRequest ++ ```c ++ // Consume the token (remove it from the cache) and return the cached request, if found. ++ ::google::protobuf::Message *RequestCache::ConsumeRequest(const std::string &token) ++ { ++ std::lock_guard lock(m_mutex); ++ ++ if (m_tokens.count(token) == 0) { ++ ERROR("Invalid token"); ++ return nullptr; ++ } ++ ++ CacheEntry ele = m_tokens[token]; ++ for (auto it = m_ll.begin(); it != m_ll.end(); it++) { ++ if (it->token == token) { ++ m_ll.erase(it); ++ break; ++ } ++ } ++ m_tokens.erase(token); ++ if (std::chrono::system_clock::now() > ele.expireTime) { ++ return nullptr; ++ } ++ ++ return ele.req; ++ } ++ ``` ++- AttachServe::SetContainerStreamRequest ++set stdin, stdout and stderr ++ ```c ++ m_request->attach_stdin = grequest->stdin() ++ m_request->attach_stdout = grequest->stdout() ++ m_request->attach_stderr = grequest->stderr() ++ ``` ++- AttachServe::ExecuteStreamCommand ++ ```c ++ // StringWriter ++ // write stdout to client if attach stdout is true ++ stdoutContext.attachWriter = m_request->attach_stdout ? WsWriteStdoutToClient : WsDoNotWriteStdoutToClient ++ // sem_post attachSem ++ stdoutstringWriter.close_func = AttachConnectClosed ++ // write stderr to client if attach stderr is true ++ stderrContext.attachWriter = m_request->attach_stderr ? WsWriteStderrToClient : WsDoNotWriteStderrToClient ++ stderrstringWriter.close_func = nullptr ++ ++ // call attach ++ // nonblocking ++ int ret = cb->container.attach(m_request, &m_response, m_request->attach_stdin ? lwsCtx->pipes.at(0) : -1, &stdoutstringWriter, &stderrstringWriter) ++ ++ // handle attach return value ++ if (ret != 0) { ++ // join io copy thread in attach callback ++ ERROR("Failed to attach container: %s", m_request->container_id); ++ ++ std::string message; ++ if (m_response != nullptr && m_response->errmsg != nullptr) { ++ message = m_response->errmsg; ++ } else { ++ message = "Failed to call attach container callback. "; ++ } ++ WsWriteStdoutToClient(lwsCtx, message.c_str(), message.length()); ++ } else { ++ // wait io copy thread complete ++ (void)sem_wait(&attachSem); ++ } ++ ``` ++- ExecServe::SetContainerStreamRequest ++set tty, stdin, stdout, stderr, container_id, args and suffix ++ ```c ++ m_request->tty = grequest->tty() ++ m_request->attach_stdin = grequest->stdin() ++ m_request->attach_stdout = grequest->stdout() ++ m_request->attach_stderr = grequest->stderr() ++ m_request->container_id = util_strdup_s(grequest->container_id().c_str()) ++ m_request->suffix = util_strdup_s(suffix.c_str()) ++ ++ // args ++ if (grequest->cmd_size() > 0) { ++ m_request->argv = (char **)util_smart_calloc_s(sizeof(char *), grequest->cmd_size()); ++ for (int i = 0; i < grequest->cmd_size(); i++) { ++ m_request->argv[i] = util_strdup_s(grequest->cmd(i).c_str()); ++ } ++ m_request->argv_len = static_cast(grequest->cmd_size()); ++ } ++ ``` ++- ExecServe::ExecuteStreamCommand ++ ```c ++ // StringWriter ++ StdoutstringWriter.write_func = WsWriteStdoutToClient ++ StdoutstringWriter.close_func = nullptr ++ StderrstringWriter.write_func = WsWriteStderrToClient ++ StderrstringWriter.close_func = nullptr ++ ++ // call exec ++ int ret = cb->container.exec(m_request, &m_response, m_request->attach_stdin ? lwsCtx->pipes.at(0) : -1, m_request->attach_stdout ? &StdoutstringWriter : nullptr, m_request->attach_stderr ? &StderrstringWriter : nullptr) ++ ++ // handle exec return value ++ if (ret != 0) { ++ std::string message; ++ if (m_response != nullptr && m_response->errmsg != nullptr) { ++ message = m_response->errmsg; ++ } else { ++ message = "Failed to call exec container callback. "; ++ } ++ WsWriteStdoutToClient(lwsCtx, message.c_str(), message.length()); ++ } ++ if (m_response != nullptr && m_response->exit_code != 0) { ++ std::string exit_info = "Exit code :" + std::to_string((int)m_response->exit_code) + "\n"; ++ WsWriteStdoutToClient(lwsCtx, exit_info.c_str(), exit_info.length()); ++ } ++ ``` +diff --git a/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg b/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..eaf33c61 +--- /dev/null ++++ b/docs/design/CRI/Attach_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ AttachGetRealContainerOrSandboxIDlabel: cri.isulad.type=containergenerate urlreturn urlget container ID ?Ygenerate urlGarbageCollectionUniqueTokenlock(m_mutex)saveURLws://localhost:10350/cri/attach/<token>CacheEntry list overflow(1000) ?Nunlock(m_mutex)bugcheck container running?ValidateAttachRequestsuccess +\ No newline at end of file +diff --git a/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg b/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..c8daa2de +--- /dev/null ++++ b/docs/design/CRI/ContainerStatus_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ ContainerStatusGetRealContainerOrSandboxIDlabel: cri.isulad.type=containerContainerStatusToGRPCreturnget container ID ?Yget inspect data?YInspectContainer +\ No newline at end of file +diff --git a/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..1fc622e9 +--- /dev/null ++++ b/docs/design/CRI/CreateContainer_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ CreateContainerGetRealContainerOrSandboxIDGetContainerOrSandboxRuntimelabel: cri.isulad.type=podsandboxGenerateCreateContainerRequestm_cb->container.createreturn response idget pod ID ?Yreturn error if not get runtimeORuse new variable to ignore error +\ No newline at end of file +diff --git a/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg b/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..b3462c59 +--- /dev/null ++++ b/docs/design/CRI/ExecSync_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ ExecSyncGetRealContainerOrSandboxIDExecSyncFromGRPClabel: cri.isulad.type=containerm_cb->container.execset exit codeget container ID ?YStdoutstringWriterStderrstringWriter +\ No newline at end of file +diff --git a/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg b/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..463a9cce +--- /dev/null ++++ b/docs/design/CRI/Exec_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ ExecGetRealContainerOrSandboxIDlabel: cri.isulad.type=containerget container ID ?Yreturn urlgenerate urlgenerate urlGarbageCollectionUniqueTokenlock(m_mutex)saveURLws://localhost:10350/cri/exec/<token>CacheEntry list overflow(1000) ?Nunlock(m_mutex)get container state ?YInspectContainerStateValidateExecRequest ?success +\ No newline at end of file +diff --git a/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg b/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..9955aefa +--- /dev/null ++++ b/docs/design/CRI/ListContainers_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ ListContainersListContainersFromGRPCListContainersToGRPCm_cb->container.list +\ No newline at end of file +diff --git a/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..0ade7967 +--- /dev/null ++++ b/docs/design/CRI/RemoveContainer_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ RemoveContainerGetRealContainerOrSandboxIDRemoveContainerLogSymlinklabel: cri.isulad.type=containercb->container.removeget container ID ?Yforce remove +\ No newline at end of file +diff --git a/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..e84bd261 +--- /dev/null ++++ b/docs/design/CRI/StartContainer_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ StartContainerGetRealContainerOrSandboxIDm_cb->container.startlabel: cri.isulad.type=containerCreateContainerLogSymlinkget container ID ?Y +\ No newline at end of file +diff --git a/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg b/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..ab619ecc +--- /dev/null ++++ b/docs/design/CRI/StopContainer_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ StopContainerGetRealContainerOrSandboxIDset id and timeoutlabel: cri.isulad.type=containerm_cb->container.stopget container ID ?Y +\ No newline at end of file +diff --git a/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg b/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..78ca3360 +--- /dev/null ++++ b/docs/design/CRI/UpdateContainerResources_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ UpdateContainerResourcesGetRealContainerOrSandboxIDgenerate hostconfiglabel: cri.isulad.type=containerm_cb->container.updateget container ID ?Y +\ No newline at end of file +diff --git a/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..8e3e589a +--- /dev/null ++++ b/docs/design/CRI/Websocket_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ WebsocketServer::Callback(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len)switch (reason)read lock m_mutexIs Last StdinComplete ?write((char *)in)LWS_CALLBACK_RECEIVESetStdinCompleteswitch *(char *)in ?write((char *)in + 1)WebsocketChannel::STDINCHANNELParseTerminalSize((char *)in + 1)WebsocketChannel::RESIZECHANNELcb->container.resizereturn 0unlock m_mutexwrite lock m_mutexLWS_CALLBACK_CLOSEDdelete session in mapreturn 0unlock m_mutexcreate std::threadCloseSessionEraseAllMessageclose pipe[1]sem_wait(session->syncCloseSem)close pipe[0]deleteget SessionData by socket_fd ?(disable an http request)return -1get token URLLWS_CALLBACK_HTTPLWS_CALLBACK_FILTER_PROTOCOL_CONNECTIONwrite lock m_mutexvalidate URL ?too many connection sessions ?GenerateSessionDatasave sessionstd::thread StreamTasklws_callback_on_writableLWS_CALLBACK_ESTABLISHEDDebug messagereturn 0unlock m_mutexreturn 0read lock m_mutexbuffer.empty() ?message=FrontMessage()lws_write(message)PopMessageLWS_CALLBACK_SERVER_WRITEABLEget SessionData by socket_fd ?sessionClosed=IsClosed()sessionClosed ?lws_callback_on_writableunlock m_mutexreturn 0socketID already exist ?YNNNYYreturn nonzero from the user callback to close the connection and callback with the reason of LWS_CALLBACK_CLOSEDYNNYYNothersDebugInvalid dataYNNYget SessionData by socket_fd ?WSSessionGCunlock m_mutexreturn 0return 0othersYNthreadWebsocket Stream +\ No newline at end of file +diff --git a/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..ac784cc6 +--- /dev/null ++++ b/docs/design/CRI/Websocket_Server_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ WebsocketServer::Startconf_get_websocket_server_listening_portWebsocketServer::CreateContextWebsocketServerlws_servicem_forceExit ?start isuladcreate threadreturnWebsocketServer::Waitwrite lock(m_mutex)unlock(m_mutex)join threadfor each SessionDatain m_wsislws_context_destroymain threaddelete SessionDataWebsocketServer::Shutdownset m_forceExitlws_cancel_servicereceive signal SIGINT/SIGTERMmainloop callback +\ No newline at end of file +diff --git a/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg b/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg +new file mode 100644 +index 00000000..9a1c60b7 +--- /dev/null ++++ b/docs/design/CRI/Websocket_Stream_Flow_Chart.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ sem_post(lwsCtx->syncCloseSem)bugcloseWsConnect?AttachServeget request by token ?earseAttachServe::SetContainerStreamRequeststdoutstringWriterstderrstringWritercb->container.attachattach return 0 ?write error messagesem_wait(&attachSem)CloseSessionsem_post(lwsCtx->syncCloseSem)YNexpired request ?YConsumeRequestNYAttachServe::ExecuteStreamCommanderrorExecServeget request by token ?earseExecServe::SetContainerStreamRequeststdoutstringWriterstderrstringWritercb->container.execexec return 0 ?write error messagewrite exit codeCloseSessionsem_post(lwsCtx->syncCloseSem)YNexpired request ?ConsumeRequestNYExecServe::ExecuteStreamCommandY +\ No newline at end of file +diff --git a/docs/design/CRI/Websocket_UML.excalidraw.svg b/docs/design/CRI/Websocket_UML.excalidraw.svg +new file mode 100644 +index 00000000..d1248c67 +--- /dev/null ++++ b/docs/design/CRI/Websocket_UML.excalidraw.svg +@@ -0,0 +1,5 @@ ++ ++ ++ ++ ++ WebsocketServer+ Start+ Wait+ Shutdown+ ...StreamingServeInterfaceinterface+ Execute+ ...AttachServe+ SetContainerStreamRequest+ ExecuteStreamCommand+ CloseConnect+ ...ExecServe+ SetContainerStreamRequest+ ExecuteStreamCommand+ CloseConnect+ ...RouteCallbackRegister+ RegisterCallback+ HandleCallback+ ...useuseWebsocket ImplStreamTask+ Run+ ...useuseRuntimeManagerService+ UpdateRuntimeConfig+ Status+ ...CRI IMPLuse +\ No newline at end of file +-- +2.25.1 + diff --git a/0033-fix-util_getgrent_r-overflow.patch b/0033-fix-util_getgrent_r-overflow.patch new file mode 100644 index 0000000..619f367 --- /dev/null +++ b/0033-fix-util_getgrent_r-overflow.patch @@ -0,0 +1,74 @@ +From 2890d9c08a60e45fa76ae0a8e72e0a37deac6a89 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 21 Mar 2023 18:52:43 +0800 +Subject: [PATCH 33/46] fix util_getgrent_r overflow + +Signed-off-by: zhangxiaoyu +--- + src/utils/cutils/utils_pwgr.c | 3 +++ + test/cutils/utils_pwgr/long_sample | 1 + + test/cutils/utils_pwgr/utils_pwgr_ut.cc | 28 +++++++++++++++++++++++++ + 3 files changed, 32 insertions(+) + create mode 100644 test/cutils/utils_pwgr/long_sample + +diff --git a/src/utils/cutils/utils_pwgr.c b/src/utils/cutils/utils_pwgr.c +index 75efadb2..064bae1f 100644 +--- a/src/utils/cutils/utils_pwgr.c ++++ b/src/utils/cutils/utils_pwgr.c +@@ -157,6 +157,9 @@ static char **hold_string_list(char **line, char *buf_start, char *buf_end, cons + } + + out: ++ if ((char *)(walker + 2) > buf_end) { ++ return NULL; ++ } + *walker = NULL; + return result; + } +diff --git a/test/cutils/utils_pwgr/long_sample b/test/cutils/utils_pwgr/long_sample +new file mode 100644 +index 00000000..7e991ffd +--- /dev/null ++++ b/test/cutils/utils_pwgr/long_sample +@@ -0,0 +1 @@ ++adm:x:4:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +\ No newline at end of file +diff --git a/test/cutils/utils_pwgr/utils_pwgr_ut.cc b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +index 431372d3..20de8a89 100644 +--- a/test/cutils/utils_pwgr/utils_pwgr_ut.cc ++++ b/test/cutils/utils_pwgr/utils_pwgr_ut.cc +@@ -140,3 +140,31 @@ TEST(utils_pwgr, test_getgrent_r) + + fclose(f_gr); + } ++ ++TEST(utils_pwgr, test_long_getgrent_r) ++{ ++ std::string path = "../../../../test/cutils/utils_pwgr/long_sample"; ++ FILE *f_gr = fopen(path.c_str(), "r"); ++ ASSERT_NE(f_gr, nullptr); ++ ++ struct group gr { ++ 0 ++ }; ++ struct group *pgr = nullptr; ++ char mark_before[BUFSIZ] = { 0 }; ++ char buf[BUFSIZ] = { 0 }; ++ char mark_after[BUFSIZ] = { 0 }; ++ ++ for (int num = 0; num < BUFSIZ; num++) { ++ mark_before[num] = 0xEE; ++ mark_after[num] = 0xDD; ++ } ++ ++ (void)util_getgrent_r(f_gr, &gr, buf, sizeof(buf), &pgr); ++ for (int num = 0; num < BUFSIZ; num++) { ++ ASSERT_EQ(mark_before[num], (char)0xEE); ++ ASSERT_EQ(mark_after[num], (char)0xDD); ++ } ++ ++ fclose(f_gr); ++} +-- +2.25.1 + diff --git a/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch b/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch new file mode 100644 index 0000000..c58e315 --- /dev/null +++ b/0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch @@ -0,0 +1,44 @@ +From be989b6beff994f1b06e885ef766f01a6a3c0969 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 20 Mar 2023 14:17:00 +0800 +Subject: [PATCH 34/46] modify the return value of the + util_waitpid_with_timeout to status + +Signed-off-by: zhongtao +--- + src/utils/cutils/utils.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 7f36d019..3cede76a 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -321,6 +321,7 @@ rep: + int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb) + { + int nret = 0; ++ int st; + time_t start_time = time(NULL); + time_t end_time; + double interval; +@@ -330,7 +331,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + } + + for (;;) { +- nret = waitpid(pid, NULL, WNOHANG); ++ nret = waitpid(pid, &st, WNOHANG); + if (nret == pid) { + break; + } +@@ -349,7 +350,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c + // sleep some time instead to avoid cpu full running and then retry. + usleep(100); + } +- return 0; ++ return st; + } + + int util_wait_for_pid_status(pid_t pid) +-- +2.25.1 + diff --git a/0035-fix-inspect-data-memleak.patch b/0035-fix-inspect-data-memleak.patch new file mode 100644 index 0000000..587a9a3 --- /dev/null +++ b/0035-fix-inspect-data-memleak.patch @@ -0,0 +1,39 @@ +From d4f524ef82ffdf2aed4847fa25795894a7bb7eda Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Tue, 28 Mar 2023 10:14:22 +0800 +Subject: [PATCH 35/46] fix inspect data memleak + +Signed-off-by: zhangxiaoyu +--- + src/daemon/entry/cri/cni_network_plugin.cc | 2 +- + src/daemon/entry/cri/cri_container_manager_service_impl.cc | 1 + + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc +index 02e75ffe..8cad0126 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.cc ++++ b/src/daemon/entry/cri/cni_network_plugin.cc +@@ -478,7 +478,7 @@ auto CniNetworkPlugin::GetNetNS(const std::string &podSandboxID, Errors &err) -> + + container_inspect *inspect_data = CRIHelpers::InspectContainer(podSandboxID, err, false); + if (inspect_data == nullptr) { +- goto cleanup; ++ return result; + } + if (inspect_data->state->pid == 0) { + err.Errorf("cannot find network namespace for the terminated container %s", podSandboxID.c_str()); +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +index 0fc1884f..d406496e 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +@@ -1100,6 +1100,7 @@ ContainerManagerServiceImpl::ContainerStatus(const std::string &containerID, Err + ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1alpha2::ContainerStatus); + if (contStatus == nullptr) { + error.SetError("Out of memory"); ++ free_container_inspect(inspect); + return nullptr; + } + +-- +2.25.1 + diff --git a/0036-containers-in-same-sandbox-should-have-same-process-.patch b/0036-containers-in-same-sandbox-should-have-same-process-.patch new file mode 100644 index 0000000..8b24921 --- /dev/null +++ b/0036-containers-in-same-sandbox-should-have-same-process-.patch @@ -0,0 +1,193 @@ +From 12b64931a90594e837e1f3dc9a0dde5292bb73b5 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sat, 8 Apr 2023 15:42:30 +0800 +Subject: [PATCH 36/46] containers in same sandbox should have same process + labels + +Signed-off-by: zhongtao +--- + .../cri/cri_container_manager_service_impl.cc | 71 ++++++++++++++++++- + .../cri/cri_container_manager_service_impl.h | 5 +- + src/daemon/entry/cri/cri_helpers.cc | 31 ++++++++ + src/daemon/entry/cri/cri_helpers.h | 3 + + 4 files changed, 107 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +index d406496e..a64d222c 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +@@ -126,8 +126,66 @@ auto ContainerManagerServiceImpl::PackCreateContainerHostConfigSecurityContext( + return 0; + } + ++auto ContainerManagerServiceImpl::DoUsePodLevelSELinuxConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, ++ host_config *hostconfig, const std::string &realPodSandboxID, Errors &error) -> int ++{ ++ int ret = -1; ++ size_t newSize = 0; ++ size_t oldSize = 0; ++ container_inspect *inspect = nullptr; ++ std::vector selinuxLabelOpts; ++ char **tmp_security_opt = nullptr; ++ std::string tmp_str; ++ ++ inspect = CRIHelpers::InspectContainer(realPodSandboxID, error, true); ++ if (error.NotEmpty()) { ++ return -1; ++ } ++ ++ if (inspect->process_label == nullptr) { ++ ret = 0; ++ goto cleanup; ++ } ++ ++ tmp_str = std::string(inspect->process_label); ++ selinuxLabelOpts = CRIHelpers::GetSELinuxLabelOpts(tmp_str, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to get SELinuxLabelOpts for container %s", containerConfig.metadata().name().c_str()); ++ goto cleanup; ++ } ++ if (selinuxLabelOpts.empty()) { ++ error.Errorf("SElinuxLabelOpts for container %s is empty", containerConfig.metadata().name().c_str()); ++ goto cleanup; ++ } ++ if (selinuxLabelOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ goto cleanup; ++ } ++ newSize = (hostconfig->security_opt_len + selinuxLabelOpts.size()) * sizeof(char *); ++ oldSize = hostconfig->security_opt_len * sizeof(char *); ++ ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); ++ if (ret != 0) { ++ ERROR("Out of memory"); ++ error.Errorf("Out of memory"); ++ goto cleanup; ++ } ++ hostconfig->security_opt = tmp_security_opt; ++ for (const auto &securityOpt : selinuxLabelOpts) { ++ hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); ++ hostconfig->security_opt_len++; ++ } ++ ++cleanup: ++ free_container_inspect(inspect); ++ return ret; ++} ++ ++ ++ + auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( +- const runtime::v1alpha2::ContainerConfig &containerConfig, Errors &error) -> host_config * ++ const runtime::v1alpha2::ContainerConfig &containerConfig, ++ const std::string &realPodSandboxID, Errors &error) -> host_config * + { + host_config *hostconfig = (host_config *)util_common_calloc_s(sizeof(host_config)); + if (hostconfig == nullptr) { +@@ -159,6 +217,15 @@ auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( + goto cleanup; + } + ++ // If selinux label is not specified in container config, use pod level SELinux config ++ if (!containerConfig.linux().has_security_context() || ++ !containerConfig.linux().security_context().has_selinux_options()) { ++ if (DoUsePodLevelSELinuxConfig(containerConfig, hostconfig, realPodSandboxID, error) != 0) { ++ error.SetError("Failed to security context to host config"); ++ goto cleanup; ++ } ++ } ++ + return hostconfig; + + cleanup: +@@ -367,7 +434,7 @@ ContainerManagerServiceImpl::GenerateCreateContainerRequest(const std::string &r + + container_config *custom_config { nullptr }; + +- host_config *hostconfig = GenerateCreateContainerHostConfig(containerConfig, error); ++ host_config *hostconfig = GenerateCreateContainerHostConfig(containerConfig, realPodSandboxID, error); + if (error.NotEmpty()) { + goto cleanup; + } +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.h b/src/daemon/entry/cri/cri_container_manager_service_impl.h +index 6ac1cbc4..dcb2cd44 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.h ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.h +@@ -83,7 +83,7 @@ private: + const std::string &podSandboxRuntime, + Errors &error) -> container_create_request *; + auto GenerateCreateContainerHostConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, +- Errors &error) -> host_config *; ++ const std::string &realPodSandboxID, Errors &error) -> host_config *; + auto GenerateCreateContainerCustomConfig(const std::string &containerName, const std::string &realPodSandboxID, + const runtime::v1alpha2::ContainerConfig &containerConfig, + const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, +@@ -92,6 +92,9 @@ private: + host_config *hostconfig, Errors &error) -> int; + auto PackCreateContainerHostConfigSecurityContext(const runtime::v1alpha2::ContainerConfig &containerConfig, + host_config *hostconfig, Errors &error) -> int; ++ auto DoUsePodLevelSELinuxConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, ++ host_config *hostconfig, ++ const std::string &realPodSandboxID, Errors &error) -> int; + void MakeContainerConfig(const runtime::v1alpha2::ContainerConfig &config, container_config *cConfig, + Errors &error); + void CreateContainerLogSymlink(const std::string &containerID, Errors &error); +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 711196ba..2bc6bed7 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -619,6 +619,37 @@ auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, E + return seccompSecurityOpts; + } + ++auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) ++-> std::vector ++{ ++ // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) ++ // New version '=' , old version ':', iSulad cri is based on v18.09, so iSulad cri use new version separator ++ const char securityOptSep { '=' }; ++ // LabeSep is consistent with the separator used when parsing labels ++ const char labeSep { ':' }; ++ std::vector selinuxOpts { }; ++ char **labelArr = nullptr; ++ size_t labelArrLen = 0; ++ std::vector opts = {"user", "role", "type", "level"}; ++ std::vector vect; ++ ++ labelArr = util_string_split_n(selinuxLabel.c_str(), labeSep, 4); ++ if (labelArr == nullptr) { ++ error.Errorf("Invalid selinux label: %s", selinuxLabel.c_str()); ++ return vect; ++ } ++ ++ labelArrLen = util_array_len((const char **)labelArr); ++ for (size_t i {}; i < labelArrLen; i++) { ++ iSuladOpt tmp = { "label", opts[i] + std::string(1, labeSep) + std::string(labelArr[i]), "" }; ++ selinuxOpts.push_back(tmp); ++ } ++ ++ util_free_array(labelArr); ++ ++ return fmtiSuladOpts(selinuxOpts, securityOptSep); ++} ++ + auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string + { + cri_checkpoint *criCheckpoint { nullptr }; +diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h +index d50759ad..4ef227d7 100644 +--- a/src/daemon/entry/cri/cri_helpers.h ++++ b/src/daemon/entry/cri/cri_helpers.h +@@ -127,6 +127,9 @@ auto ToIsuladContainerStatus(const runtime::v1alpha2::ContainerStateValue &state + auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) + -> std::vector; + ++auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) ++-> std::vector; ++ + auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string; + + void GetCheckpoint(const std::string &jsonCheckPoint, CRI::PodSandboxCheckpoint &checkpoint, Errors &error); +-- +2.25.1 + diff --git a/0037-clean-container-process-after-execSync-timeout-exit.patch b/0037-clean-container-process-after-execSync-timeout-exit.patch new file mode 100644 index 0000000..405581f --- /dev/null +++ b/0037-clean-container-process-after-execSync-timeout-exit.patch @@ -0,0 +1,434 @@ +From 53935dc36121b10975e047438ae8135ffa702920 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 4 Apr 2023 16:30:54 +0800 +Subject: [PATCH 37/46] clean container process after execSync timeout exit + +Signed-off-by: zhongtao +--- + src/cmd/isulad-shim/common.c | 30 ++++ + src/cmd/isulad-shim/common.h | 11 ++ + src/cmd/isulad-shim/main.c | 15 +- + src/cmd/isulad-shim/process.c | 158 +++++++++++++----- + src/cmd/isulad-shim/process.h | 2 +- + .../modules/runtime/isula/isula_rt_ops.c | 40 ++--- + 6 files changed, 188 insertions(+), 68 deletions(-) + +diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c +index 0c345187..f188da1e 100644 +--- a/src/cmd/isulad-shim/common.c ++++ b/src/cmd/isulad-shim/common.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + + int set_fd_no_inherited(int fd) + { +@@ -316,3 +317,32 @@ int open_no_inherit(const char *path, int flag, mode_t mode) + + return fd; + } ++ ++static bool is_invalid_error_str(const char *err_str, const char *numstr) ++{ ++ return err_str == NULL || err_str == numstr || *err_str != '\0'; ++} ++ ++int shim_util_safe_uint64(const char *numstr, uint64_t *converted) ++{ ++ char *err_str = NULL; ++ uint64_t ull; ++ ++ if (numstr == NULL || converted == NULL) { ++ return -EINVAL; ++ } ++ ++ errno = 0; ++ ull = strtoull(numstr, &err_str, 0); ++ if (errno > 0) { ++ return -errno; ++ } ++ ++ if (is_invalid_error_str(err_str, numstr)) { ++ return -EINVAL; ++ } ++ ++ *converted = (uint64_t)ull; ++ return 0; ++} ++ +diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h +index d06c5256..91808295 100644 +--- a/src/cmd/isulad-shim/common.h ++++ b/src/cmd/isulad-shim/common.h +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + + #ifdef __cplusplus + extern "C" { +@@ -31,6 +32,14 @@ extern "C" { + #define SHIM_ERR (-1) + #define SHIM_ERR_WAIT (-2) + #define SHIM_ERR_NOT_REQUIRED (-3) ++#define SHIM_ERR_TIMEOUT (-4) ++ ++// common exit code is defined in stdlib.h ++// EXIT_FAILURE 1 : Failing exit status. ++// EXIT_SUCCESS 0 : Successful exit status. ++// custom shim exit code ++// SHIM_EXIT_TIMEOUT 2: Container process timeout exit code ++#define SHIM_EXIT_TIMEOUT 2 + + #define INFO_MSG "info" + #define WARN_MSG "warn" +@@ -68,6 +77,8 @@ void close_fd(int *pfd); + + int open_no_inherit(const char *path, int flag, mode_t mode); + ++int shim_util_safe_uint64(const char *numstr, uint64_t *converted); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c +index eedd8fda..68e99e53 100644 +--- a/src/cmd/isulad-shim/main.c ++++ b/src/cmd/isulad-shim/main.c +@@ -62,7 +62,8 @@ static int set_subreaper() + return SHIM_OK; + } + +-static int parse_args(int argc, char **argv, char **cid, char **bundle, char **rt_name, char **log_level) ++static int parse_args(int argc, char **argv, char **cid, char **bundle, char **rt_name, char **log_level, ++ uint64_t *timeout) + { + if (argc < 4) { + return SHIM_ERR; +@@ -82,6 +83,12 @@ static int parse_args(int argc, char **argv, char **cid, char **bundle, char **r + } + } + ++ if (argc > 5) { ++ if (shim_util_safe_uint64(strdup(argv[5]), timeout) != 0) { ++ return SHIM_ERR; ++ } ++ } ++ + return SHIM_OK; + } + +@@ -99,6 +106,8 @@ int main(int argc, char **argv) + int efd = -1; + process_t *p = NULL; + pthread_t tid_accept; ++ // execSync timeout ++ uint64_t timeout = 0; + + g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); + if (g_log_fd < 0) { +@@ -117,7 +126,7 @@ int main(int argc, char **argv) + exit(EXIT_FAILURE); + } + +- ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level); ++ ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level, &timeout); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "parse args failed:%d", ret); + exit(EXIT_FAILURE); +@@ -167,5 +176,5 @@ int main(int argc, char **argv) + + released_timeout_exit(); + +- return process_signal_handle_routine(p, tid_accept); ++ return process_signal_handle_routine(p, tid_accept, timeout); + } +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 5222629c..02609911 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1213,69 +1213,145 @@ static int try_wait_all_child(void) + return 1; + } + +-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept) ++static int waitpid_with_timeout(int ctr_pid, int *status, const int64_t timeout) + { +- int ret = SHIM_ERR; +- bool exit_shim = false; + int nret = 0; +- int i; +- struct timespec ts; ++ time_t start_time = time(NULL); ++ time_t end_time; ++ double interval; ++ int st; + + for (;;) { +- int status; +- ret = reap_container(p->ctr_pid, &status); ++ nret = waitpid(-1, &st, WNOHANG); ++ if (nret == ctr_pid) { ++ break; ++ } ++ end_time = time(NULL); ++ interval = difftime(end_time, start_time); ++ if (nret == 0 && interval >= timeout) { ++ return SHIM_ERR_TIMEOUT; ++ } ++ // sleep some time instead to avoid cpu full running and then retry. ++ usleep(1000); ++ } ++ ++ if (WIFSIGNALED(st)) { ++ *status = EXIT_SIGNAL_OFFSET + WTERMSIG(st); ++ } else { ++ *status = WEXITSTATUS(st); ++ } ++ ++ if (*status == CONTAINER_ACTION_REBOOT) { ++ nret = setenv("CONTAINER_ACTION", "reboot", 1); ++ if (nret != SHIM_OK) { ++ write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); ++ } ++ } else if (*status == CONTAINER_ACTION_SHUTDOWN) { ++ nret = setenv("CONTAINER_ACTION", "shutdown", 1); ++ if (nret != SHIM_OK) { ++ write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); ++ } ++ } ++ return SHIM_OK; ++} ++ ++/* ++ * If timeout <= 0, blocking wait in reap_container. ++ * If timeout > 0, non-blocking wait pid with timeout. ++ */ ++static int wait_container_process_with_timeout(process_t *p, const unsigned int timeout, int *status) ++{ ++ int ret = SHIM_ERR; ++ ++ if (timeout > 0) { ++ return waitpid_with_timeout(p->ctr_pid, status, timeout); ++ } ++ ++ for (;;) { ++ ret = reap_container(p->ctr_pid, status); + if (ret == SHIM_OK) { +- exit_shim = true; +- if (status == CONTAINER_ACTION_REBOOT) { ++ if (*status == CONTAINER_ACTION_REBOOT) { + ret = setenv("CONTAINER_ACTION", "reboot", 1); + if (ret != SHIM_OK) { + write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); + } +- } else if (status == CONTAINER_ACTION_SHUTDOWN) { ++ } else if (*status == CONTAINER_ACTION_SHUTDOWN) { + ret = setenv("CONTAINER_ACTION", "shutdown", 1); + if (ret != SHIM_OK) { + write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); + } + } +- } else if (ret == SHIM_ERR_WAIT) { ++ return SHIM_OK; ++ } ++ ++ if (ret == SHIM_ERR_WAIT) { + /* avoid thread entering the infinite loop */ + usleep(1000); ++ } ++ ++ if (ret == SHIM_ERR) { ++ // if the child process is not expected, retry. + continue; + } +- if (exit_shim) { +- process_kill_all(p); ++ } + +- // wait atmost 120 seconds +- DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); +- if (nret != 0) { +- write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds"); +- } ++} + +- process_delete(p); +- if (p->exit_fd > 0) { +- (void)write_nointr(p->exit_fd, &status, sizeof(int)); +- } +- // wait for task_console_accept thread termination. In order to make sure that +- // the io_copy connection is established and io_thread is not used by multiple threads. +- if (p->state->terminal) { +- if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { +- write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); +- nret = pthread_join(tid_accept, NULL); +- } else { +- // Set the maximum waiting time to 60s to prevent stuck. +- ts.tv_sec += 60; +- nret = pthread_timedjoin_np(tid_accept, NULL, &ts); +- } ++int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout) ++{ ++ int i; ++ int nret = 0; ++ int ret = 0; ++ int status = 0; ++ struct timespec ts; + +- if (nret != 0) { +- write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); +- } +- } ++ ret = wait_container_process_with_timeout(p, timeout, &status); ++ if (ret == SHIM_ERR_TIMEOUT) { ++ // kill container process to ensure process_kill_all effective ++ nret = kill(p->ctr_pid, SIGKILL); ++ if (nret < 0 && errno != ESRCH) { ++ write_message(g_log_fd, ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid); ++ exit(EXIT_FAILURE); ++ } ++ } + +- for (i = 0; i < 3; i++) { +- destroy_io_thread(p, i); +- } +- return status; ++ process_kill_all(p); ++ ++ // wait atmost 120 seconds ++ DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); ++ if (nret != 0) { ++ write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds"); ++ } ++ ++ process_delete(p); ++ if (p->exit_fd > 0) { ++ (void)write_nointr(p->exit_fd, &status, sizeof(int)); ++ } ++ // wait for task_console_accept thread termination. In order to make sure that ++ // the io_copy connection is established and io_thread is not used by multiple threads. ++ if (p->state->terminal) { ++ if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { ++ write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); ++ nret = pthread_join(tid_accept, NULL); ++ } else { ++ // Set the maximum waiting time to 60s to prevent stuck. ++ ts.tv_sec += 60; ++ nret = pthread_timedjoin_np(tid_accept, NULL, &ts); + } ++ ++ if (nret != 0) { ++ write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); ++ } ++ } ++ ++ for (i = 0; i < 3; i++) { ++ destroy_io_thread(p, i); + } ++ ++ if (ret == SHIM_ERR_TIMEOUT) { ++ write_message(g_log_fd, INFO_MSG, "Wait %d timeout", p->ctr_pid); ++ exit(SHIM_EXIT_TIMEOUT); ++ } ++ return status; ++ + } +diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h +index 66820f68..7e3259e8 100644 +--- a/src/cmd/isulad-shim/process.h ++++ b/src/cmd/isulad-shim/process.h +@@ -97,7 +97,7 @@ process_t* new_process(char *id, char *bundle, char *runtime); + int open_io(process_t *p, pthread_t *tid_accept); + int process_io_init(process_t *p); + int create_process(process_t *p); +-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept); ++int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout); + + #ifdef __cplusplus + } +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index e974964a..5a01b8c6 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -54,6 +54,7 @@ + #define SHIM_LOG_SIZE ((BUFSIZ - 100) / 2) + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 ++#define SHIM_EXIT_TIMEOUT 2 + + // file name formats of cgroup resources json + #define RESOURCE_FNAME_FORMATS "%s/resources.json" +@@ -692,27 +693,6 @@ static int status_to_exit_code(int status) + return exit_code; + } + +-static int try_wait_pid(pid_t pid) +-{ +- if (waitpid(pid, NULL, WNOHANG) == pid) { +- return 0; +- } +- +- return 1; +-} +- +-static void kill_and_show_err(pid_t pid) +-{ +- int nret = 0; +- kill(pid, SIGKILL); +- // wait atmost 0.5 seconds +- DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid); +- if (nret != 0) { +- WARN("Fail to wait isulad-shim"); +- } +- isulad_set_error_message("Exec container error;exec timeout"); +-} +- + static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, + int *exit_code, const int64_t timeout) + { +@@ -731,7 +711,14 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + params[i++] = bundle; + params[i++] = runtime_cmd; + params[i++] = "info"; +- params[i++] = "2m0s"; ++ // execSync timeout ++ if (timeout > 0) { ++ params[i] = util_int_to_string(timeout); ++ if (params[i] == NULL) { ++ ERROR("Failed to convert execSync timeout %ld to string", timeout); ++ return -1; ++ } ++ } + runtime_exec_param_dump(params); + + if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) { +@@ -805,7 +792,7 @@ realexec: + goto out; + } + +- status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err); ++ status = util_wait_for_pid_status(pid); + if (status < 0) { + ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno)); + ret = -1; +@@ -1204,6 +1191,13 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + goto errlog_out; + } + ++ if (*exit_code == SHIM_EXIT_TIMEOUT) { ++ ret = -1; ++ isulad_set_error_message("Exec container error;exec timeout"); ++ ERROR("isulad-shim %d exit for execing timeout", pid); ++ goto errlog_out; ++ } ++ + pid = get_container_process_pid(workdir); + if (pid < 0) { + ERROR("%s: failed get exec process id", workdir); +-- +2.25.1 + diff --git a/0038-support-to-config-selinux-label-in-cri.patch b/0038-support-to-config-selinux-label-in-cri.patch new file mode 100644 index 0000000..5d6591e --- /dev/null +++ b/0038-support-to-config-selinux-label-in-cri.patch @@ -0,0 +1,346 @@ +From d231a8e5f8ea1c8a5584fdc8baa579b18d1b6a19 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 29 Mar 2023 03:26:45 +0800 +Subject: [PATCH 38/46] support to config selinux label in cri + +Signed-off-by: zhongtao +--- + .../cri/cri_container_manager_service_impl.cc | 47 +++++---- + .../cri/cri_container_manager_service_impl.h | 1 + + src/daemon/entry/cri/cri_helpers.cc | 95 ++++++++++++++++++- + src/daemon/entry/cri/cri_helpers.h | 17 +++- + .../cri_pod_sandbox_manager_service_impl.cc | 47 ++++----- + 5 files changed, 149 insertions(+), 58 deletions(-) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +index a64d222c..6278512f 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +@@ -98,30 +98,21 @@ auto ContainerManagerServiceImpl::PackCreateContainerHostConfigSecurityContext( + // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) + // New version '=' , old version ':', iSulad cri is based on v18.09, so iSulad cri use new version separator + const char securityOptSep { '=' }; +- std::vector securityOpts = CRIHelpers::GetSecurityOpts( +- containerConfig.linux().security_context().seccomp_profile_path(), securityOptSep, error); ++ const ::runtime::v1alpha2::LinuxContainerSecurityContext &context = containerConfig.linux().security_context(); ++ CRIHelpers::commonSecurityContext commonContext = { ++ .hasSELinuxOption = context.has_selinux_options(), ++ .selinuxOption = context.selinux_options(), ++ .seccompProfile = context.seccomp_profile_path(), ++ }; ++ std::vector securityOpts = CRIHelpers::GetSecurityOpts(commonContext, securityOptSep, error); + if (error.NotEmpty()) { +- error.Errorf("failed to generate security options for container %s", containerConfig.metadata().name().c_str()); ++ error.Errorf("Failed to generate security options for container %s", containerConfig.metadata().name().c_str()); + return -1; + } +- if (!securityOpts.empty()) { +- char **tmp_security_opt = nullptr; +- if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { +- error.Errorf("Out of memory"); +- return -1; +- } +- size_t newSize = (hostconfig->security_opt_len + securityOpts.size()) * sizeof(char *); +- size_t oldSize = hostconfig->security_opt_len * sizeof(char *); +- int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); +- if (ret != 0) { +- error.Errorf("Out of memory"); +- return -1; +- } +- hostconfig->security_opt = tmp_security_opt; +- for (const auto &securityOpt : securityOpts) { +- hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); +- hostconfig->security_opt_len++; +- } ++ CRIHelpers::AddSecurityOptsToHostConfig(securityOpts, hostconfig, error); ++ if (error.NotEmpty()) { ++ error.Errorf("Failed to add securityOpts to hostconfig for container %s", containerConfig.metadata().name().c_str()); ++ return -1; + } + return 0; + } +@@ -148,7 +139,7 @@ auto ContainerManagerServiceImpl::DoUsePodLevelSELinuxConfig(const runtime::v1al + } + + tmp_str = std::string(inspect->process_label); +- selinuxLabelOpts = CRIHelpers::GetSELinuxLabelOpts(tmp_str, error); ++ selinuxLabelOpts = CRIHelpers::GetPodSELinuxLabelOpts(tmp_str, error); + if (error.NotEmpty()) { + ERROR("Failed to get SELinuxLabelOpts for container %s", containerConfig.metadata().name().c_str()); + goto cleanup; +@@ -181,7 +172,14 @@ cleanup: + return ret; + } + +- ++auto ContainerManagerServiceImpl::IsSELinuxLabelEmpty(const ::runtime::v1alpha2::SELinuxOption &selinuxOption) -> bool ++{ ++ if (selinuxOption.user().length() == 0 && selinuxOption.role().length() == 0 && selinuxOption.type().length() == 0 && ++ selinuxOption.level().length() == 0) { ++ return true; ++ } ++ return false; ++} + + auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( + const runtime::v1alpha2::ContainerConfig &containerConfig, +@@ -219,7 +217,8 @@ auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig( + + // If selinux label is not specified in container config, use pod level SELinux config + if (!containerConfig.linux().has_security_context() || +- !containerConfig.linux().security_context().has_selinux_options()) { ++ !containerConfig.linux().security_context().has_selinux_options() || ++ IsSELinuxLabelEmpty(containerConfig.linux().security_context().selinux_options())) { + if (DoUsePodLevelSELinuxConfig(containerConfig, hostconfig, realPodSandboxID, error) != 0) { + error.SetError("Failed to security context to host config"); + goto cleanup; +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.h b/src/daemon/entry/cri/cri_container_manager_service_impl.h +index dcb2cd44..ad53a65f 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.h ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.h +@@ -84,6 +84,7 @@ private: + Errors &error) -> container_create_request *; + auto GenerateCreateContainerHostConfig(const runtime::v1alpha2::ContainerConfig &containerConfig, + const std::string &realPodSandboxID, Errors &error) -> host_config *; ++ auto IsSELinuxLabelEmpty(const ::runtime::v1alpha2::SELinuxOption &selinuxOption) -> bool; + auto GenerateCreateContainerCustomConfig(const std::string &containerName, const std::string &realPodSandboxID, + const runtime::v1alpha2::ContainerConfig &containerConfig, + const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig, +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 2bc6bed7..ec14d0b2 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -68,6 +68,8 @@ const std::string Constants::CNI_MUTL_NET_EXTENSION_KEY { "extension.network.kub + const std::string Constants::CNI_MUTL_NET_EXTENSION_ARGS_KEY { "CNI_MUTLINET_EXTENSION" }; + const std::string Constants::CNI_ARGS_EXTENSION_PREFIX_KEY { "extension.network.kubernetes.io/cniargs/" }; + const std::string Constants::IMAGE_NAME_ANNOTATION_KEY { "io.kubernetes.cri.image-name" }; ++// Usually, the format of level is "s0:c60,c525" or "s0-s0:c40.c23" ++const std::string Constants::SELINUX_LABEL_LEVEL_PATTERN { "^s[0-9](-s[0-9])?(:c[0-9]{1,4}(\\.c[0-9]{1,4})?(,c[0-9]{1,4}(\\.c[0-9]{1,4})?)*)?$" }; + + const char *InternalLabelKeys[] = { CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY.c_str(), + CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str(), +@@ -598,6 +600,34 @@ auto GetSeccompiSuladOpts(const std::string &seccompProfile, Errors &error) -> s + return ret; + } + ++ ++auto GetSelinuxiSuladOpts(const ::runtime::v1alpha2::SELinuxOption &selinux, Errors &error)-> std::vector ++{ ++ std::vector selinuxOpts { }; ++ // LabeSep is consistent with the separator used when parsing labels ++ const char labeSep { ':' }; ++ ++ if (selinux.level().length() != 0 && ++ util_reg_match(CRIHelpers::Constants::SELINUX_LABEL_LEVEL_PATTERN.c_str(), selinux.level().c_str()) != 0) { ++ error.Errorf("The format of 'level' %s is not correct", selinux.level().c_str()); ++ return selinuxOpts; ++ } ++ ++ if (selinux.user().length() > 0) { ++ selinuxOpts.push_back({ "label", std::string("user") + std::string(1, labeSep) + selinux.user(), "" }); ++ } ++ if (selinux.role().length() > 0) { ++ selinuxOpts.push_back({ "label", std::string("role") + std::string(1, labeSep) + selinux.role(), "" }); ++ } ++ if (selinux.type().length() > 0) { ++ selinuxOpts.push_back({ "label", std::string("type") + std::string(1, labeSep) + selinux.type(), "" }); ++ } ++ if (selinux.level().length() > 0) { ++ selinuxOpts.push_back({ "label", std::string("level") + std::string(1, labeSep) + selinux.level(), "" }); ++ } ++ return selinuxOpts; ++} ++ + auto GetSeccompSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) + -> std::vector + { +@@ -609,17 +639,44 @@ auto GetSeccompSecurityOpts(const std::string &seccompProfile, const char &separ + return fmtiSuladOpts(seccompOpts, separator); + } + +-auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) ++auto GetSELinuxLabelOpts(const bool hasSELinuxOption, const ::runtime::v1alpha2::SELinuxOption &selinux, ++ const char &separator, Errors &error) + -> std::vector + { +- std::vector seccompSecurityOpts = GetSeccompSecurityOpts(seccompProfile, separator, error); ++ if (!hasSELinuxOption) { ++ return std::vector(); ++ } ++ ++ std::vector selinuxOpts = GetSelinuxiSuladOpts(selinux, error); + if (error.NotEmpty()) { +- error.Errorf("failed to generate seccomp security options for container: %s", error.GetMessage().c_str()); ++ return std::vector(); + } +- return seccompSecurityOpts; ++ ++ return fmtiSuladOpts(selinuxOpts, separator); + } + +-auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) ++auto GetSecurityOpts(const commonSecurityContext &context, const char &separator, Errors &error) ++-> std::vector ++{ ++ std::vector securityOpts; ++ std::vector seccompSecurityOpts = GetSeccompSecurityOpts(context.seccompProfile, separator, error); ++ if (error.NotEmpty()) { ++ error.Errorf("Failed to generate seccomp security options for container: %s", error.GetMessage().c_str()); ++ return securityOpts; ++ } ++ ++ std::vector selinuxOpts = CRIHelpers::GetSELinuxLabelOpts(context.hasSELinuxOption, ++ context.selinuxOption, separator, error); ++ if (error.NotEmpty()) { ++ error.Errorf("Failed to generate SELinuxLabel options for container %s", error.GetMessage().c_str()); ++ return securityOpts; ++ } ++ securityOpts.insert(securityOpts.end(), seccompSecurityOpts.begin(), seccompSecurityOpts.end()); ++ securityOpts.insert(securityOpts.end(), selinuxOpts.begin(), selinuxOpts.end()); ++ return securityOpts; ++} ++ ++auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) + -> std::vector + { + // security Opt Separator Change Version : k8s v1.23.0 (Corresponds to docker 1.11.x) +@@ -650,6 +707,34 @@ auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) + return fmtiSuladOpts(selinuxOpts, securityOptSep); + } + ++void AddSecurityOptsToHostConfig(std::vector &securityOpts, host_config *hostconfig, Errors &error) ++{ ++ if (securityOpts.empty()) { ++ return; ++ } ++ ++ char **tmp_security_opt = nullptr; ++ if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hostconfig->security_opt_len) { ++ error.Errorf("Too many securityOpts"); ++ ERROR("Too many securityOpts"); ++ return; ++ } ++ size_t newSize = (hostconfig->security_opt_len + securityOpts.size()) * sizeof(char *); ++ size_t oldSize = hostconfig->security_opt_len * sizeof(char *); ++ int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hostconfig->security_opt, oldSize); ++ if (ret != 0) { ++ error.Errorf("Out of memory"); ++ ERROR("Out of memory"); ++ return; ++ } ++ hostconfig->security_opt = tmp_security_opt; ++ for (const auto &securityOpt : securityOpts) { ++ hostconfig->security_opt[hostconfig->security_opt_len] = util_strdup_s(securityOpt.c_str()); ++ hostconfig->security_opt_len++; ++ } ++ ++} ++ + auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string + { + cri_checkpoint *criCheckpoint { nullptr }; +diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h +index 4ef227d7..a5d6cf61 100644 +--- a/src/daemon/entry/cri/cri_helpers.h ++++ b/src/daemon/entry/cri/cri_helpers.h +@@ -67,10 +67,17 @@ public: + static const std::string CNI_MUTL_NET_EXTENSION_KEY; + static const std::string CNI_MUTL_NET_EXTENSION_ARGS_KEY; + static const std::string CNI_ARGS_EXTENSION_PREFIX_KEY; ++ static const std::string SELINUX_LABEL_LEVEL_PATTERN; + + static const std::string IMAGE_NAME_ANNOTATION_KEY; + }; + ++struct commonSecurityContext { ++ const bool hasSELinuxOption; ++ const ::runtime::v1alpha2::SELinuxOption selinuxOption; ++ const std::string seccompProfile; ++}; ++ + auto GetDefaultSandboxImage(Errors &err) -> std::string; + + auto MakeLabels(const google::protobuf::Map &mapLabels, Errors &error) +@@ -124,12 +131,18 @@ auto ValidateCheckpointKey(const std::string &key, Errors &error) -> bool; + + auto ToIsuladContainerStatus(const runtime::v1alpha2::ContainerStateValue &state) -> std::string; + +-auto GetSecurityOpts(const std::string &seccompProfile, const char &separator, Errors &error) ++auto GetSELinuxLabelOpts(const bool hasSELinuxOption, const ::runtime::v1alpha2::SELinuxOption &selinux, ++ const char &separator, Errors &error) + -> std::vector; + +-auto GetSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) ++auto GetSecurityOpts(const commonSecurityContext &context, const char &separator, Errors &error) + -> std::vector; + ++auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) ++-> std::vector; ++ ++void AddSecurityOptsToHostConfig(std::vector &securityOpts, host_config *hostconfig, Errors &error); ++ + auto CreateCheckpoint(CRI::PodSandboxCheckpoint &checkpoint, Errors &error) -> std::string; + + void GetCheckpoint(const std::string &jsonCheckPoint, CRI::PodSandboxCheckpoint &checkpoint, Errors &error); +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +index 7ff545db..1cb3254d 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +@@ -197,34 +197,27 @@ void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alph + const char securityOptSep = '='; + + // Security Opts +- if (c.linux().has_security_context()) { +- std::vector securityOpts = +- CRIHelpers::GetSecurityOpts(c.linux().security_context().seccomp_profile_path(), securityOptSep, error); +- if (error.NotEmpty()) { +- error.Errorf("failed to generate security options for sandbox %s: %s", +- c.metadata().name().c_str(), error.GetMessage().c_str()); +- return; +- } +- if (!securityOpts.empty()) { +- char **tmp_security_opt = nullptr; ++ if (!c.linux().has_security_context()) { ++ return; ++ } + +- if (securityOpts.size() > (SIZE_MAX / sizeof(char *)) - hc->security_opt_len) { +- error.Errorf("Out of memory"); +- return; +- } +- size_t newSize = (hc->security_opt_len + securityOpts.size()) * sizeof(char *); +- size_t oldSize = hc->security_opt_len * sizeof(char *); +- int ret = util_mem_realloc((void **)(&tmp_security_opt), newSize, (void *)hc->security_opt, oldSize); +- if (ret != 0) { +- error.Errorf("Out of memory"); +- return; +- } +- hc->security_opt = tmp_security_opt; +- for (const auto &securityOpt : securityOpts) { +- hc->security_opt[hc->security_opt_len] = util_strdup_s(securityOpt.c_str()); +- hc->security_opt_len++; +- } +- } ++ CRIHelpers::commonSecurityContext commonContext = { ++ .hasSELinuxOption = c.linux().security_context().has_selinux_options(), ++ .selinuxOption = c.linux().security_context().selinux_options(), ++ .seccompProfile = c.linux().security_context().seccomp_profile_path(), ++ }; ++ ++ std::vector securityOpts = CRIHelpers::GetSecurityOpts(commonContext, securityOptSep, error); ++ if (error.NotEmpty()) { ++ error.Errorf("Failed to generate security options for sandbox %s: %s", ++ c.metadata().name().c_str(), error.GetMessage().c_str()); ++ return; ++ } ++ CRIHelpers::AddSecurityOptsToHostConfig(securityOpts, hc, error); ++ if (error.NotEmpty()) { ++ error.Errorf("Failed to add securityOpts to hostconfig for sandbox %s: %s", c.metadata().name().c_str(), ++ error.GetMessage().c_str()); ++ return; + } + } + +-- +2.25.1 + diff --git a/0039-add-files_limit-to-oci-spec.patch b/0039-add-files_limit-to-oci-spec.patch new file mode 100644 index 0000000..db23d49 --- /dev/null +++ b/0039-add-files_limit-to-oci-spec.patch @@ -0,0 +1,94 @@ +From 6b5a0a345ae767cc245d1008377b8489acf2c0c7 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sat, 8 Apr 2023 14:50:20 +0800 +Subject: [PATCH 39/46] add files_limit to oci spec + +Signed-off-by: zhongtao +--- + src/daemon/modules/spec/specs.c | 49 +++++++++++++++++++++++++++++++++ + 1 file changed, 49 insertions(+) + +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index f32ff911..ad6d01d2 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -890,6 +890,27 @@ static int make_sure_oci_spec_linux_resources_pids(oci_runtime_spec *oci_spec) + return 0; + } + ++static int make_sure_oci_spec_linux_resources_files(oci_runtime_spec *oci_spec) ++{ ++ int ret = 0; ++ ++ ret = make_sure_oci_spec_linux_resources(oci_spec); ++ if (ret < 0) { ++ return -1; ++ } ++ ++ if (oci_spec->linux->resources->files != NULL) { ++ return 0; ++ } ++ ++ oci_spec->linux->resources->files = util_common_calloc_s(sizeof(defs_resources_files)); ++ if (oci_spec->linux->resources->files == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ return 0; ++} ++ + static int merge_pids_limit(oci_runtime_spec *oci_spec, int64_t pids_limit) + { + int ret = 0; +@@ -905,6 +926,20 @@ out: + return ret; + } + ++static int merge_files_limit(oci_runtime_spec *oci_spec, int64_t files_limit) ++{ ++ int ret = 0; ++ ++ ret = make_sure_oci_spec_linux_resources_files(oci_spec); ++ if (ret < 0) { ++ ERROR("Failed to merge files limit"); ++ return ret; ++ } ++ ++ oci_spec->linux->resources->files->limit = files_limit; ++ return ret; ++} ++ + static int merge_hostname(oci_runtime_spec *oci_spec, const host_config *host_spec, container_config *container_spec) + { + free(oci_spec->hostname); +@@ -1251,6 +1286,15 @@ out: + return ret; + } + ++static int merge_conf_files_limit(oci_runtime_spec *oci_spec, const host_config *host_spec) ++{ ++ if (host_spec->files_limit == 0) { ++ return 0; ++ } ++ ++ return merge_files_limit(oci_spec, host_spec->files_limit); ++} ++ + int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec) + { + int ret = 0; +@@ -1290,6 +1334,11 @@ int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec) + goto out; + } + ++ ret = merge_conf_files_limit(oci_spec, host_spec); ++ if (ret != 0) { ++ goto out; ++ } ++ + out: + return ret; + } +-- +2.25.1 + diff --git a/0040-support-setting-pod-to-privilege.patch b/0040-support-setting-pod-to-privilege.patch new file mode 100644 index 0000000..b0ef0ee --- /dev/null +++ b/0040-support-setting-pod-to-privilege.patch @@ -0,0 +1,25 @@ +From 28ee9b5b9e09e0b9e4eaf87356af900d0e380ff0 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 30 Mar 2023 06:15:56 +0800 +Subject: [PATCH 40/46] support setting pod to privilege + +Signed-off-by: zhongtao +--- + src/daemon/entry/cri/cri_security_context.cc | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/daemon/entry/cri/cri_security_context.cc b/src/daemon/entry/cri/cri_security_context.cc +index 1cd9287d..b2843626 100644 +--- a/src/daemon/entry/cri/cri_security_context.cc ++++ b/src/daemon/entry/cri/cri_security_context.cc +@@ -209,6 +209,7 @@ void ApplySandboxSecurityContext(const runtime::v1alpha2::LinuxPodSandboxConfig + } + if (lc.has_security_context()) { + const runtime::v1alpha2::LinuxSandboxSecurityContext &old = lc.security_context(); ++ sc->set_privileged(old.privileged()); + if (old.has_run_as_user()) { + *sc->mutable_run_as_user() = old.run_as_user(); + } +-- +2.25.1 + diff --git a/0041-add-hugepage_limit.patch b/0041-add-hugepage_limit.patch new file mode 100644 index 0000000..d110157 --- /dev/null +++ b/0041-add-hugepage_limit.patch @@ -0,0 +1,76 @@ +From 1a4e5174a9abcc83c9ace0cf7cabbdaf03697ae3 Mon Sep 17 00:00:00 2001 +From: shijiaqi1 +Date: Wed, 8 Feb 2023 13:31:36 +0800 +Subject: [PATCH 41/46] add hugepage_limit + +--- + .../cri/cri_container_manager_service_impl.cc | 19 +++++++++++++++++ + src/daemon/entry/cri/cri_helpers.cc | 21 ++++++++++++++++++- + 2 files changed, 39 insertions(+), 1 deletion(-) + +diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +index 6278512f..5398c088 100644 +--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc +@@ -1226,6 +1226,25 @@ void ContainerManagerServiceImpl::UpdateContainerResources(const std::string &co + if (!resources.cpuset_mems().empty()) { + hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str()); + } ++ if (resources.hugepage_limits_size() != 0) { ++ hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s( ++ sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size()); ++ if (hostconfig->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < resources.hugepage_limits_size(); i++) { ++ hostconfig->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hostconfig->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ goto cleanup; ++ } ++ hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str()); ++ hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit(); ++ hostconfig->hugetlbs_len++; ++ } ++ } + + request->host_config = host_config_generate_json(hostconfig, &ctx, &perror); + if (request->host_config == nullptr) { +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index ec14d0b2..e588b6c4 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -447,8 +447,27 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc, + hc->cpuset_mems = util_strdup_s(rOpts.cpuset_mems().c_str()); + } + hc->oom_score_adj = rOpts.oom_score_adj(); +- } + ++ if (rOpts.hugepage_limits_size() != 0) { ++ hc->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(sizeof(host_config_hugetlbs_element *), ++ rOpts.hugepage_limits_size()); ++ if (hc->hugetlbs == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ for (int i = 0; i < rOpts.hugepage_limits_size(); i++) { ++ hc->hugetlbs[i] = ++ (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element)); ++ if (hc->hugetlbs[i] == nullptr) { ++ error.SetError("Out of memory"); ++ return; ++ } ++ hc->hugetlbs[i]->page_size = util_strdup_s(rOpts.hugepage_limits(i).page_size().c_str()); ++ hc->hugetlbs[i]->limit = rOpts.hugepage_limits(i).limit(); ++ hc->hugetlbs_len++; ++ } ++ } ++ } + createConfig->open_stdin = config.stdin(); + createConfig->tty = config.tty(); + } +-- +2.25.1 + diff --git a/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch b/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch new file mode 100644 index 0000000..ef6db49 --- /dev/null +++ b/0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch @@ -0,0 +1,232 @@ +From 8e1bd51183eeb2ca2713b85afd52b12e584bc7a8 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 12 Apr 2023 21:19:56 +0800 +Subject: [PATCH 42/46] add effective and permitted type of cap to oci spec + +Signed-off-by: zhongtao +--- + src/contrib/config/config.json | 28 ++++++ + .../config/systemcontainer_config.json | 28 ++++++ + src/daemon/modules/spec/specs_security.c | 88 ++++++++++++++----- + 3 files changed, 120 insertions(+), 24 deletions(-) + +diff --git a/src/contrib/config/config.json b/src/contrib/config/config.json +index f84f3394..9070a893 100644 +--- a/src/contrib/config/config.json ++++ b/src/contrib/config/config.json +@@ -36,10 +36,38 @@ + "CAP_AUDIT_WRITE" + ], + "effective": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" + ], + "inheritable": [ + ], + "permitted": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" + ], + "ambient": [ + ] +diff --git a/src/contrib/config/systemcontainer_config.json b/src/contrib/config/systemcontainer_config.json +index 8ebce8c6..9169956e 100644 +--- a/src/contrib/config/systemcontainer_config.json ++++ b/src/contrib/config/systemcontainer_config.json +@@ -36,10 +36,38 @@ + "CAP_AUDIT_WRITE" + ], + "effective": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" + ], + "inheritable": [ + ], + "permitted": [ ++ "CAP_CHOWN", ++ "CAP_DAC_OVERRIDE", ++ "CAP_FSETID", ++ "CAP_FOWNER", ++ "CAP_MKNOD", ++ "CAP_NET_RAW", ++ "CAP_SETGID", ++ "CAP_SETUID", ++ "CAP_SETFCAP", ++ "CAP_SETPCAP", ++ "CAP_NET_BIND_SERVICE", ++ "CAP_SYS_CHROOT", ++ "CAP_KILL", ++ "CAP_AUDIT_WRITE" + ], + "ambient": [ + ] +diff --git a/src/daemon/modules/spec/specs_security.c b/src/daemon/modules/spec/specs_security.c +index 62f67082..d4884097 100644 +--- a/src/daemon/modules/spec/specs_security.c ++++ b/src/daemon/modules/spec/specs_security.c +@@ -261,36 +261,80 @@ free_out: + return ret; + } + ++// tweak_all_type_capabilities can tweak all support type capabilities by adding or dropping capabilities ++// based on the basic capabilities. ++static int tweak_all_type_capabilities(defs_process_capabilities **caps, const char **adds, size_t adds_len, ++ const char **drops, size_t drops_len) ++{ ++ int ret = 0; ++ ret = tweak_capabilities(&((*caps)->bounding), &((*caps)->bounding_len), adds, adds_len, drops, drops_len); ++ if (ret != 0) { ++ ERROR("Failed to tweak bounding capabilities"); ++ return -1; ++ } ++ ++ ret = tweak_capabilities(&((*caps)->permitted), &((*caps)->permitted_len), adds, adds_len, drops, drops_len); ++ if (ret != 0) { ++ ERROR("Failed to tweak permitted capabilities"); ++ return -1; ++ } ++ ++ ret = tweak_capabilities(&((*caps)->effective), &((*caps)->effective_len), adds, adds_len, drops, drops_len); ++ if (ret != 0) { ++ ERROR("Failed to tweak effective capabilities"); ++ return -1; ++ } ++ return ret; ++} ++ ++static void clear_caps(defs_process_capabilities **caps) ++{ ++ util_free_array_by_len((*caps)->bounding, (*caps)->bounding_len); ++ (*caps)->bounding_len = 0; ++ (*caps)->bounding = NULL; ++ ++ util_free_array_by_len((*caps)->permitted, (*caps)->permitted_len); ++ (*caps)->permitted_len = 0; ++ (*caps)->permitted = NULL; ++ ++ util_free_array_by_len((*caps)->effective, (*caps)->effective_len); ++ (*caps)->effective_len = 0; ++ (*caps)->effective = NULL; ++} ++ + int refill_oci_process_capabilities(defs_process_capabilities **caps, const char **src_caps, size_t src_caps_len) + { + int ret = 0; +- size_t i = 0; + + if (*caps == NULL) { + *caps = util_common_calloc_s(sizeof(defs_process_capabilities)); + if (*caps == NULL) { +- ret = -1; +- goto out; ++ return -1; + } + } + +- if ((*caps)->bounding != NULL) { +- // free current capabilities +- for (i = 0; i < ((*caps)->bounding_len); i++) { +- free((*caps)->bounding[i]); +- (*caps)->bounding[i] = NULL; +- } +- free((*caps)->bounding); +- (*caps)->bounding = NULL; +- } +- (*caps)->bounding_len = 0; ++ // clear current capabilities ++ clear_caps(caps); + + // copy capabilities + ret = copy_capabilities(&((*caps)->bounding), &((*caps)->bounding_len), src_caps, src_caps_len); + if (ret != 0) { +- ERROR("Failed to copy all capabilities"); ++ ERROR("Failed to copy all bounding capabilities"); ++ return -1; + } +-out: ++ ++ ret = copy_capabilities(&((*caps)->permitted), &((*caps)->permitted_len), src_caps, src_caps_len); ++ if (ret != 0) { ++ ERROR("Failed to copy all permitted capabilities"); ++ return -1; ++ } ++ ++ ret = copy_capabilities(&((*caps)->effective), &((*caps)->effective_len), src_caps, src_caps_len); ++ if (ret != 0) { ++ ERROR("Failed to copy all effective capabilities"); ++ return -1; ++ } ++ + return ret; + } + +@@ -823,25 +867,21 @@ int merge_caps(oci_runtime_spec *oci_spec, const char **adds, size_t adds_len, c + + ret = make_sure_oci_spec_process_capabilities(oci_spec); + if (ret < 0) { +- goto out; ++ return ret; + } + + if (adds_len > LIST_SIZE_MAX || drops_len > LIST_SIZE_MAX) { + ERROR("Too many capabilities to add or drop, the limit is %lld", LIST_SIZE_MAX); + isulad_set_error_message("Too many capabilities to add or drop, the limit is %d", LIST_SIZE_MAX); +- ret = -1; +- goto out; ++ return -1; + } + +- ret = tweak_capabilities(&oci_spec->process->capabilities->bounding, &oci_spec->process->capabilities->bounding_len, +- adds, adds_len, drops, drops_len); ++ ret = tweak_all_type_capabilities(&oci_spec->process->capabilities, adds, adds_len, drops, drops_len); + if (ret != 0) { +- ERROR("Failed to tweak capabilities"); +- ret = -1; +- goto out; ++ ERROR("Failed to tweak all type capabilities"); ++ return -1; + } + +-out: + return ret; + } + +-- +2.25.1 + diff --git a/0043-isulad-shim-fix-log-loss-bug.patch b/0043-isulad-shim-fix-log-loss-bug.patch new file mode 100644 index 0000000..28fdf44 --- /dev/null +++ b/0043-isulad-shim-fix-log-loss-bug.patch @@ -0,0 +1,30 @@ +From 795f9f56bd33ed382ae1e68300247ef651d77864 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Sun, 16 Apr 2023 19:57:48 +1400 +Subject: [PATCH 43/46] isulad-shim: fix log loss bug + +Signed-off-by: zhongtao +--- + src/cmd/isulad-shim/process.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index 02609911..a676e7ce 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -1348,6 +1348,12 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons + destroy_io_thread(p, i); + } + ++ if (!p->state->exec) { ++ // if log did not contain "/n", print remaind container log when exit isulad-shim ++ shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0); ++ shim_write_container_log_file(p->terminal, STDID_ERR, NULL, 0); ++ } ++ + if (ret == SHIM_ERR_TIMEOUT) { + write_message(g_log_fd, INFO_MSG, "Wait %d timeout", p->ctr_pid); + exit(SHIM_EXIT_TIMEOUT); +-- +2.25.1 + diff --git a/0044-remove-unused-func.patch b/0044-remove-unused-func.patch new file mode 100644 index 0000000..b9a8693 --- /dev/null +++ b/0044-remove-unused-func.patch @@ -0,0 +1,69 @@ +From 2b3a35ec1e0e3afd090618f2120fd8e756c4c0c6 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Wed, 19 Apr 2023 07:22:42 +0800 +Subject: [PATCH 44/46] remove unused func + +Signed-off-by: zhongtao +--- + .../modules/runtime/isula/isula_rt_ops.c | 31 ++----------------- + 1 file changed, 2 insertions(+), 29 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 5a01b8c6..3a6269a1 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -1078,28 +1078,6 @@ int rt_isula_rm(const char *id, const char *runtime, const rt_rm_params_t *param + return 0; + } + +-static char *try_generate_exec_id() +-{ +- char *id = NULL; +- +- id = util_common_calloc_s(sizeof(char) * (CONTAINER_EXEC_ID_MAX_LEN + 1)); +- if (id == NULL) { +- ERROR("Out of memory"); +- return NULL; +- } +- +- if (util_generate_random_str(id, (size_t)CONTAINER_EXEC_ID_MAX_LEN) != 0) { +- ERROR("Generate id failed"); +- goto err_out; +- } +- +- return id; +- +-err_out: +- free(id); +- return NULL; +-} +- + static bool fg_exec(const rt_exec_params_t *params) + { + if (params->console_fifos[0] != NULL || params->console_fifos[1] != NULL || params->console_fifos[2] != NULL) { +@@ -1122,7 +1100,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + int pid = 0; + shim_client_process_state p = { 0 }; + +- if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL) { ++ if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL || params->suffix == NULL) { + ERROR("nullptr arguments not allowed"); + return -1; + } +@@ -1135,12 +1113,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + return -1; + } + +- if (params->suffix != NULL) { +- exec_id = util_strdup_s(params->suffix); +- } else { +- exec_id = try_generate_exec_id(); +- } +- ++ exec_id = util_strdup_s(params->suffix); + if (exec_id == NULL) { + ERROR("Out of memory or generate exec id failed"); + return -1; +-- +2.25.1 + diff --git a/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch b/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch new file mode 100644 index 0000000..3470175 --- /dev/null +++ b/0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch @@ -0,0 +1,32 @@ +From 26c4a702ad9c919d8413b64f498d58b13375bbda Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 18 Apr 2023 17:30:03 +0800 +Subject: [PATCH 45/46] if the exit code in the response of execSync is not + zero, an error will be returned in cri + +Signed-off-by: zhongtao +--- + src/daemon/entry/connect/grpc/runtime_runtime_service.cc | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +index 451eeeef..4cc5a4ee 100644 +--- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc +@@ -289,6 +289,13 @@ grpc::Status RuntimeRuntimeServiceImpl::ExecSync(grpc::ServerContext *context, + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } + ++ if (reply->exit_code() != 0) { ++ ERROR("Object: CRI, Type: Sync exec in container: %s with exit code: %d", request->container_id().c_str(), ++ reply->exit_code()); ++ error.SetError(reply->stderr()); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); ++ } ++ + WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str()); + + return grpc::Status::OK; +-- +2.25.1 + diff --git a/0046-free-timeout-when-shim_create-finished.patch b/0046-free-timeout-when-shim_create-finished.patch new file mode 100644 index 0000000..3cabcb5 --- /dev/null +++ b/0046-free-timeout-when-shim_create-finished.patch @@ -0,0 +1,89 @@ +From 4dfc94f2beb816eb2e26ede07e803a230405b193 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 20 Apr 2023 13:28:26 +0800 +Subject: [PATCH 46/46] free timeout when shim_create finished + +Signed-off-by: zhongtao +--- + .../modules/runtime/isula/isula_rt_ops.c | 30 +++++++++++++------ + 1 file changed, 21 insertions(+), 9 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index 3a6269a1..ceaf464e 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -694,7 +694,7 @@ static int status_to_exit_code(int status) + } + + static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, +- int *exit_code, const int64_t timeout) ++ int *exit_code, const char* timeout) + { + pid_t pid = 0; + int exec_fd[2] = { -1, -1 }; +@@ -712,12 +712,8 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char + params[i++] = runtime_cmd; + params[i++] = "info"; + // execSync timeout +- if (timeout > 0) { +- params[i] = util_int_to_string(timeout); +- if (params[i] == NULL) { +- ERROR("Failed to convert execSync timeout %ld to string", timeout); +- return -1; +- } ++ if (timeout != NULL) { ++ params[i++] = timeout; + } + runtime_exec_param_dump(params); + +@@ -917,7 +913,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1); ++ ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, NULL); + if (ret != 0) { + runtime_call_delete_force(workdir, runtime, id); + ERROR("%s: failed create shim process", id); +@@ -1099,6 +1095,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + char bundle[PATH_MAX] = { 0 }; + int pid = 0; + shim_client_process_state p = { 0 }; ++ char *timeout = NULL; + + if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL || params->suffix == NULL) { + ERROR("nullptr arguments not allowed"); +@@ -1158,7 +1155,18 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p + } + + get_runtime_cmd(runtime, &cmd); +- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout); ++ ++ // execSync timeout ++ if (params->timeout > 0) { ++ timeout = util_int_to_string(params->timeout); ++ if (timeout == NULL) { ++ ERROR("Failed to convert execSync timeout %ld to string", params->timeout); ++ ret = -1; ++ goto del_out; ++ } ++ } ++ ++ ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, timeout); + if (ret != 0) { + ERROR("%s: failed create shim process for exec %s", id, exec_id); + goto errlog_out; +@@ -1183,6 +1191,10 @@ errlog_out: + show_shim_runtime_errlog(workdir); + } + ++ if (timeout != NULL) { ++ free(timeout); ++ } ++ + del_out: + if (util_recursive_rmdir(workdir, 0)) { + ERROR("rmdir %s failed", workdir); +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec index 6f73309..436fa22 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.0.18 -%global _release 4 +%global _release 5 %global is_systemd 1 %global enable_shimv2 1 %global is_embedded 1 @@ -39,6 +39,26 @@ Patch0023: 0023-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch Patch0024: 0024-adapt-to-repo-of-openeuler-url-changed.patch Patch0025: 0025-modify-sleep-time.patch Patch0026: 0026-change-goto-branch.patch +Patch0027: 0027-modifying-cpurt-file-permissions.patch +Patch0028: 0028-add-design-docs-for-cri-manager.patch +Patch0029: 0029-improve-check-of-process-failure.patch +Patch0030: 0030-support-isula-update-when-runtime-is-runc.patch +Patch0031: 0031-when-calling-runc-start-unset-NOTIFY_-SOCKET.patch +Patch0032: 0032-add-CRI-container-design-doc.patch +Patch0033: 0033-fix-util_getgrent_r-overflow.patch +Patch0034: 0034-modify-the-return-value-of-the-util_waitpid_with_tim.patch +Patch0035: 0035-fix-inspect-data-memleak.patch +Patch0036: 0036-containers-in-same-sandbox-should-have-same-process-.patch +Patch0037: 0037-clean-container-process-after-execSync-timeout-exit.patch +Patch0038: 0038-support-to-config-selinux-label-in-cri.patch +Patch0039: 0039-add-files_limit-to-oci-spec.patch +Patch0040: 0040-support-setting-pod-to-privilege.patch +Patch0041: 0041-add-hugepage_limit.patch +Patch0042: 0042-add-effective-and-permitted-type-of-cap-to-oci-spec.patch +Patch0043: 0043-isulad-shim-fix-log-loss-bug.patch +Patch0044: 0044-remove-unused-func.patch +Patch0045: 0045-if-the-exit-code-in-the-response-of-execSync-is-not-.patch +Patch0046: 0046-free-timeout-when-shim_create-finished.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -283,6 +303,12 @@ fi %endif %changelog +* Mon Apr 24 2023 zhangxiaoyu - 2.0.18-5 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: bugfix for runc and cri + * Wed Mar 09 2023 zhangxiaoyu - 2.0.18-4 - Type: bugfix - ID: NA