packages/iSulad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
iSulad/0057-change-isulad-shim-epoll-struct.patch
zhangxiaoyu 22ae3de547 upgrade from upstream 
Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
(cherry picked from commit 4b64b19964f903cdbea9493ab2eaeace8c823cad)
2023-05-05 09:49:33 +08:00

1794 lines
54 KiB
Diff
Raw Blame History

From 15275529aed5b4f2e317902dfd3390ee114b310f Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Sun, 16 Apr 2023 13:56:18 +0800
Subject: [PATCH 57/57] change isulad-shim epoll struct
Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
src/cmd/isulad-shim/common.c | 9 +
src/cmd/isulad-shim/common.h | 2 +
src/cmd/isulad-shim/linked_list.h | 134 ++++
src/cmd/isulad-shim/main.c | 13 +-
src/cmd/isulad-shim/mainloop.c | 168 +++++
src/cmd/isulad-shim/mainloop.h | 53 ++
src/cmd/isulad-shim/process.c | 941 ++++++++++++-------------
src/cmd/isulad-shim/process.h | 47 +-
test/cmd/isulad-shim/CMakeLists.txt | 1 +
test/cmd/isulad-shim/isulad-shim_ut.cc | 1 +
10 files changed, 825 insertions(+), 544 deletions(-)
create mode 100644 src/cmd/isulad-shim/linked_list.h
create mode 100644 src/cmd/isulad-shim/mainloop.c
create mode 100644 src/cmd/isulad-shim/mainloop.h
diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
index e1ca96e1..18443385 100644
--- a/src/cmd/isulad-shim/common.c
+++ b/src/cmd/isulad-shim/common.c
@@ -524,4 +524,13 @@ err_out:
util_free_array(res_array);
errno = tmp_errno;
return NULL;
+}
+
+void *util_common_calloc_s(size_t size)
+{
+ if (size == 0 || size > MAX_MEMORY_SIZE) {
+ return NULL;
+ }
+
+ return calloc((size_t)1, size);
}
\ No newline at end of file
diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
index 3de16ace..d733823b 100644
--- a/src/cmd/isulad-shim/common.h
+++ b/src/cmd/isulad-shim/common.h
@@ -128,6 +128,8 @@ char *util_strdup_s(const char *src);
char **util_string_split_multi(const char *src_str, char delim);
+void *util_common_calloc_s(size_t size);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/cmd/isulad-shim/linked_list.h b/src/cmd/isulad-shim/linked_list.h
new file mode 100644
index 00000000..3a87990a
--- /dev/null
+++ b/src/cmd/isulad-shim/linked_list.h
@@ -0,0 +1,134 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container linked list function definition
+ ******************************************************************************/
+#ifndef __LINKED_LIST_H
+#define __LINKED_LIST_H
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct linked_list {
+ void *elem;
+ struct linked_list *next;
+ struct linked_list *prev;
+};
+
+/* Iterate through an linked list. */
+#define linked_list_for_each(__iterator, __list) \
+ for ((__iterator) = (__list)->next; \
+ (__iterator) != (__list); \
+ (__iterator) = (__iterator)->next)
+
+/* Iterate safely through an linked list. */
+#define linked_list_for_each_safe(__iterator, __list, __next) \
+ for ((__iterator) = (__list)->next, (__next) = (__iterator)->next; \
+ (__iterator) != (__list); \
+ (__iterator) = (__next), (__next) = (__next)->next)
+
+/* Initialize list. */
+static inline void linked_list_init(struct linked_list *list)
+{
+ list->elem = NULL;
+ list->next = list->prev = list;
+}
+
+/* Add an element to a list. See linked_list_add() and linked_list_add_tail() for an
+ * idiom. */
+static inline void linked_list_add_elem(struct linked_list *list, void *elem)
+{
+ list->elem = elem;
+}
+
+/* Retrieve first element of list. */
+static inline void *linked_list_first_elem(struct linked_list *list)
+{
+ return list->next->elem;
+}
+
+/* Retrieve last element of list. */
+static inline void *linked_list_last_elem(struct linked_list *list)
+{
+ return list->prev->elem;
+}
+
+/* Retrieve first node of list. */
+static inline void *linked_list_first_node(struct linked_list *list)
+{
+ return list->next;
+}
+
+/* Determine if list is empty. */
+static inline int linked_list_empty(struct linked_list *list)
+{
+ return list == list->next;
+}
+
+/* Workhorse to be called from linked_list_add() and linked_list_add_tail(). */
+static inline void __linked_list_add(struct linked_list *newlist,
+ struct linked_list *prev,
+ struct linked_list *next)
+{
+ next->prev = newlist;
+ newlist->next = next;
+ newlist->prev = prev;
+ prev->next = newlist;
+}
+
+/* Idiom to add an element to the beginning of an linked list */
+static inline void linked_list_add(struct linked_list *head,
+ struct linked_list *list)
+{
+ __linked_list_add(list, head, head->next);
+}
+
+/* Idiom to add an element to the end of an linked list */
+static inline void linked_list_add_tail(struct linked_list *head,
+ struct linked_list *list)
+{
+ __linked_list_add(list, head->prev, head);
+}
+
+/* Idiom to free an linked list */
+static inline void linked_list_del(const struct linked_list *list)
+{
+ struct linked_list *next = NULL;
+ struct linked_list *prev = NULL;
+
+ next = list->next;
+ prev = list->prev;
+ next->prev = prev;
+ prev->next = next;
+}
+
+/* Return length of the list. */
+static inline size_t linked_list_len(struct linked_list *list)
+{
+ size_t i = 0;
+ struct linked_list *iter = NULL;
+ linked_list_for_each(iter, list) {
+ i++;
+ }
+
+ return i;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c
index 68e99e53..0cccdf06 100644
--- a/src/cmd/isulad-shim/main.c
+++ b/src/cmd/isulad-shim/main.c
@@ -105,9 +105,9 @@ int main(int argc, char **argv)
int ret = SHIM_ERR;
int efd = -1;
process_t *p = NULL;
- pthread_t tid_accept;
// execSync timeout
uint64_t timeout = 0;
+ pthread_t tid_epoll;
g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640);
if (g_log_fd < 0) {
@@ -154,18 +154,13 @@ int main(int argc, char **argv)
}
}
- /* create main loop and start epoll for io copy */
- ret = process_io_init(p);
+ /* start epoll for io copy */
+ ret = process_io_start(p, &tid_epoll);
if (ret != SHIM_OK) {
write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret);
exit(EXIT_FAILURE);
}
- ret = open_io(p, &tid_accept);
- if (ret != SHIM_OK) {
- exit(EXIT_FAILURE);
- }
-
ret = create_process(p);
if (ret != SHIM_OK) {
if (p->console_sock_path != NULL) {
@@ -176,5 +171,5 @@ int main(int argc, char **argv)
released_timeout_exit();
- return process_signal_handle_routine(p, tid_accept, timeout);
+ return process_signal_handle_routine(p, tid_epoll, timeout);
}
diff --git a/src/cmd/isulad-shim/mainloop.c b/src/cmd/isulad-shim/mainloop.c
new file mode 100644
index 00000000..53c4d856
--- /dev/null
+++ b/src/cmd/isulad-shim/mainloop.c
@@ -0,0 +1,168 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container mainloop functions
+ ******************************************************************************/
+#include "mainloop.h"
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/epoll.h>
+
+#include "common.h"
+
+struct epoll_loop_handler {
+ epoll_loop_callback_t cb;
+ int cbfd;
+ void *cbdata;
+};
+
+#define MAX_EVENTS 100
+
+/* epoll loop */
+int epoll_loop(struct epoll_descr *descr, int t)
+{
+ int i;
+ int ret = 0;
+ struct epoll_loop_handler *epoll_handler = NULL;
+ struct epoll_event evs[MAX_EVENTS];
+
+ while (1) {
+ int ep_fds = epoll_wait(descr->fd, evs, MAX_EVENTS, t);
+ if (ep_fds < 0) {
+ if (errno == EINTR) {
+ continue;
+ }
+ ret = -1;
+ goto out;
+ }
+
+ for (i = 0; i < ep_fds; i++) {
+ epoll_handler = (struct epoll_loop_handler *)(evs[i].data.ptr);
+ if (epoll_handler->cb(epoll_handler->cbfd, evs[i].events, epoll_handler->cbdata, descr) !=
+ EPOLL_LOOP_HANDLE_CONTINUE) {
+ goto out;
+ }
+ }
+
+ if (ep_fds == 0 && t != 0) {
+ if (descr->timeout_cb != NULL) {
+ descr->timeout_cb(descr->timeout_cbdata);
+ }
+ goto out;
+ }
+
+ if (linked_list_empty(&descr->handler_list)) {
+ goto out;
+ }
+ }
+out:
+ return ret;
+}
+
+/* epoll loop add handler */
+int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data)
+{
+ struct epoll_event ev = { 0 };
+ struct epoll_loop_handler *epoll_handler = NULL;
+ struct linked_list *node = NULL;
+
+ // if fd == -1; dose not add handler for it.
+ if (fd < 0) {
+ return 0;
+ }
+
+ epoll_handler = util_common_calloc_s(sizeof(*epoll_handler));
+ if (epoll_handler == NULL) {
+ goto fail_out;
+ }
+
+ epoll_handler->cbfd = fd;
+ epoll_handler->cb = callback;
+ epoll_handler->cbdata = data;
+
+ ev.events = EPOLLIN;
+ ev.data.ptr = epoll_handler;
+
+ if (epoll_ctl(descr->fd, EPOLL_CTL_ADD, fd, &ev) < 0) {
+ goto fail_out;
+ }
+
+ node = util_common_calloc_s(sizeof(struct linked_list));
+ if (node == NULL) {
+ goto fail_out;
+ }
+
+ node->elem = epoll_handler;
+ linked_list_add(&descr->handler_list, node);
+ return 0;
+
+fail_out:
+ (void)epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, &ev);
+ free(epoll_handler);
+ return -1;
+}
+
+/* epoll loop del handler */
+int epoll_loop_del_handler(struct epoll_descr *descr, int fd)
+{
+ struct epoll_loop_handler *epoll_handler = NULL;
+ struct linked_list *index = NULL;
+
+ linked_list_for_each(index, &descr->handler_list) {
+ epoll_handler = index->elem;
+
+ if (fd == epoll_handler->cbfd) {
+ if (epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, NULL)) {
+ goto fail_out;
+ }
+
+ linked_list_del(index);
+ free(index->elem);
+ free(index);
+ return 0;
+ }
+ }
+
+fail_out:
+ return -1;
+}
+
+/* epoll loop open */
+int epoll_loop_open(struct epoll_descr *descr)
+{
+ descr->fd = epoll_create1(EPOLL_CLOEXEC);
+ if (descr->fd < 0) {
+ return -1;
+ }
+
+ linked_list_init(&(descr->handler_list));
+ descr->timeout_cb = NULL;
+ descr->timeout_cbdata = NULL;
+ return 0;
+}
+
+/* epoll loop close */
+int epoll_loop_close(struct epoll_descr *descr)
+{
+ struct linked_list *index = NULL;
+ struct linked_list *next = NULL;
+
+ linked_list_for_each_safe(index, &(descr->handler_list), next) {
+ linked_list_del(index);
+ free(index->elem);
+ free(index);
+ }
+
+ return close(descr->fd);
+}
diff --git a/src/cmd/isulad-shim/mainloop.h b/src/cmd/isulad-shim/mainloop.h
new file mode 100644
index 00000000..7a4f1cfd
--- /dev/null
+++ b/src/cmd/isulad-shim/mainloop.h
@@ -0,0 +1,53 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container mainloop definition
+ ******************************************************************************/
+#ifndef UTILS_CUTILS_MAINLOOP_H
+#define UTILS_CUTILS_MAINLOOP_H
+
+#include <stdint.h>
+#include "linked_list.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*epoll_timeout_callback_t)(void *data);
+
+struct epoll_descr {
+ int fd;
+ struct linked_list handler_list;
+ epoll_timeout_callback_t timeout_cb;
+ void *timeout_cbdata;
+};
+
+#define EPOLL_LOOP_HANDLE_CONTINUE 0
+#define EPOLL_LOOP_HANDLE_CLOSE 1
+
+typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data, struct epoll_descr *descr);
+
+extern int epoll_loop(struct epoll_descr *descr, int t);
+
+extern int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data);
+
+extern int epoll_loop_del_handler(struct epoll_descr *descr, int fd);
+
+extern int epoll_loop_open(struct epoll_descr *descr);
+
+extern int epoll_loop_close(struct epoll_descr *descr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
index a5e0bd39..c31f8af6 100644
--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
@@ -28,6 +28,7 @@
#include <string.h>
#include <errno.h>
#include <sys/ioctl.h>
+#include <sys/eventfd.h>
#include <termios.h> // IWYU pragma: keep
#include <sys/resource.h> // IWYU pragma: keep
#include <isula_libutils/json_common.h>
@@ -37,6 +38,7 @@
#include "common.h"
#include "terminal.h"
+#include "mainloop.h"
#define MAX_EVENTS 100
#define DEFAULT_IO_COPY_BUF (16 * 1024)
@@ -78,7 +80,7 @@ static int receive_fd(int sock)
u_char *pfd = NULL;
int fd = -1;
int cmsgsize = CMSG_LEN(sizeof(int));
- struct cmsghdr *cmptr = (struct cmsghdr *)calloc(1, cmsgsize);
+ struct cmsghdr *cmptr = (struct cmsghdr *)util_common_calloc_s(cmsgsize);
if (cmptr == NULL) {
return -1;
}
@@ -128,107 +130,6 @@ static bool check_fd(int fd)
return true;
}
-static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to)
-{
- int ret = SHIM_ERR;
-
- if (io_thd == NULL || io_thd->ioc == NULL) {
- return SHIM_ERR;
- }
-
- io_copy_t *ioc = io_thd->ioc;
-
- if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
- return SHIM_ERR;
- }
- /* add src fd */
- if (from != -1 && ioc->fd_from == -1) {
- ioc->fd_from = from;
- struct epoll_event ev;
- ev.events = EPOLLIN;
- ev.data.ptr = io_thd;
-
- ret = epoll_ctl(epfd, EPOLL_CTL_ADD, from, &ev);
- if (ret != SHIM_OK) {
- write_message(g_log_fd, ERR_MSG, "add fd %d to epoll loop failed:%d", from, SHIM_SYS_ERR(errno));
- pthread_mutex_unlock(&(ioc->mutex));
- return SHIM_ERR;
- }
- }
-
- /* add dest fd */
- if (to != -1) {
- /* new fd_node_t for dest fd */
- fd_node_t *fn = (fd_node_t *)calloc(1, sizeof(fd_node_t));
- if (fn == NULL) {
- pthread_mutex_unlock(&(ioc->mutex));
- return SHIM_ERR;
- }
- fn->fd = to;
- fn->is_log = false;
- if (io_thd->terminal != NULL && to == io_thd->terminal->fd) {
- fn->is_log = true;
- }
- fn->next = NULL;
-
- if (ioc->fd_to == NULL) {
- ioc->fd_to = fn;
- } else {
- fd_node_t *tmp = ioc->fd_to;
- while (tmp->next != NULL) {
- tmp = tmp->next;
- }
- tmp->next = fn;
- }
- }
- pthread_mutex_unlock(&(ioc->mutex));
-
- return SHIM_OK;
-}
-
-static void remove_io_dispatch(io_thread_t *io_thd, int from, int to)
-{
- if (io_thd == NULL || io_thd->ioc == NULL) {
- return;
- }
- io_copy_t *ioc = io_thd->ioc;
-
- fd_node_t *tmp = NULL;
- do {
- /* remove src fd */
- if (from != -1 && from == ioc->fd_from) {
- struct epoll_event ev;
- ev.events = EPOLLIN;
- ev.data.fd = ioc->fd_from;
- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev);
- }
-
- /* remove dest fd */
- if (ioc->fd_to == NULL) {
- break;
- }
- if (ioc->fd_to->fd == to) {
- /* remove the first fd node */
- tmp = ioc->fd_to;
- ioc->fd_to = ioc->fd_to->next;
- break;
- }
- fd_node_t *pre = ioc->fd_to;
- tmp = ioc->fd_to->next;
- while (tmp != NULL && tmp->fd != to) {
- pre = tmp;
- tmp = tmp->next;
- }
- if (tmp != NULL) {
- pre->next = tmp->next;
- }
- } while (0);
- if (tmp != NULL) {
- free(tmp);
- tmp = NULL;
- }
-}
-
static int get_exec_winsize(const char *buf, struct winsize *wsize)
{
char **array = NULL;
@@ -261,347 +162,287 @@ out:
return ret;
}
-static void *do_io_copy(void *data)
+static int sync_exit_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
{
- io_thread_t *io_thd = (io_thread_t *)data;
- if (io_thd == NULL || io_thd->ioc == NULL) {
- return NULL;
- }
- io_copy_t *ioc = io_thd->ioc;
- char *buf = calloc(1, DEFAULT_IO_COPY_BUF + 1);
- if (buf == NULL) {
- return NULL;
- }
+ return EPOLL_LOOP_HANDLE_CLOSE;
+}
- for (;;) {
- memset(buf, 0, DEFAULT_IO_COPY_BUF);
- (void)sem_wait(&(io_thd->sem_thd));
- if (io_thd->is_stdin && io_thd->shutdown) {
- break;
- }
+static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
+{
+ process_t *p = (process_t *)cbdata;
+ int r_count = 0;
+ int w_count = 0;
+ int *fd_to = NULL;
- int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
- if (r_count == -1) {
- if (errno == EAGAIN || errno == EINTR) {
- continue;
- }
- break;
- } else if (r_count == 0) {
- /* End of file. The remote has closed the connection */
- break;
- } else if (ioc->id != EXEC_RESIZE) {
- if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
- continue;
- }
+ if (events & EPOLLHUP) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
+ }
- fd_node_t *fn = ioc->fd_to;
- fd_node_t *next = NULL;
- for (; fn != NULL; fn = next) {
- next = fn->next;
- if (fn->is_log) {
- shim_write_container_log_file(io_thd->terminal, ioc->id, buf, r_count);
- } else {
- int w_count = write_nointr_in_total(fn->fd, buf, r_count);
- if (w_count < 0) {
- /* When any error occurs, remove the write fd */
- remove_io_dispatch(io_thd, -1, fn->fd);
- }
- }
- }
- pthread_mutex_unlock(&(ioc->mutex));
- } else {
- if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
- continue;
- }
+ if (!(events & EPOLLIN)) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
+ }
- int resize_fd = ioc->fd_to->fd;
- struct winsize wsize = { 0x00 };
- if (get_exec_winsize(buf, &wsize) < 0) {
- break;
- }
- if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) {
- break;
- }
- pthread_mutex_unlock(&(ioc->mutex));
- }
+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
- /*
- In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe.
- So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs.
- */
- if (io_thd->shutdown && r_count <= 0) {
- break;
- }
+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+ if (r_count <= 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- struct epoll_event ev;
- ev.events = EPOLLIN;
- ev.data.fd = ioc->fd_from;
- (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev);
- free(buf);
+ if (p->state->terminal) {
+ fd_to = &(p->recv_fd);
+ } else {
+ fd_to = &(p->shim_io->in);
+ }
+ w_count = write_nointr_in_total(*fd_to, p->buf, r_count);
+ if (w_count < 0) {
+ /* When any error occurs, set the write fd -1 */
+ write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno));
+ close(*fd_to);
+ *fd_to = -1;
+ }
- return NULL;
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
-static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data)
+static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
{
- io_thread_t *thd = (io_thread_t *)data;
- if (thd->ioc == NULL || fd != thd->ioc->fd_from) {
- return;
+ process_t *p = (process_t *)cbdata;
+ int r_count = 0;
+ int w_count = 0;
+
+ if (events & EPOLLHUP) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- if (event & EPOLLIN) {
- (void)sem_post(&thd->sem_thd);
- } else if (event & EPOLLHUP) {
- thd->shutdown = true;
- (void)sem_post(&thd->sem_thd);
+ if (!(events & EPOLLIN)) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
-}
-static int create_io_copy_thread(process_t *p, int std_id)
-{
- int ret = SHIM_ERR;
- io_thread_t *io_thd = NULL;
- io_copy_t *ioc = NULL;
+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
- ioc = (io_copy_t *)calloc(1, sizeof(io_copy_t));
- if (ioc == NULL) {
- goto failure;
+ if (p->block_read) {
+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+ } else {
+ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF);
}
- ioc->id = std_id;
- ioc->fd_from = -1;
- ioc->fd_to = NULL;
- if (pthread_mutex_init(&(ioc->mutex), NULL) != 0) {
- goto failure;
+ if (r_count <= 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- io_thd = (io_thread_t *)calloc(1, sizeof(io_thread_t));
- if (io_thd == NULL) {
- goto failure;
+ shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count);
+
+ if (p->isulad_io->out == -1) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
- if (sem_init(&io_thd->sem_thd, 0, 0) != 0) {
- write_message(g_log_fd, ERR_MSG, "sem init failed:%d", SHIM_SYS_ERR(errno));
- goto failure;
+
+ w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count);
+ if (w_count < 0) {
+ /* When any error occurs, set the write fd -1 */
+ write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno));
+ close(p->isulad_io->out);
+ p->isulad_io->out = -1;
}
- io_thd->epfd = p->io_loop_fd;
- io_thd->ioc = ioc;
- io_thd->shutdown = false;
- io_thd->is_stdin = std_id == STDID_IN ? true : false;
- io_thd->terminal = std_id != STDID_IN ? p->terminal : NULL;
- p->io_threads[std_id] = io_thd;
+ return EPOLL_LOOP_HANDLE_CONTINUE;
+}
- ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd);
- if (ret != SHIM_OK) {
- write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno));
- goto failure;
+static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
+{
+ process_t *p = (process_t *)cbdata;
+ int r_count = 0;
+ int w_count = 0;
+
+ if (events & EPOLLHUP) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- ret = SHIM_OK;
+ if (!(events & EPOLLIN)) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
+ }
- return ret;
+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
-failure:
- if (ioc != NULL) {
- pthread_mutex_destroy(&(ioc->mutex));
- free(ioc);
+ if (p->block_read) {
+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+ } else {
+ r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF);
}
- if (io_thd != NULL) {
- free(io_thd);
+ if (r_count <= 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- return SHIM_ERR;
-}
-
-static int start_io_copy_threads(process_t *p)
-{
- int ret = SHIM_ERR;
- int i;
-
- /* 4 threads for stdin, stdout, stderr and exec resize */
- for (i = 0; i < 4; i++) {
- /*
- * if the terminal is used, we do not need to active the io copy of stderr pipe,
- * for stderr and stdout are mixed together
- */
- if (i == STDID_ERR && p->state->terminal) {
- continue;
- }
+ shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count);
- ret = create_io_copy_thread(p, i);
- if (ret != SHIM_OK) {
- return SHIM_ERR;
- }
+ if (p->isulad_io->err == -1) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
- return SHIM_OK;
-}
-static void destroy_io_thread(process_t *p, int std_id)
-{
- io_thread_t *io_thd = p->io_threads[std_id];
- if (io_thd == NULL) {
- return;
+ w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count);
+ if (w_count < 0) {
+ /* When any error occurs, set the write fd -1 */
+ write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno));
+ close(p->isulad_io->err);
+ p->isulad_io->err = -1;
}
- io_thd->shutdown = true;
- (void)sem_post(&io_thd->sem_thd);
- pthread_join(io_thd->tid, NULL);
- if (io_thd->ioc != NULL) {
- free(io_thd->ioc);
- }
- free(io_thd);
- p->io_threads[std_id] = NULL;
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
-/*
- std_id: channel type
- isulad_stdio: one side of the isulad fifo file
- fd: one side of the shim io pipe
- ---------------------------------------------------------------
- | CHANNEL | iSulad Fifo Side | Flow Direction | fd |
- ---------------------------------------------------------------
- | STDIN | READ | --> | WRITE |
- ---------------------------------------------------------------
- | STDOUT | WRITE | <-- | READ |
- ---------------------------------------------------------------
- | STDERR | WRITE | <-- | READ |
- ---------------------------------------------------------------
- | RESIZE | READ | --> | WRITE |
- ---------------------------------------------------------------
-*/
-static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd)
+static int resize_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
{
- mode_t mode;
- int fd_isulad = -1;
- int *fd_from = NULL;
- int *fd_to = NULL;
+ process_t *p = (process_t *)cbdata;
+ int r_count = 0;
+ int resize_fd = -1;
- if (std_id == STDID_IN || std_id == EXEC_RESIZE) {
- mode = O_RDONLY;
- fd_from = &fd_isulad;
- fd_to = &fd;
- } else {
- mode = O_WRONLY;
- fd_from = &fd;
- fd_to = &fd_isulad;
+ if (events & EPOLLHUP) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- if (isulad_stdio != NULL && file_exists(isulad_stdio)) {
- fd_isulad = open_fifo_noblock(isulad_stdio, mode);
- if (fd_isulad < 0) {
- return SHIM_ERR;
- }
- /* open dummy fd to avoid resize epoll hub */
- if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) {
- return SHIM_ERR;
- }
+ if (!(events & EPOLLIN)) {
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
- if (*fd_from != -1) {
- if (std_id != STDID_IN && std_id != EXEC_RESIZE && p->io_threads[std_id]->terminal != NULL) {
- (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd);
- }
- return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to);
+ (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
+ r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+ if (r_count <= 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
}
- /* if no I/O source is available, the I/O thread nead to be destroyed */
- destroy_io_thread(p, std_id);
+ resize_fd = p->recv_fd;
+ struct winsize wsize = { 0x00 };
+ if (get_exec_winsize(p->buf, &wsize) < 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
+ }
+ if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) {
+ return EPOLL_LOOP_HANDLE_CLOSE;
+ }
- return SHIM_OK;
+ return EPOLL_LOOP_HANDLE_CONTINUE;
}
-static void *task_console_accept(void *data)
+static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
{
+ process_t *p = (process_t *)cbdata;
int conn_fd = -1;
- int recv_fd = -1;
int ret = SHIM_ERR;
- console_accept_t *ac = (console_accept_t *)data;
- conn_fd = accept(ac->listen_fd, NULL, NULL);
+ conn_fd = accept(p->listen_fd, NULL, NULL);
if (conn_fd < 0) {
- write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno));
+ write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno));
goto out;
}
- recv_fd = receive_fd(conn_fd);
- if (check_fd(recv_fd) != true) {
+ p->recv_fd = receive_fd(conn_fd);
+ if (check_fd(p->recv_fd) != true) {
write_message(g_log_fd, ERR_MSG, "check console fd failed");
goto out;
}
/* do console io copy */
- /* p.state.stdin---->runtime.console */
- ret = connect_to_isulad(ac->p, STDID_IN, ac->p->state->isulad_stdin, recv_fd);
+ // p->isulad_io->in ----> p->recv_fd
+ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
goto out;
}
-
- /* p.state.stdout<------runtime.console */
- ret = connect_to_isulad(ac->p, STDID_OUT, ac->p->state->isulad_stdout, recv_fd);
+ // p->recv_fd ----> p->isulad_io->out
+ ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p);
if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno));
goto out;
}
-
- /* p.state.resize_fifo------>runtime.console */
- ret = connect_to_isulad(ac->p, EXEC_RESIZE, ac->p->state->resize_fifo, recv_fd);
+ // p->isulad_io->resize ----> p->recv_fd
+ ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p);
if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno));
goto out;
}
out:
/* release listen socket at the first time */
- close_fd(&ac->listen_fd);
- if (ac->p->console_sock_path != NULL) {
- (void)unlink(ac->p->console_sock_path);
- free(ac->p->console_sock_path);
- ac->p->console_sock_path = NULL;
- }
- free(ac);
- if (ret != SHIM_OK) {
- /*
- * When an error occurs during the receiving of the fd , the process
- * exits directly. The files created in the working directory will be
- * deleted by its parent process isulad
- */
- exit(EXIT_FAILURE);
+ close_fd(&p->listen_fd);
+ if (p->console_sock_path != NULL) {
+ (void)unlink(p->console_sock_path);
+ free(p->console_sock_path);
+ p->console_sock_path = NULL;
}
- return NULL;
+ return ret;
}
-static void *io_epoll_loop(void *data)
+static int stdio_chown(int (*stdio_fd)[2], int uid, int gid)
{
- process_t *p = (process_t *)data;
- int wait_fds = 0;
- struct epoll_event evs[MAX_EVENTS];
- int i;
+ int i, j;
- if ((pthread_detach(pthread_self())) != 0) {
- write_message(g_log_fd, ERR_MSG, "detach thread failed");
- return NULL;
+ for (i = 0; i < 3; i++) {
+ for (j = 0; j < 2; j++) {
+ int ret = fchown(stdio_fd[i][j], uid, gid);
+ if (ret != SHIM_OK) {
+ return SHIM_ERR;
+ }
+ }
}
+ return SHIM_OK;
+}
- p->io_loop_fd = epoll_create1(EPOLL_CLOEXEC);
- if (p->io_loop_fd < 0) {
- write_message(g_log_fd, ERR_MSG, "epoll create failed:%d", SHIM_SYS_ERR(errno));
- exit(EXIT_FAILURE);
- }
- (void)sem_post(&p->sem_mainloop);
+static void stdio_release(int (*stdio_fd)[2])
+{
+ int i, j;
- for (;;) {
- wait_fds = epoll_wait(p->io_loop_fd, evs, MAX_EVENTS, -1);
- if (wait_fds < 0) {
- if (errno == EINTR) {
- continue;
+ for (i = 0; i < 3; i++) {
+ for (j = 0; j < 2; j++) {
+ if (stdio_fd[i][j] > 0) {
+ close(stdio_fd[i][j]);
}
- _exit(EXIT_FAILURE);
}
+ }
+}
- for (i = 0; i < wait_fds; i++) {
- io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr;
- sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io);
- }
+static stdio_t *initialize_io(process_t *p)
+{
+ int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } };
+
+ stdio_t *stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+ p->stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+ if (p->stdio == NULL || stdio == NULL) {
+ goto failure;
+ }
+
+ /* don't open resize pipe */
+ if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) ||
+ (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) {
+ write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
+ goto failure;
}
+
+ p->stdio->in = stdio_fd[0][0]; // r
+ stdio->in = stdio_fd[0][1]; // w
+ p->stdio->out = stdio_fd[1][1]; // w
+ stdio->out = stdio_fd[1][0]; // r
+ p->stdio->err = stdio_fd[2][1]; // w
+ stdio->err = stdio_fd[2][0]; // r
+ p->stdio->resize = stdio_fd[3][0]; // r
+ stdio->resize = stdio_fd[3][1]; // w
+
+ if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) {
+ goto failure;
+ }
+
+ return stdio;
+
+failure:
+ if (stdio != NULL) {
+ free(stdio);
+ stdio = NULL;
+ }
+ if (p->stdio != NULL) {
+ free(p->stdio);
+ p->stdio = NULL;
+ }
+ stdio_release(stdio_fd);
+
+ return NULL;
}
static int new_temp_console_path(process_t *p)
@@ -614,7 +455,7 @@ static int new_temp_console_path(process_t *p)
if (ret != SHIM_OK) {
return SHIM_ERR;
}
- p->console_sock_path = (char *)calloc(1, MAX_CONSOLE_SOCK_LEN + 1);
+ p->console_sock_path = (char *)util_common_calloc_s(MAX_CONSOLE_SOCK_LEN + 1);
if (p->console_sock_path == NULL) {
return SHIM_ERR;
}
@@ -628,12 +469,11 @@ static int new_temp_console_path(process_t *p)
return SHIM_OK;
}
-static int console_init(process_t *p, pthread_t *tid_accept)
+static int console_init(process_t *p, struct epoll_descr *descr)
{
int ret = SHIM_ERR;
int fd = -1;
struct sockaddr_un addr;
- console_accept_t *ac = NULL;
fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd < 0) {
@@ -657,145 +497,182 @@ static int console_init(process_t *p, pthread_t *tid_accept)
goto failure;
}
- ac = (console_accept_t *)calloc(1, sizeof(console_accept_t));
- if (ac == NULL) {
- goto failure;
- }
- ac->p = p;
- ac->listen_fd = fd;
+ p->listen_fd = fd;
- ret = pthread_create(tid_accept, NULL, task_console_accept, ac);
+ ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p);
if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno));
goto failure;
}
return SHIM_OK;
failure:
close_fd(&fd);
- if (ac != NULL) {
- free(ac);
- ac = NULL;
- }
(void)unlink(p->console_sock_path);
return SHIM_ERR;
}
-static int stdio_chown(int (*stdio_fd)[2], int uid, int gid)
+static int open_terminal_io(process_t *p, struct epoll_descr *descr)
{
- int i, j;
+ int ret = SHIM_ERR;
- for (i = 0; i < 3; i++) {
- for (j = 0; j < 2; j++) {
- int ret = fchown(stdio_fd[i][j], uid, gid);
- if (ret != SHIM_OK) {
- return SHIM_ERR;
- }
- }
+ ret = new_temp_console_path(p);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "get temp console sock path failed");
+ return SHIM_ERR;
}
- return SHIM_OK;
-}
-
-static void stdio_release(int (*stdio_fd)[2])
-{
- int i, j;
- for (i = 0; i < 3; i++) {
- for (j = 0; j < 2; j++) {
- if (stdio_fd[i][j] > 0) {
- close(stdio_fd[i][j]);
- }
- }
- }
+ /* begin listen from p->console_sock_path */
+ return console_init(p, descr);
}
-static stdio_t *initialize_io(process_t *p)
+static int open_generic_io(process_t *p, struct epoll_descr *descr)
{
- int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } };
-
- stdio_t *stdio = (stdio_t *)calloc(1, sizeof(stdio_t));
- p->stdio = (stdio_t *)calloc(1, sizeof(stdio_t));
- if (p->stdio == NULL || stdio == NULL) {
- goto failure;
- }
+ int ret = SHIM_ERR;
- /* don't open resize pipe */
- if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) ||
- (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) {
- write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
- goto failure;
+ // io: in: w out/err: r
+ stdio_t *io = initialize_io(p);
+ if (io == NULL) {
+ return SHIM_ERR;
}
+ p->shim_io = io;
- p->stdio->in = stdio_fd[0][0]; // r
- stdio->in = stdio_fd[0][1]; // w
- p->stdio->out = stdio_fd[1][1]; // w
- stdio->out = stdio_fd[1][0]; // r
- p->stdio->err = stdio_fd[2][1]; // w
- stdio->err = stdio_fd[2][0]; // r
- p->stdio->resize = stdio_fd[3][0]; // r
- stdio->resize = stdio_fd[3][1]; // w
-
- if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) {
- goto failure;
+ // p->isulad_io->in ----> p->shim_io->in
+ ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
+ return SHIM_ERR;
}
-
- return stdio;
-
-failure:
- if (stdio != NULL) {
- free(stdio);
- stdio = NULL;
+ // p->shim_io->out ----> p->isulad_io->out
+ ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno));
+ return SHIM_ERR;
}
- if (p->stdio != NULL) {
- free(p->stdio);
- p->stdio = NULL;
+ // p->shim_io->err ----> p->isulad_io->err
+ ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno));
+ return SHIM_ERR;
}
- stdio_release(stdio_fd);
- return NULL;
+ return SHIM_OK;
}
-static int open_terminal_io(process_t *p, pthread_t *tid_accept)
+static int set_non_block(int fd)
{
+ int flag = -1;
int ret = SHIM_ERR;
- ret = new_temp_console_path(p);
- if (ret != SHIM_OK) {
- write_message(g_log_fd, ERR_MSG, "get temp console sock path failed");
+ flag = fcntl(fd, F_GETFL, 0);
+ if (flag < 0) {
+ return SHIM_ERR;
+ }
+
+ ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK);
+ if (ret != 0) {
return SHIM_ERR;
}
- /* begin listen and accept fd from p->console_sock_path */
- return console_init(p, tid_accept);
+ return SHIM_OK;
}
-static int open_generic_io(process_t *p)
+/*
+ std_id: channel type
+ isulad_stdio: one side of the isulad fifo file
+ fd: one side of the shim io pipe
+ ---------------------------------------------------------------
+ | CHANNEL | iSulad Fifo Side | Flow Direction | fd |
+ ---------------------------------------------------------------
+ | STDIN | READ | --> | WRITE |
+ ---------------------------------------------------------------
+ | STDOUT | WRITE | <-- | READ |
+ ---------------------------------------------------------------
+ | STDERR | WRITE | <-- | READ |
+ ---------------------------------------------------------------
+ | RESIZE | READ | --> | WRITE |
+ ---------------------------------------------------------------
+*/
+static void *io_epoll_loop(void *data)
{
- int ret = SHIM_ERR;
+ int ret = 0;
+ int fd_out = -1;
+ int fd_err = -1;
+ process_t *p = (process_t *)data;
+ struct epoll_descr descr;
- // io: in: w out/err: r
- stdio_t *io = initialize_io(p);
- if (io == NULL) {
- return SHIM_ERR;
+ ret = epoll_loop_open(&descr);
+ if (ret != 0) {
+ write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno));
+ exit(EXIT_FAILURE);
}
- p->shim_io = io;
- /* stdin */
- ret = connect_to_isulad(p, STDID_IN, p->state->isulad_stdin, io->in);
- if (ret != SHIM_OK) {
- return SHIM_ERR;
+
+ // sync fd: epoll loop will exit when recive sync fd event.
+ ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p);
+ if (ret != 0) {
+ write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno));
+ exit(EXIT_FAILURE);
}
- /* stdout */
- ret = connect_to_isulad(p, STDID_OUT, p->state->isulad_stdout, io->out);
- if (ret != SHIM_OK) {
- return SHIM_ERR;
+
+ if (p->state->terminal) {
+ ret = open_terminal_io(p, &descr);
+ } else {
+ ret = open_generic_io(p, &descr);
}
- /* stderr */
- ret = connect_to_isulad(p, STDID_ERR, p->state->isulad_stderr, io->err);
if (ret != SHIM_OK) {
- return SHIM_ERR;
+ write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno));
+ exit(EXIT_FAILURE);
}
- return SHIM_OK;
+ (void)sem_post(&p->sem_mainloop);
+
+ ret = epoll_loop(&descr, -1);
+ if (ret != 0) {
+ write_message(g_log_fd, ERR_MSG, "epoll loop failed");
+ exit(EXIT_FAILURE);
+ }
+
+ // in order to avoid data loss, set fd non-block and read it
+ p->block_read = false;
+ if (p->state->terminal) {
+ fd_out = p->recv_fd;
+ } else {
+ fd_out = p->shim_io->out;
+ fd_err = p->shim_io->err;
+ }
+
+ if (fd_out > 0) {
+ ret = set_non_block(fd_out);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ for (;;) {
+ ret = stdout_cb(fd_out, EPOLLIN, p, &descr);
+ if (ret == EPOLL_LOOP_HANDLE_CLOSE) {
+ break;
+ }
+ }
+ }
+
+ if (fd_err > 0) {
+ ret = set_non_block(fd_err);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ for (;;) {
+ ret = stderr_cb(fd_err, EPOLLIN, p, &descr);
+ if (ret == EPOLL_LOOP_HANDLE_CLOSE) {
+ break;
+ }
+ }
+ }
+
+ return NULL;
}
static void adapt_for_isulad_stdin(process_t *p)
@@ -813,7 +690,7 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s
{
log_terminal *log_term = NULL;
- log_term = calloc(1, sizeof(log_terminal));
+ log_term = util_common_calloc_s(sizeof(log_terminal));
if (log_term == NULL) {
write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal");
goto clean_out;
@@ -858,11 +735,90 @@ clean_out:
return SHIM_ERR;
}
+static int open_isulad_fd(int std_id, const char *isulad_stdio, int *fd)
+{
+ mode_t mode = O_WRONLY;
+
+ if (std_id == STDID_IN || std_id == EXEC_RESIZE) {
+ mode = O_RDONLY;
+ }
+
+ if (isulad_stdio != NULL && file_exists(isulad_stdio)) {
+ *(fd) = open_fifo_noblock(isulad_stdio, mode);
+ if (*(fd) < 0) {
+ return -1;
+ }
+ /* open dummy fd to avoid resize epoll hub */
+ if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) {
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+
+static int init_isulad_stdio(process_t *p)
+{
+ int ret = SHIM_OK;
+ p->isulad_io = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+ if (p->isulad_io == NULL) {
+ return SHIM_ERR;
+ }
+
+ p->isulad_io->in = -1;
+ p->isulad_io->out = -1;
+ p->isulad_io->err = -1;
+ p->isulad_io->resize = -1;
+
+ ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin);
+ goto failure;
+ }
+
+ ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout);
+ goto failure;
+ }
+
+ ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr);
+ goto failure;
+ }
+
+ ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize);
+ if (ret != SHIM_OK) {
+ write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo);
+ goto failure;
+ }
+ return SHIM_OK;
+failure:
+ if (p->isulad_io != NULL) {
+ free(p->isulad_io);
+ p->isulad_io = NULL;
+ }
+ if (p->isulad_io->in > 0) {
+ close(p->isulad_io->in);
+ }
+ if (p->isulad_io->out > 0) {
+ close(p->isulad_io->out);
+ }
+ if (p->isulad_io->err > 0) {
+ close(p->isulad_io->err);
+ }
+ if (p->isulad_io->resize > 0) {
+ close(p->isulad_io->resize);
+ }
+ return SHIM_ERR;
+}
+
process_t *new_process(char *id, char *bundle, char *runtime)
{
shim_client_process_state *p_state;
process_t *p = NULL;
- int i;
int ret;
p_state = load_process();
@@ -870,7 +826,7 @@ process_t *new_process(char *id, char *bundle, char *runtime)
return NULL;
}
- p = (process_t *)calloc(1, sizeof(process_t));
+ p = (process_t *)util_common_calloc_s(sizeof(process_t));
if (p == NULL) {
return NULL;
}
@@ -889,16 +845,31 @@ process_t *new_process(char *id, char *bundle, char *runtime)
p->bundle = bundle;
p->runtime = runtime;
p->state = p_state;
-
+ p->block_read = true;
p->console_sock_path = NULL;
p->exit_fd = -1;
p->io_loop_fd = -1;
p->ctr_pid = -1;
+ p->listen_fd = -1;
+ p->recv_fd = -1;
p->stdio = NULL;
p->shim_io = NULL;
+ p->isulad_io = NULL;
- for (i = 0; i < 3; i++) {
- p->io_threads[i] = NULL;
+ p->sync_fd = eventfd(0, EFD_CLOEXEC);
+ if (p->sync_fd < 0) {
+ write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno));
+ goto failure;
+ }
+
+ ret = init_isulad_stdio(p);
+ if (ret != SHIM_OK) {
+ goto failure;
+ }
+
+ p->buf = util_common_calloc_s(DEFAULT_IO_COPY_BUF + 1);
+ if (p->buf == NULL) {
+ goto failure;
}
return p;
@@ -909,28 +880,11 @@ failure:
return NULL;
}
-int open_io(process_t *p, pthread_t *tid_accept)
+int process_io_start(process_t *p, pthread_t *tid_epoll)
{
int ret = SHIM_ERR;
- ret = start_io_copy_threads(p);
- if (ret != SHIM_OK) {
- return SHIM_ERR;
- }
-
- if (p->state->terminal) {
- return open_terminal_io(p, tid_accept);
- }
-
- return open_generic_io(p);
-}
-
-int process_io_init(process_t *p)
-{
- int ret = SHIM_ERR;
-
- pthread_t tid_loop;
- ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p);
+ ret = pthread_create(tid_epoll, NULL, io_epoll_loop, p);
if (ret != SHIM_OK) {
return SHIM_SYS_ERR(errno);
}
@@ -1091,8 +1045,8 @@ static void exec_runtime_process(process_t *p, int exec_fd)
}
char *cwd = getcwd(NULL, 0);
- char *log_path = (char *)calloc(1, PATH_MAX);
- char *pid_path = (char *)calloc(1, PATH_MAX);
+ char *log_path = (char *)util_common_calloc_s(PATH_MAX);
+ char *pid_path = (char *)util_common_calloc_s(PATH_MAX);
if (cwd == NULL || log_path == NULL || pid_path == NULL) {
(void)dprintf(exec_fd, "memory error: %s", strerror(errno));
_exit(EXIT_FAILURE);
@@ -1294,13 +1248,11 @@ static int wait_container_process_with_timeout(process_t *p, const unsigned int
}
-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout)
+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout)
{
- int i;
int nret = 0;
int ret = 0;
int status = 0;
- struct timespec ts;
ret = wait_container_process_with_timeout(p, timeout, &status);
if (ret == SHIM_ERR_TIMEOUT) {
@@ -1324,27 +1276,20 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons
if (p->exit_fd > 0) {
(void)write_nointr(p->exit_fd, &status, sizeof(int));
}
- // wait for task_console_accept thread termination. In order to make sure that
- // the io_copy connection is established and io_thread is not used by multiple threads.
- if (p->state->terminal) {
- if (clock_gettime(CLOCK_REALTIME, &ts) == -1) {
- write_message(g_log_fd, ERR_MSG, "Failed to get realtime");
- nret = pthread_join(tid_accept, NULL);
- } else {
- // Set the maximum waiting time to 60s to prevent stuck.
- ts.tv_sec += 60;
- nret = pthread_timedjoin_np(tid_accept, NULL, &ts);
- }
- if (nret != 0) {
- write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread");
+ if (p->sync_fd > 0) {
+ if (eventfd_write(p->sync_fd, 1)) {
+ write_message(g_log_fd, ERR_MSG, "Failed to write sync fd");
}
}
- for (i = 0; i < 3; i++) {
- destroy_io_thread(p, i);
+ nret = pthread_join(tid_epoll, NULL);
+ if (nret != 0) {
+ write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread");
}
+ close(p->sync_fd);
+
if (!p->state->exec) {
// if log did not contain "/n", print remaind container log when exit isulad-shim
shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0);
diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h
index 7e3259e8..4cb6462c 100644
--- a/src/cmd/isulad-shim/process.h
+++ b/src/cmd/isulad-shim/process.h
@@ -40,64 +40,37 @@ typedef struct {
int resize;
} stdio_t;
-typedef struct fd_node {
- int fd;
- bool is_log;
- struct fd_node *next;
-} fd_node_t;
-
-typedef struct {
- int fd_from;
- fd_node_t *fd_to;
- int id;// 0,1,2,3
- pthread_mutex_t mutex;
-} io_copy_t;
-
-typedef struct {
- int epfd;
- pthread_t tid;
- pthread_attr_t attr;
- sem_t sem_thd;
- io_copy_t *ioc;
- bool shutdown;
- bool is_stdin;
- log_terminal *terminal;// just used by stdout and stderr
-} io_thread_t;
-
typedef struct process {
char *id;
char *bundle;
char *runtime;
- char *console_sock_path;// pty socket path
+ char *console_sock_path; // pty socket path
int io_loop_fd;
int exit_fd;
int ctr_pid;
+ int sync_fd;
+ int listen_fd;
+ int recv_fd;
+ bool block_read;
log_terminal *terminal;
- stdio_t *stdio;// shim to on runtime side, in:r out/err: w
+ stdio_t *stdio; // shim to on runtime side, in:r out/err: w
stdio_t *shim_io; // shim io on isulad side, in: w out/err: r
- io_thread_t *io_threads[4];// stdin,stdout,stderr,exec_resize
+ stdio_t *isulad_io; // isulad io, in:r out/err: w
shim_client_process_state *state;
sem_t sem_mainloop;
+ char *buf;
} process_t;
-typedef struct {
- int listen_fd;
- process_t *p;
-} console_accept_t;
-
typedef struct {
int pid;
int status;
} process_exit_t;
-
-
process_t* new_process(char *id, char *bundle, char *runtime);
-int open_io(process_t *p, pthread_t *tid_accept);
-int process_io_init(process_t *p);
+int process_io_start(process_t *p, pthread_t *tid_epoll);
int create_process(process_t *p);
-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout);
+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout);
#ifdef __cplusplus
}
diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt
index e5c1cd6e..122538ff 100644
--- a/test/cmd/isulad-shim/CMakeLists.txt
+++ b/test/cmd/isulad-shim/CMakeLists.txt
@@ -6,6 +6,7 @@ add_executable(${EXE}
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/mainloop.c
isulad-shim_ut.cc)
target_include_directories(${EXE} PUBLIC
diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc
index 34ecd452..8c116ac8 100644
--- a/test/cmd/isulad-shim/isulad-shim_ut.cc
+++ b/test/cmd/isulad-shim/isulad-shim_ut.cc
@@ -11,6 +11,7 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
+#include "mainloop.h"
#include "process.h"
#include "common.h"
--
2.25.1
Reference in New Issue View Git Blame Copy Permalink