From 15275529aed5b4f2e317902dfd3390ee114b310f Mon Sep 17 00:00:00 2001 From: zhongtao Date: Sun, 16 Apr 2023 13:56:18 +0800 Subject: [PATCH 57/57] change isulad-shim epoll struct Signed-off-by: zhongtao --- src/cmd/isulad-shim/common.c | 9 + src/cmd/isulad-shim/common.h | 2 + src/cmd/isulad-shim/linked_list.h | 134 ++++ src/cmd/isulad-shim/main.c | 13 +- src/cmd/isulad-shim/mainloop.c | 168 +++++ src/cmd/isulad-shim/mainloop.h | 53 ++ src/cmd/isulad-shim/process.c | 941 ++++++++++++------------- src/cmd/isulad-shim/process.h | 47 +- test/cmd/isulad-shim/CMakeLists.txt | 1 + test/cmd/isulad-shim/isulad-shim_ut.cc | 1 + 10 files changed, 825 insertions(+), 544 deletions(-) create mode 100644 src/cmd/isulad-shim/linked_list.h create mode 100644 src/cmd/isulad-shim/mainloop.c create mode 100644 src/cmd/isulad-shim/mainloop.h diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c index e1ca96e1..18443385 100644 --- a/src/cmd/isulad-shim/common.c +++ b/src/cmd/isulad-shim/common.c @@ -524,4 +524,13 @@ err_out: util_free_array(res_array); errno = tmp_errno; return NULL; +} + +void *util_common_calloc_s(size_t size) +{ + if (size == 0 || size > MAX_MEMORY_SIZE) { + return NULL; + } + + return calloc((size_t)1, size); } \ No newline at end of file diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h index 3de16ace..d733823b 100644 --- a/src/cmd/isulad-shim/common.h +++ b/src/cmd/isulad-shim/common.h @@ -128,6 +128,8 @@ char *util_strdup_s(const char *src); char **util_string_split_multi(const char *src_str, char delim); +void *util_common_calloc_s(size_t size); + #ifdef __cplusplus } #endif diff --git a/src/cmd/isulad-shim/linked_list.h b/src/cmd/isulad-shim/linked_list.h new file mode 100644 index 00000000..3a87990a --- /dev/null +++ b/src/cmd/isulad-shim/linked_list.h @@ -0,0 +1,134 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: tanyifeng + * Create: 2018-11-08 + * Description: provide container linked list function definition + ******************************************************************************/ +#ifndef __LINKED_LIST_H +#define __LINKED_LIST_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct linked_list { + void *elem; + struct linked_list *next; + struct linked_list *prev; +}; + +/* Iterate through an linked list. */ +#define linked_list_for_each(__iterator, __list) \ + for ((__iterator) = (__list)->next; \ + (__iterator) != (__list); \ + (__iterator) = (__iterator)->next) + +/* Iterate safely through an linked list. */ +#define linked_list_for_each_safe(__iterator, __list, __next) \ + for ((__iterator) = (__list)->next, (__next) = (__iterator)->next; \ + (__iterator) != (__list); \ + (__iterator) = (__next), (__next) = (__next)->next) + +/* Initialize list. */ +static inline void linked_list_init(struct linked_list *list) +{ + list->elem = NULL; + list->next = list->prev = list; +} + +/* Add an element to a list. See linked_list_add() and linked_list_add_tail() for an + * idiom. */ +static inline void linked_list_add_elem(struct linked_list *list, void *elem) +{ + list->elem = elem; +} + +/* Retrieve first element of list. */ +static inline void *linked_list_first_elem(struct linked_list *list) +{ + return list->next->elem; +} + +/* Retrieve last element of list. */ +static inline void *linked_list_last_elem(struct linked_list *list) +{ + return list->prev->elem; +} + +/* Retrieve first node of list. */ +static inline void *linked_list_first_node(struct linked_list *list) +{ + return list->next; +} + +/* Determine if list is empty. */ +static inline int linked_list_empty(struct linked_list *list) +{ + return list == list->next; +} + +/* Workhorse to be called from linked_list_add() and linked_list_add_tail(). */ +static inline void __linked_list_add(struct linked_list *newlist, + struct linked_list *prev, + struct linked_list *next) +{ + next->prev = newlist; + newlist->next = next; + newlist->prev = prev; + prev->next = newlist; +} + +/* Idiom to add an element to the beginning of an linked list */ +static inline void linked_list_add(struct linked_list *head, + struct linked_list *list) +{ + __linked_list_add(list, head, head->next); +} + +/* Idiom to add an element to the end of an linked list */ +static inline void linked_list_add_tail(struct linked_list *head, + struct linked_list *list) +{ + __linked_list_add(list, head->prev, head); +} + +/* Idiom to free an linked list */ +static inline void linked_list_del(const struct linked_list *list) +{ + struct linked_list *next = NULL; + struct linked_list *prev = NULL; + + next = list->next; + prev = list->prev; + next->prev = prev; + prev->next = next; +} + +/* Return length of the list. */ +static inline size_t linked_list_len(struct linked_list *list) +{ + size_t i = 0; + struct linked_list *iter = NULL; + linked_list_for_each(iter, list) { + i++; + } + + return i; +} + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c index 68e99e53..0cccdf06 100644 --- a/src/cmd/isulad-shim/main.c +++ b/src/cmd/isulad-shim/main.c @@ -105,9 +105,9 @@ int main(int argc, char **argv) int ret = SHIM_ERR; int efd = -1; process_t *p = NULL; - pthread_t tid_accept; // execSync timeout uint64_t timeout = 0; + pthread_t tid_epoll; g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640); if (g_log_fd < 0) { @@ -154,18 +154,13 @@ int main(int argc, char **argv) } } - /* create main loop and start epoll for io copy */ - ret = process_io_init(p); + /* start epoll for io copy */ + ret = process_io_start(p, &tid_epoll); if (ret != SHIM_OK) { write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret); exit(EXIT_FAILURE); } - ret = open_io(p, &tid_accept); - if (ret != SHIM_OK) { - exit(EXIT_FAILURE); - } - ret = create_process(p); if (ret != SHIM_OK) { if (p->console_sock_path != NULL) { @@ -176,5 +171,5 @@ int main(int argc, char **argv) released_timeout_exit(); - return process_signal_handle_routine(p, tid_accept, timeout); + return process_signal_handle_routine(p, tid_epoll, timeout); } diff --git a/src/cmd/isulad-shim/mainloop.c b/src/cmd/isulad-shim/mainloop.c new file mode 100644 index 00000000..53c4d856 --- /dev/null +++ b/src/cmd/isulad-shim/mainloop.c @@ -0,0 +1,168 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: tanyifeng + * Create: 2018-11-08 + * Description: provide container mainloop functions + ******************************************************************************/ +#include "mainloop.h" +#include +#include +#include +#include +#include + +#include "common.h" + +struct epoll_loop_handler { + epoll_loop_callback_t cb; + int cbfd; + void *cbdata; +}; + +#define MAX_EVENTS 100 + +/* epoll loop */ +int epoll_loop(struct epoll_descr *descr, int t) +{ + int i; + int ret = 0; + struct epoll_loop_handler *epoll_handler = NULL; + struct epoll_event evs[MAX_EVENTS]; + + while (1) { + int ep_fds = epoll_wait(descr->fd, evs, MAX_EVENTS, t); + if (ep_fds < 0) { + if (errno == EINTR) { + continue; + } + ret = -1; + goto out; + } + + for (i = 0; i < ep_fds; i++) { + epoll_handler = (struct epoll_loop_handler *)(evs[i].data.ptr); + if (epoll_handler->cb(epoll_handler->cbfd, evs[i].events, epoll_handler->cbdata, descr) != + EPOLL_LOOP_HANDLE_CONTINUE) { + goto out; + } + } + + if (ep_fds == 0 && t != 0) { + if (descr->timeout_cb != NULL) { + descr->timeout_cb(descr->timeout_cbdata); + } + goto out; + } + + if (linked_list_empty(&descr->handler_list)) { + goto out; + } + } +out: + return ret; +} + +/* epoll loop add handler */ +int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data) +{ + struct epoll_event ev = { 0 }; + struct epoll_loop_handler *epoll_handler = NULL; + struct linked_list *node = NULL; + + // if fd == -1; dose not add handler for it. + if (fd < 0) { + return 0; + } + + epoll_handler = util_common_calloc_s(sizeof(*epoll_handler)); + if (epoll_handler == NULL) { + goto fail_out; + } + + epoll_handler->cbfd = fd; + epoll_handler->cb = callback; + epoll_handler->cbdata = data; + + ev.events = EPOLLIN; + ev.data.ptr = epoll_handler; + + if (epoll_ctl(descr->fd, EPOLL_CTL_ADD, fd, &ev) < 0) { + goto fail_out; + } + + node = util_common_calloc_s(sizeof(struct linked_list)); + if (node == NULL) { + goto fail_out; + } + + node->elem = epoll_handler; + linked_list_add(&descr->handler_list, node); + return 0; + +fail_out: + (void)epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, &ev); + free(epoll_handler); + return -1; +} + +/* epoll loop del handler */ +int epoll_loop_del_handler(struct epoll_descr *descr, int fd) +{ + struct epoll_loop_handler *epoll_handler = NULL; + struct linked_list *index = NULL; + + linked_list_for_each(index, &descr->handler_list) { + epoll_handler = index->elem; + + if (fd == epoll_handler->cbfd) { + if (epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, NULL)) { + goto fail_out; + } + + linked_list_del(index); + free(index->elem); + free(index); + return 0; + } + } + +fail_out: + return -1; +} + +/* epoll loop open */ +int epoll_loop_open(struct epoll_descr *descr) +{ + descr->fd = epoll_create1(EPOLL_CLOEXEC); + if (descr->fd < 0) { + return -1; + } + + linked_list_init(&(descr->handler_list)); + descr->timeout_cb = NULL; + descr->timeout_cbdata = NULL; + return 0; +} + +/* epoll loop close */ +int epoll_loop_close(struct epoll_descr *descr) +{ + struct linked_list *index = NULL; + struct linked_list *next = NULL; + + linked_list_for_each_safe(index, &(descr->handler_list), next) { + linked_list_del(index); + free(index->elem); + free(index); + } + + return close(descr->fd); +} diff --git a/src/cmd/isulad-shim/mainloop.h b/src/cmd/isulad-shim/mainloop.h new file mode 100644 index 00000000..7a4f1cfd --- /dev/null +++ b/src/cmd/isulad-shim/mainloop.h @@ -0,0 +1,53 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. + * iSulad licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: tanyifeng + * Create: 2018-11-08 + * Description: provide container mainloop definition + ******************************************************************************/ +#ifndef UTILS_CUTILS_MAINLOOP_H +#define UTILS_CUTILS_MAINLOOP_H + +#include +#include "linked_list.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*epoll_timeout_callback_t)(void *data); + +struct epoll_descr { + int fd; + struct linked_list handler_list; + epoll_timeout_callback_t timeout_cb; + void *timeout_cbdata; +}; + +#define EPOLL_LOOP_HANDLE_CONTINUE 0 +#define EPOLL_LOOP_HANDLE_CLOSE 1 + +typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data, struct epoll_descr *descr); + +extern int epoll_loop(struct epoll_descr *descr, int t); + +extern int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data); + +extern int epoll_loop_del_handler(struct epoll_descr *descr, int fd); + +extern int epoll_loop_open(struct epoll_descr *descr); + +extern int epoll_loop_close(struct epoll_descr *descr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c index a5e0bd39..c31f8af6 100644 --- a/src/cmd/isulad-shim/process.c +++ b/src/cmd/isulad-shim/process.c @@ -28,6 +28,7 @@ #include #include #include +#include #include // IWYU pragma: keep #include // IWYU pragma: keep #include @@ -37,6 +38,7 @@ #include "common.h" #include "terminal.h" +#include "mainloop.h" #define MAX_EVENTS 100 #define DEFAULT_IO_COPY_BUF (16 * 1024) @@ -78,7 +80,7 @@ static int receive_fd(int sock) u_char *pfd = NULL; int fd = -1; int cmsgsize = CMSG_LEN(sizeof(int)); - struct cmsghdr *cmptr = (struct cmsghdr *)calloc(1, cmsgsize); + struct cmsghdr *cmptr = (struct cmsghdr *)util_common_calloc_s(cmsgsize); if (cmptr == NULL) { return -1; } @@ -128,107 +130,6 @@ static bool check_fd(int fd) return true; } -static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to) -{ - int ret = SHIM_ERR; - - if (io_thd == NULL || io_thd->ioc == NULL) { - return SHIM_ERR; - } - - io_copy_t *ioc = io_thd->ioc; - - if (pthread_mutex_lock(&(ioc->mutex)) != 0) { - return SHIM_ERR; - } - /* add src fd */ - if (from != -1 && ioc->fd_from == -1) { - ioc->fd_from = from; - struct epoll_event ev; - ev.events = EPOLLIN; - ev.data.ptr = io_thd; - - ret = epoll_ctl(epfd, EPOLL_CTL_ADD, from, &ev); - if (ret != SHIM_OK) { - write_message(g_log_fd, ERR_MSG, "add fd %d to epoll loop failed:%d", from, SHIM_SYS_ERR(errno)); - pthread_mutex_unlock(&(ioc->mutex)); - return SHIM_ERR; - } - } - - /* add dest fd */ - if (to != -1) { - /* new fd_node_t for dest fd */ - fd_node_t *fn = (fd_node_t *)calloc(1, sizeof(fd_node_t)); - if (fn == NULL) { - pthread_mutex_unlock(&(ioc->mutex)); - return SHIM_ERR; - } - fn->fd = to; - fn->is_log = false; - if (io_thd->terminal != NULL && to == io_thd->terminal->fd) { - fn->is_log = true; - } - fn->next = NULL; - - if (ioc->fd_to == NULL) { - ioc->fd_to = fn; - } else { - fd_node_t *tmp = ioc->fd_to; - while (tmp->next != NULL) { - tmp = tmp->next; - } - tmp->next = fn; - } - } - pthread_mutex_unlock(&(ioc->mutex)); - - return SHIM_OK; -} - -static void remove_io_dispatch(io_thread_t *io_thd, int from, int to) -{ - if (io_thd == NULL || io_thd->ioc == NULL) { - return; - } - io_copy_t *ioc = io_thd->ioc; - - fd_node_t *tmp = NULL; - do { - /* remove src fd */ - if (from != -1 && from == ioc->fd_from) { - struct epoll_event ev; - ev.events = EPOLLIN; - ev.data.fd = ioc->fd_from; - (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); - } - - /* remove dest fd */ - if (ioc->fd_to == NULL) { - break; - } - if (ioc->fd_to->fd == to) { - /* remove the first fd node */ - tmp = ioc->fd_to; - ioc->fd_to = ioc->fd_to->next; - break; - } - fd_node_t *pre = ioc->fd_to; - tmp = ioc->fd_to->next; - while (tmp != NULL && tmp->fd != to) { - pre = tmp; - tmp = tmp->next; - } - if (tmp != NULL) { - pre->next = tmp->next; - } - } while (0); - if (tmp != NULL) { - free(tmp); - tmp = NULL; - } -} - static int get_exec_winsize(const char *buf, struct winsize *wsize) { char **array = NULL; @@ -261,347 +162,287 @@ out: return ret; } -static void *do_io_copy(void *data) +static int sync_exit_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) { - io_thread_t *io_thd = (io_thread_t *)data; - if (io_thd == NULL || io_thd->ioc == NULL) { - return NULL; - } - io_copy_t *ioc = io_thd->ioc; - char *buf = calloc(1, DEFAULT_IO_COPY_BUF + 1); - if (buf == NULL) { - return NULL; - } + return EPOLL_LOOP_HANDLE_CLOSE; +} - for (;;) { - memset(buf, 0, DEFAULT_IO_COPY_BUF); - (void)sem_wait(&(io_thd->sem_thd)); - if (io_thd->is_stdin && io_thd->shutdown) { - break; - } +static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) +{ + process_t *p = (process_t *)cbdata; + int r_count = 0; + int w_count = 0; + int *fd_to = NULL; - int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF); - if (r_count == -1) { - if (errno == EAGAIN || errno == EINTR) { - continue; - } - break; - } else if (r_count == 0) { - /* End of file. The remote has closed the connection */ - break; - } else if (ioc->id != EXEC_RESIZE) { - if (pthread_mutex_lock(&(ioc->mutex)) != 0) { - continue; - } + if (events & EPOLLHUP) { + return EPOLL_LOOP_HANDLE_CLOSE; + } - fd_node_t *fn = ioc->fd_to; - fd_node_t *next = NULL; - for (; fn != NULL; fn = next) { - next = fn->next; - if (fn->is_log) { - shim_write_container_log_file(io_thd->terminal, ioc->id, buf, r_count); - } else { - int w_count = write_nointr_in_total(fn->fd, buf, r_count); - if (w_count < 0) { - /* When any error occurs, remove the write fd */ - remove_io_dispatch(io_thd, -1, fn->fd); - } - } - } - pthread_mutex_unlock(&(ioc->mutex)); - } else { - if (pthread_mutex_lock(&(ioc->mutex)) != 0) { - continue; - } + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } - int resize_fd = ioc->fd_to->fd; - struct winsize wsize = { 0x00 }; - if (get_exec_winsize(buf, &wsize) < 0) { - break; - } - if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { - break; - } - pthread_mutex_unlock(&(ioc->mutex)); - } + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); - /* - In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe. - So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs. - */ - if (io_thd->shutdown && r_count <= 0) { - break; - } + r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + if (r_count <= 0) { + return EPOLL_LOOP_HANDLE_CLOSE; } - struct epoll_event ev; - ev.events = EPOLLIN; - ev.data.fd = ioc->fd_from; - (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev); - free(buf); + if (p->state->terminal) { + fd_to = &(p->recv_fd); + } else { + fd_to = &(p->shim_io->in); + } + w_count = write_nointr_in_total(*fd_to, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); + close(*fd_to); + *fd_to = -1; + } - return NULL; + return EPOLL_LOOP_HANDLE_CONTINUE; } -static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data) +static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) { - io_thread_t *thd = (io_thread_t *)data; - if (thd->ioc == NULL || fd != thd->ioc->fd_from) { - return; + process_t *p = (process_t *)cbdata; + int r_count = 0; + int w_count = 0; + + if (events & EPOLLHUP) { + return EPOLL_LOOP_HANDLE_CLOSE; } - if (event & EPOLLIN) { - (void)sem_post(&thd->sem_thd); - } else if (event & EPOLLHUP) { - thd->shutdown = true; - (void)sem_post(&thd->sem_thd); + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; } -} -static int create_io_copy_thread(process_t *p, int std_id) -{ - int ret = SHIM_ERR; - io_thread_t *io_thd = NULL; - io_copy_t *ioc = NULL; + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); - ioc = (io_copy_t *)calloc(1, sizeof(io_copy_t)); - if (ioc == NULL) { - goto failure; + if (p->block_read) { + r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + } else { + r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); } - ioc->id = std_id; - ioc->fd_from = -1; - ioc->fd_to = NULL; - if (pthread_mutex_init(&(ioc->mutex), NULL) != 0) { - goto failure; + if (r_count <= 0) { + return EPOLL_LOOP_HANDLE_CLOSE; } - io_thd = (io_thread_t *)calloc(1, sizeof(io_thread_t)); - if (io_thd == NULL) { - goto failure; + shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count); + + if (p->isulad_io->out == -1) { + return EPOLL_LOOP_HANDLE_CONTINUE; } - if (sem_init(&io_thd->sem_thd, 0, 0) != 0) { - write_message(g_log_fd, ERR_MSG, "sem init failed:%d", SHIM_SYS_ERR(errno)); - goto failure; + + w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); + close(p->isulad_io->out); + p->isulad_io->out = -1; } - io_thd->epfd = p->io_loop_fd; - io_thd->ioc = ioc; - io_thd->shutdown = false; - io_thd->is_stdin = std_id == STDID_IN ? true : false; - io_thd->terminal = std_id != STDID_IN ? p->terminal : NULL; - p->io_threads[std_id] = io_thd; + return EPOLL_LOOP_HANDLE_CONTINUE; +} - ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd); - if (ret != SHIM_OK) { - write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno)); - goto failure; +static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) +{ + process_t *p = (process_t *)cbdata; + int r_count = 0; + int w_count = 0; + + if (events & EPOLLHUP) { + return EPOLL_LOOP_HANDLE_CLOSE; } - ret = SHIM_OK; + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; + } - return ret; + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); -failure: - if (ioc != NULL) { - pthread_mutex_destroy(&(ioc->mutex)); - free(ioc); + if (p->block_read) { + r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + } else { + r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF); } - if (io_thd != NULL) { - free(io_thd); + if (r_count <= 0) { + return EPOLL_LOOP_HANDLE_CLOSE; } - return SHIM_ERR; -} - -static int start_io_copy_threads(process_t *p) -{ - int ret = SHIM_ERR; - int i; - - /* 4 threads for stdin, stdout, stderr and exec resize */ - for (i = 0; i < 4; i++) { - /* - * if the terminal is used, we do not need to active the io copy of stderr pipe, - * for stderr and stdout are mixed together - */ - if (i == STDID_ERR && p->state->terminal) { - continue; - } + shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count); - ret = create_io_copy_thread(p, i); - if (ret != SHIM_OK) { - return SHIM_ERR; - } + if (p->isulad_io->err == -1) { + return EPOLL_LOOP_HANDLE_CONTINUE; } - return SHIM_OK; -} -static void destroy_io_thread(process_t *p, int std_id) -{ - io_thread_t *io_thd = p->io_threads[std_id]; - if (io_thd == NULL) { - return; + w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count); + if (w_count < 0) { + /* When any error occurs, set the write fd -1 */ + write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); + close(p->isulad_io->err); + p->isulad_io->err = -1; } - io_thd->shutdown = true; - (void)sem_post(&io_thd->sem_thd); - pthread_join(io_thd->tid, NULL); - if (io_thd->ioc != NULL) { - free(io_thd->ioc); - } - free(io_thd); - p->io_threads[std_id] = NULL; + return EPOLL_LOOP_HANDLE_CONTINUE; } -/* - std_id: channel type - isulad_stdio: one side of the isulad fifo file - fd: one side of the shim io pipe - --------------------------------------------------------------- - | CHANNEL | iSulad Fifo Side | Flow Direction | fd | - --------------------------------------------------------------- - | STDIN | READ | --> | WRITE | - --------------------------------------------------------------- - | STDOUT | WRITE | <-- | READ | - --------------------------------------------------------------- - | STDERR | WRITE | <-- | READ | - --------------------------------------------------------------- - | RESIZE | READ | --> | WRITE | - --------------------------------------------------------------- -*/ -static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd) +static int resize_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) { - mode_t mode; - int fd_isulad = -1; - int *fd_from = NULL; - int *fd_to = NULL; + process_t *p = (process_t *)cbdata; + int r_count = 0; + int resize_fd = -1; - if (std_id == STDID_IN || std_id == EXEC_RESIZE) { - mode = O_RDONLY; - fd_from = &fd_isulad; - fd_to = &fd; - } else { - mode = O_WRONLY; - fd_from = &fd; - fd_to = &fd_isulad; + if (events & EPOLLHUP) { + return EPOLL_LOOP_HANDLE_CLOSE; } - if (isulad_stdio != NULL && file_exists(isulad_stdio)) { - fd_isulad = open_fifo_noblock(isulad_stdio, mode); - if (fd_isulad < 0) { - return SHIM_ERR; - } - /* open dummy fd to avoid resize epoll hub */ - if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { - return SHIM_ERR; - } + if (!(events & EPOLLIN)) { + return EPOLL_LOOP_HANDLE_CONTINUE; } - if (*fd_from != -1) { - if (std_id != STDID_IN && std_id != EXEC_RESIZE && p->io_threads[std_id]->terminal != NULL) { - (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd); - } - return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to); + (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF); + r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); + if (r_count <= 0) { + return EPOLL_LOOP_HANDLE_CLOSE; } - /* if no I/O source is available, the I/O thread nead to be destroyed */ - destroy_io_thread(p, std_id); + resize_fd = p->recv_fd; + struct winsize wsize = { 0x00 }; + if (get_exec_winsize(p->buf, &wsize) < 0) { + return EPOLL_LOOP_HANDLE_CLOSE; + } + if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) { + return EPOLL_LOOP_HANDLE_CLOSE; + } - return SHIM_OK; + return EPOLL_LOOP_HANDLE_CONTINUE; } -static void *task_console_accept(void *data) +static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) { + process_t *p = (process_t *)cbdata; int conn_fd = -1; - int recv_fd = -1; int ret = SHIM_ERR; - console_accept_t *ac = (console_accept_t *)data; - conn_fd = accept(ac->listen_fd, NULL, NULL); + conn_fd = accept(p->listen_fd, NULL, NULL); if (conn_fd < 0) { - write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno)); + write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); goto out; } - recv_fd = receive_fd(conn_fd); - if (check_fd(recv_fd) != true) { + p->recv_fd = receive_fd(conn_fd); + if (check_fd(p->recv_fd) != true) { write_message(g_log_fd, ERR_MSG, "check console fd failed"); goto out; } /* do console io copy */ - /* p.state.stdin---->runtime.console */ - ret = connect_to_isulad(ac->p, STDID_IN, ac->p->state->isulad_stdin, recv_fd); + // p->isulad_io->in ----> p->recv_fd + ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); goto out; } - - /* p.state.stdout<------runtime.console */ - ret = connect_to_isulad(ac->p, STDID_OUT, ac->p->state->isulad_stdout, recv_fd); + // p->recv_fd ----> p->isulad_io->out + ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p); if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); goto out; } - - /* p.state.resize_fifo------>runtime.console */ - ret = connect_to_isulad(ac->p, EXEC_RESIZE, ac->p->state->resize_fifo, recv_fd); + // p->isulad_io->resize ----> p->recv_fd + ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p); if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); goto out; } out: /* release listen socket at the first time */ - close_fd(&ac->listen_fd); - if (ac->p->console_sock_path != NULL) { - (void)unlink(ac->p->console_sock_path); - free(ac->p->console_sock_path); - ac->p->console_sock_path = NULL; - } - free(ac); - if (ret != SHIM_OK) { - /* - * When an error occurs during the receiving of the fd , the process - * exits directly. The files created in the working directory will be - * deleted by its parent process isulad - */ - exit(EXIT_FAILURE); + close_fd(&p->listen_fd); + if (p->console_sock_path != NULL) { + (void)unlink(p->console_sock_path); + free(p->console_sock_path); + p->console_sock_path = NULL; } - return NULL; + return ret; } -static void *io_epoll_loop(void *data) +static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) { - process_t *p = (process_t *)data; - int wait_fds = 0; - struct epoll_event evs[MAX_EVENTS]; - int i; + int i, j; - if ((pthread_detach(pthread_self())) != 0) { - write_message(g_log_fd, ERR_MSG, "detach thread failed"); - return NULL; + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + int ret = fchown(stdio_fd[i][j], uid, gid); + if (ret != SHIM_OK) { + return SHIM_ERR; + } + } } + return SHIM_OK; +} - p->io_loop_fd = epoll_create1(EPOLL_CLOEXEC); - if (p->io_loop_fd < 0) { - write_message(g_log_fd, ERR_MSG, "epoll create failed:%d", SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); - } - (void)sem_post(&p->sem_mainloop); +static void stdio_release(int (*stdio_fd)[2]) +{ + int i, j; - for (;;) { - wait_fds = epoll_wait(p->io_loop_fd, evs, MAX_EVENTS, -1); - if (wait_fds < 0) { - if (errno == EINTR) { - continue; + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + if (stdio_fd[i][j] > 0) { + close(stdio_fd[i][j]); } - _exit(EXIT_FAILURE); } + } +} - for (i = 0; i < wait_fds; i++) { - io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr; - sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io); - } +static stdio_t *initialize_io(process_t *p) +{ + int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; + + stdio_t *stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); + p->stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); + if (p->stdio == NULL || stdio == NULL) { + goto failure; + } + + /* don't open resize pipe */ + if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || + (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { + write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); + goto failure; } + + p->stdio->in = stdio_fd[0][0]; // r + stdio->in = stdio_fd[0][1]; // w + p->stdio->out = stdio_fd[1][1]; // w + stdio->out = stdio_fd[1][0]; // r + p->stdio->err = stdio_fd[2][1]; // w + stdio->err = stdio_fd[2][0]; // r + p->stdio->resize = stdio_fd[3][0]; // r + stdio->resize = stdio_fd[3][1]; // w + + if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { + goto failure; + } + + return stdio; + +failure: + if (stdio != NULL) { + free(stdio); + stdio = NULL; + } + if (p->stdio != NULL) { + free(p->stdio); + p->stdio = NULL; + } + stdio_release(stdio_fd); + + return NULL; } static int new_temp_console_path(process_t *p) @@ -614,7 +455,7 @@ static int new_temp_console_path(process_t *p) if (ret != SHIM_OK) { return SHIM_ERR; } - p->console_sock_path = (char *)calloc(1, MAX_CONSOLE_SOCK_LEN + 1); + p->console_sock_path = (char *)util_common_calloc_s(MAX_CONSOLE_SOCK_LEN + 1); if (p->console_sock_path == NULL) { return SHIM_ERR; } @@ -628,12 +469,11 @@ static int new_temp_console_path(process_t *p) return SHIM_OK; } -static int console_init(process_t *p, pthread_t *tid_accept) +static int console_init(process_t *p, struct epoll_descr *descr) { int ret = SHIM_ERR; int fd = -1; struct sockaddr_un addr; - console_accept_t *ac = NULL; fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) { @@ -657,145 +497,182 @@ static int console_init(process_t *p, pthread_t *tid_accept) goto failure; } - ac = (console_accept_t *)calloc(1, sizeof(console_accept_t)); - if (ac == NULL) { - goto failure; - } - ac->p = p; - ac->listen_fd = fd; + p->listen_fd = fd; - ret = pthread_create(tid_accept, NULL, task_console_accept, ac); + ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p); if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); goto failure; } return SHIM_OK; failure: close_fd(&fd); - if (ac != NULL) { - free(ac); - ac = NULL; - } (void)unlink(p->console_sock_path); return SHIM_ERR; } -static int stdio_chown(int (*stdio_fd)[2], int uid, int gid) +static int open_terminal_io(process_t *p, struct epoll_descr *descr) { - int i, j; + int ret = SHIM_ERR; - for (i = 0; i < 3; i++) { - for (j = 0; j < 2; j++) { - int ret = fchown(stdio_fd[i][j], uid, gid); - if (ret != SHIM_OK) { - return SHIM_ERR; - } - } + ret = new_temp_console_path(p); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); + return SHIM_ERR; } - return SHIM_OK; -} - -static void stdio_release(int (*stdio_fd)[2]) -{ - int i, j; - for (i = 0; i < 3; i++) { - for (j = 0; j < 2; j++) { - if (stdio_fd[i][j] > 0) { - close(stdio_fd[i][j]); - } - } - } + /* begin listen from p->console_sock_path */ + return console_init(p, descr); } -static stdio_t *initialize_io(process_t *p) +static int open_generic_io(process_t *p, struct epoll_descr *descr) { - int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } }; - - stdio_t *stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); - p->stdio = (stdio_t *)calloc(1, sizeof(stdio_t)); - if (p->stdio == NULL || stdio == NULL) { - goto failure; - } + int ret = SHIM_ERR; - /* don't open resize pipe */ - if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) || - (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) { - write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); - goto failure; + // io: in: w out/err: r + stdio_t *io = initialize_io(p); + if (io == NULL) { + return SHIM_ERR; } + p->shim_io = io; - p->stdio->in = stdio_fd[0][0]; // r - stdio->in = stdio_fd[0][1]; // w - p->stdio->out = stdio_fd[1][1]; // w - stdio->out = stdio_fd[1][0]; // r - p->stdio->err = stdio_fd[2][1]; // w - stdio->err = stdio_fd[2][0]; // r - p->stdio->resize = stdio_fd[3][0]; // r - stdio->resize = stdio_fd[3][1]; // w - - if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) { - goto failure; + // p->isulad_io->in ----> p->shim_io->in + ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); + return SHIM_ERR; } - - return stdio; - -failure: - if (stdio != NULL) { - free(stdio); - stdio = NULL; + // p->shim_io->out ----> p->isulad_io->out + ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); + return SHIM_ERR; } - if (p->stdio != NULL) { - free(p->stdio); - p->stdio = NULL; + // p->shim_io->err ----> p->isulad_io->err + ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); + return SHIM_ERR; } - stdio_release(stdio_fd); - return NULL; + return SHIM_OK; } -static int open_terminal_io(process_t *p, pthread_t *tid_accept) +static int set_non_block(int fd) { + int flag = -1; int ret = SHIM_ERR; - ret = new_temp_console_path(p); - if (ret != SHIM_OK) { - write_message(g_log_fd, ERR_MSG, "get temp console sock path failed"); + flag = fcntl(fd, F_GETFL, 0); + if (flag < 0) { + return SHIM_ERR; + } + + ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK); + if (ret != 0) { return SHIM_ERR; } - /* begin listen and accept fd from p->console_sock_path */ - return console_init(p, tid_accept); + return SHIM_OK; } -static int open_generic_io(process_t *p) +/* + std_id: channel type + isulad_stdio: one side of the isulad fifo file + fd: one side of the shim io pipe + --------------------------------------------------------------- + | CHANNEL | iSulad Fifo Side | Flow Direction | fd | + --------------------------------------------------------------- + | STDIN | READ | --> | WRITE | + --------------------------------------------------------------- + | STDOUT | WRITE | <-- | READ | + --------------------------------------------------------------- + | STDERR | WRITE | <-- | READ | + --------------------------------------------------------------- + | RESIZE | READ | --> | WRITE | + --------------------------------------------------------------- +*/ +static void *io_epoll_loop(void *data) { - int ret = SHIM_ERR; + int ret = 0; + int fd_out = -1; + int fd_err = -1; + process_t *p = (process_t *)data; + struct epoll_descr descr; - // io: in: w out/err: r - stdio_t *io = initialize_io(p); - if (io == NULL) { - return SHIM_ERR; + ret = epoll_loop_open(&descr); + if (ret != 0) { + write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); } - p->shim_io = io; - /* stdin */ - ret = connect_to_isulad(p, STDID_IN, p->state->isulad_stdin, io->in); - if (ret != SHIM_OK) { - return SHIM_ERR; + + // sync fd: epoll loop will exit when recive sync fd event. + ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p); + if (ret != 0) { + write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); } - /* stdout */ - ret = connect_to_isulad(p, STDID_OUT, p->state->isulad_stdout, io->out); - if (ret != SHIM_OK) { - return SHIM_ERR; + + if (p->state->terminal) { + ret = open_terminal_io(p, &descr); + } else { + ret = open_generic_io(p, &descr); } - /* stderr */ - ret = connect_to_isulad(p, STDID_ERR, p->state->isulad_stderr, io->err); if (ret != SHIM_OK) { - return SHIM_ERR; + write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); } - return SHIM_OK; + (void)sem_post(&p->sem_mainloop); + + ret = epoll_loop(&descr, -1); + if (ret != 0) { + write_message(g_log_fd, ERR_MSG, "epoll loop failed"); + exit(EXIT_FAILURE); + } + + // in order to avoid data loss, set fd non-block and read it + p->block_read = false; + if (p->state->terminal) { + fd_out = p->recv_fd; + } else { + fd_out = p->shim_io->out; + fd_err = p->shim_io->err; + } + + if (fd_out > 0) { + ret = set_non_block(fd_out); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); + } + + for (;;) { + ret = stdout_cb(fd_out, EPOLLIN, p, &descr); + if (ret == EPOLL_LOOP_HANDLE_CLOSE) { + break; + } + } + } + + if (fd_err > 0) { + ret = set_non_block(fd_err); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno)); + exit(EXIT_FAILURE); + } + + for (;;) { + ret = stderr_cb(fd_err, EPOLLIN, p, &descr); + if (ret == EPOLL_LOOP_HANDLE_CLOSE) { + break; + } + } + } + + return NULL; } static void adapt_for_isulad_stdin(process_t *p) @@ -813,7 +690,7 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s { log_terminal *log_term = NULL; - log_term = calloc(1, sizeof(log_terminal)); + log_term = util_common_calloc_s(sizeof(log_terminal)); if (log_term == NULL) { write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal"); goto clean_out; @@ -858,11 +735,90 @@ clean_out: return SHIM_ERR; } +static int open_isulad_fd(int std_id, const char *isulad_stdio, int *fd) +{ + mode_t mode = O_WRONLY; + + if (std_id == STDID_IN || std_id == EXEC_RESIZE) { + mode = O_RDONLY; + } + + if (isulad_stdio != NULL && file_exists(isulad_stdio)) { + *(fd) = open_fifo_noblock(isulad_stdio, mode); + if (*(fd) < 0) { + return -1; + } + /* open dummy fd to avoid resize epoll hub */ + if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) { + return -1; + } + } + + return 0; +} + + +static int init_isulad_stdio(process_t *p) +{ + int ret = SHIM_OK; + p->isulad_io = (stdio_t *)util_common_calloc_s(sizeof(stdio_t)); + if (p->isulad_io == NULL) { + return SHIM_ERR; + } + + p->isulad_io->in = -1; + p->isulad_io->out = -1; + p->isulad_io->err = -1; + p->isulad_io->resize = -1; + + ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin); + goto failure; + } + + ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout); + goto failure; + } + + ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr); + goto failure; + } + + ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize); + if (ret != SHIM_OK) { + write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo); + goto failure; + } + return SHIM_OK; +failure: + if (p->isulad_io != NULL) { + free(p->isulad_io); + p->isulad_io = NULL; + } + if (p->isulad_io->in > 0) { + close(p->isulad_io->in); + } + if (p->isulad_io->out > 0) { + close(p->isulad_io->out); + } + if (p->isulad_io->err > 0) { + close(p->isulad_io->err); + } + if (p->isulad_io->resize > 0) { + close(p->isulad_io->resize); + } + return SHIM_ERR; +} + process_t *new_process(char *id, char *bundle, char *runtime) { shim_client_process_state *p_state; process_t *p = NULL; - int i; int ret; p_state = load_process(); @@ -870,7 +826,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) return NULL; } - p = (process_t *)calloc(1, sizeof(process_t)); + p = (process_t *)util_common_calloc_s(sizeof(process_t)); if (p == NULL) { return NULL; } @@ -889,16 +845,31 @@ process_t *new_process(char *id, char *bundle, char *runtime) p->bundle = bundle; p->runtime = runtime; p->state = p_state; - + p->block_read = true; p->console_sock_path = NULL; p->exit_fd = -1; p->io_loop_fd = -1; p->ctr_pid = -1; + p->listen_fd = -1; + p->recv_fd = -1; p->stdio = NULL; p->shim_io = NULL; + p->isulad_io = NULL; - for (i = 0; i < 3; i++) { - p->io_threads[i] = NULL; + p->sync_fd = eventfd(0, EFD_CLOEXEC); + if (p->sync_fd < 0) { + write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno)); + goto failure; + } + + ret = init_isulad_stdio(p); + if (ret != SHIM_OK) { + goto failure; + } + + p->buf = util_common_calloc_s(DEFAULT_IO_COPY_BUF + 1); + if (p->buf == NULL) { + goto failure; } return p; @@ -909,28 +880,11 @@ failure: return NULL; } -int open_io(process_t *p, pthread_t *tid_accept) +int process_io_start(process_t *p, pthread_t *tid_epoll) { int ret = SHIM_ERR; - ret = start_io_copy_threads(p); - if (ret != SHIM_OK) { - return SHIM_ERR; - } - - if (p->state->terminal) { - return open_terminal_io(p, tid_accept); - } - - return open_generic_io(p); -} - -int process_io_init(process_t *p) -{ - int ret = SHIM_ERR; - - pthread_t tid_loop; - ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p); + ret = pthread_create(tid_epoll, NULL, io_epoll_loop, p); if (ret != SHIM_OK) { return SHIM_SYS_ERR(errno); } @@ -1091,8 +1045,8 @@ static void exec_runtime_process(process_t *p, int exec_fd) } char *cwd = getcwd(NULL, 0); - char *log_path = (char *)calloc(1, PATH_MAX); - char *pid_path = (char *)calloc(1, PATH_MAX); + char *log_path = (char *)util_common_calloc_s(PATH_MAX); + char *pid_path = (char *)util_common_calloc_s(PATH_MAX); if (cwd == NULL || log_path == NULL || pid_path == NULL) { (void)dprintf(exec_fd, "memory error: %s", strerror(errno)); _exit(EXIT_FAILURE); @@ -1294,13 +1248,11 @@ static int wait_container_process_with_timeout(process_t *p, const unsigned int } -int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout) +int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout) { - int i; int nret = 0; int ret = 0; int status = 0; - struct timespec ts; ret = wait_container_process_with_timeout(p, timeout, &status); if (ret == SHIM_ERR_TIMEOUT) { @@ -1324,27 +1276,20 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons if (p->exit_fd > 0) { (void)write_nointr(p->exit_fd, &status, sizeof(int)); } - // wait for task_console_accept thread termination. In order to make sure that - // the io_copy connection is established and io_thread is not used by multiple threads. - if (p->state->terminal) { - if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { - write_message(g_log_fd, ERR_MSG, "Failed to get realtime"); - nret = pthread_join(tid_accept, NULL); - } else { - // Set the maximum waiting time to 60s to prevent stuck. - ts.tv_sec += 60; - nret = pthread_timedjoin_np(tid_accept, NULL, &ts); - } - if (nret != 0) { - write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread"); + if (p->sync_fd > 0) { + if (eventfd_write(p->sync_fd, 1)) { + write_message(g_log_fd, ERR_MSG, "Failed to write sync fd"); } } - for (i = 0; i < 3; i++) { - destroy_io_thread(p, i); + nret = pthread_join(tid_epoll, NULL); + if (nret != 0) { + write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread"); } + close(p->sync_fd); + if (!p->state->exec) { // if log did not contain "/n", print remaind container log when exit isulad-shim shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0); diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h index 7e3259e8..4cb6462c 100644 --- a/src/cmd/isulad-shim/process.h +++ b/src/cmd/isulad-shim/process.h @@ -40,64 +40,37 @@ typedef struct { int resize; } stdio_t; -typedef struct fd_node { - int fd; - bool is_log; - struct fd_node *next; -} fd_node_t; - -typedef struct { - int fd_from; - fd_node_t *fd_to; - int id;// 0,1,2,3 - pthread_mutex_t mutex; -} io_copy_t; - -typedef struct { - int epfd; - pthread_t tid; - pthread_attr_t attr; - sem_t sem_thd; - io_copy_t *ioc; - bool shutdown; - bool is_stdin; - log_terminal *terminal;// just used by stdout and stderr -} io_thread_t; - typedef struct process { char *id; char *bundle; char *runtime; - char *console_sock_path;// pty socket path + char *console_sock_path; // pty socket path int io_loop_fd; int exit_fd; int ctr_pid; + int sync_fd; + int listen_fd; + int recv_fd; + bool block_read; log_terminal *terminal; - stdio_t *stdio;// shim to on runtime side, in:r out/err: w + stdio_t *stdio; // shim to on runtime side, in:r out/err: w stdio_t *shim_io; // shim io on isulad side, in: w out/err: r - io_thread_t *io_threads[4];// stdin,stdout,stderr,exec_resize + stdio_t *isulad_io; // isulad io, in:r out/err: w shim_client_process_state *state; sem_t sem_mainloop; + char *buf; } process_t; -typedef struct { - int listen_fd; - process_t *p; -} console_accept_t; - typedef struct { int pid; int status; } process_exit_t; - - process_t* new_process(char *id, char *bundle, char *runtime); -int open_io(process_t *p, pthread_t *tid_accept); -int process_io_init(process_t *p); +int process_io_start(process_t *p, pthread_t *tid_epoll); int create_process(process_t *p); -int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout); +int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout); #ifdef __cplusplus } diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt index e5c1cd6e..122538ff 100644 --- a/test/cmd/isulad-shim/CMakeLists.txt +++ b/test/cmd/isulad-shim/CMakeLists.txt @@ -6,6 +6,7 @@ add_executable(${EXE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/mainloop.c isulad-shim_ut.cc) target_include_directories(${EXE} PUBLIC diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc index 34ecd452..8c116ac8 100644 --- a/test/cmd/isulad-shim/isulad-shim_ut.cc +++ b/test/cmd/isulad-shim/isulad-shim_ut.cc @@ -11,6 +11,7 @@ #include #include +#include "mainloop.h" #include "process.h" #include "common.h" -- 2.25.1