From ec5736c76acc80b6099c82cd21fdad5b8da79b73 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 19 Dec 2023 19:47:18 +0800 Subject: [PATCH 65/65] refactor isulad-shim log Signed-off-by: zhongtao --- src/cmd/isulad-shim/common.c | 179 ++++++++------- src/cmd/isulad-shim/common.h | 33 ++- src/cmd/isulad-shim/main.c | 60 ++--- src/cmd/isulad-shim/process.c | 192 +++++++++------- .../modules/runtime/isula/isula_rt_ops.c | 212 +++++++++++++----- 5 files changed, 425 insertions(+), 251 deletions(-) diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c index 3cc7d2a7..d96b6c65 100644 --- a/src/cmd/isulad-shim/common.c +++ b/src/cmd/isulad-shim/common.c @@ -31,34 +31,124 @@ #include #include +#include +#include +#include -int g_log_fd = -1; -int g_attach_log_fd = -1; +char *g_shim_errmsg = NULL; +bool log_to_stderr = false; -int init_shim_log(void) +int isulad_shim_log_init(const char *file, const char *priority) { - g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); - if (g_log_fd < 0) { - return SHIM_ERR; + __isula_auto_free char *full_path = NULL; + const char *pre_name = "fifo:"; + size_t pre_len = 0; + struct isula_libutils_log_config lconf = { 0 }; + + if (file == NULL || priority == NULL || strncmp(file, pre_name, pre_len) != 0) { + goto err_out; } - return SHIM_OK; + + pre_len = strlen(pre_name); + lconf.name = "engine"; + /* File has prefix "fifo:", */ + full_path = isula_strdup_s(file + pre_len); + lconf.file = full_path; + lconf.driver = "fifo"; + lconf.priority = priority; + if (isula_libutils_log_enable(&lconf) != 0) { + goto err_out; + } + + return 0; +err_out: + // because shim log init error, print error msg to stderr. + // isulad can obtain the reason why shim exits. + dprintf(STDERR_FILENO, "failed to init shim log"); + return -1; } -int init_attach_log(void) +void shim_set_error_message(const char *format, ...) { - g_attach_log_fd = open_no_inherit(ATTACH_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); - if (g_attach_log_fd < 0) { - return SHIM_ERR; + int ret = 0; + char errbuf[BUFSIZ + 1] = { 0 }; + va_list argp; + + if (format == NULL) { + return; } - return SHIM_OK; + va_start(argp, format); + + ret = vsnprintf(errbuf, BUFSIZ, format, argp); + va_end(argp); + if (ret < 0) { + return; + } + + SHIM_CLEAR_ERRMSG(); + g_shim_errmsg = isula_strdup_s(errbuf); +} + +void shim_try_set_error_message(const char *format, ...) +{ + int ret = 0; + + if (g_shim_errmsg != NULL) { + return; + } + char errbuf[BUFSIZ + 1] = { 0 }; + + va_list argp; + va_start(argp, format); + + ret = vsnprintf(errbuf, BUFSIZ, format, argp); + va_end(argp); + if (ret < 0) { + return; + } + + g_shim_errmsg = isula_strdup_s(errbuf); +} + +void shim_append_error_message(const char *format, ...) +{ + int ret = 0; + char errbuf[BUFSIZ + 1] = { 0 }; + char *result = NULL; + + va_list argp; + va_start(argp, format); + + ret = vsnprintf(errbuf, BUFSIZ, format, argp); + va_end(argp); + if (ret < 0) { + return; + } + result = isula_string_append(g_shim_errmsg, errbuf); + if (result == NULL) { + return; + } + if (g_shim_errmsg != NULL) { + free(g_shim_errmsg); + } + g_shim_errmsg = result; +} + +void error_exit(int exit_code) +{ + if (log_to_stderr && g_shim_errmsg != NULL) { + dprintf(STDERR_FILENO, "shim-error: %s\n", g_shim_errmsg); + } + exit(exit_code); } void signal_routine(int sig) { switch (sig) { case SIGALRM: - write_message(ERR_MSG, "runtime timeout"); - exit(EXIT_FAILURE); + ERROR("runtime timeout"); + shim_set_error_message("runtime timeout"); + error_exit(EXIT_FAILURE); default: break; } @@ -172,67 +262,6 @@ int generate_random_str(char *id, size_t len) return SHIM_OK; } -#define MAX_MSG_JSON_TEMPLATE 32 -#define MAX_MESSAGE_CONTENT_LEN 128 -#define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN) - -static void format_log_msg(const char *level, const char *buf, char *msg, int max_message_len) -{ - time_t current_time = time(NULL); - struct tm *local_time = localtime(¤t_time); - char time_str[20]; - - strftime(time_str, sizeof(time_str), "%Y-%m-%d %H:%M:%S", local_time); - - (void)snprintf(msg, max_message_len - 1, "{\"time\": \"%s\", \"level\": \"%s\", \"msg\": \"%s\"}\n", time_str, level, - buf); -} - -void write_message(const char *level, const char *fmt, ...) -{ - if (g_log_fd < 0) { - return; - } - - char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; - char msg[MAX_MESSAGE_LEN] = { 0 }; - int nwrite = -1; - - va_list arg_list; - va_start(arg_list, fmt); - nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); - va_end(arg_list); - if (nwrite < 0) { - return; - } - - format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); - - (void)isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); -} - -void write_attach_message(const char *level, const char *fmt, ...) -{ - char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; - char msg[MAX_MESSAGE_LEN] = { 0 }; - int nwrite = -1; - - if (g_attach_log_fd < 0) { - return; - } - va_list arg_list; - va_start(arg_list, fmt); - nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); - va_end(arg_list); - if (nwrite < 0) { - return; - } - - format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); - - (void)isula_file_total_write_nointr(g_attach_log_fd, msg, strlen(msg)); -} - /* note: This function can only read small text file. */ char *read_text_file(const char *path) { diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h index c4f86d24..9ac11b61 100644 --- a/src/cmd/isulad-shim/common.h +++ b/src/cmd/isulad-shim/common.h @@ -55,7 +55,8 @@ extern "C" { #define MAX_RUNTIME_ARGS 100 #define SHIM_BINARY "isulad-shim" -#define SHIM_LOG_NAME "shim-log.json" +#define DEFAULT_ISULAD_SHIIM_LOG_PATH "fifo:/var/run/isulad/isulad_log_gather_fifo" +#define DEFAULT_ISULAD_SHIIM_LOG_LEVEL "ERROR" #define CONTAINER_ACTION_REBOOT 129 #define CONTAINER_ACTION_SHUTDOWN 130 @@ -72,9 +73,7 @@ extern "C" { #define SOCKET_DIRECTORY_MODE 0600 #define ATTACH_FIFOPATH_MODE 0600 -int init_shim_log(void); - -int init_attach_log(void); +int isulad_shim_log_init(const char *file, const char *priority); void signal_routine(int sig); @@ -114,13 +113,31 @@ struct shim_fifos_fd { int err_fd; }; -char *read_text_file(const char *path); +/* record the isulad-shim errmsg */ +extern char *g_shim_errmsg; -int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); +extern bool log_to_stderr; + +/* clear the g_shim_errmsg */ +#define SHIM_CLEAR_ERRMSG() \ + do { \ + if (g_shim_errmsg != NULL) { \ + free(g_shim_errmsg); \ + g_shim_errmsg = NULL; \ + } \ + } while (0) + +void shim_set_error_message(const char *format, ...); -void write_message(const char *level, const char *fmt, ...); +void shim_try_set_error_message(const char *format, ...); -void write_attach_message(const char *level, const char *fmt, ...); +void shim_append_error_message(const char *format, ...); + +void error_exit(int exit_code); + +char *read_text_file(const char *path); + +int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); int generate_random_str(char *id, size_t len); diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c index deb07271..a12e459f 100644 --- a/src/cmd/isulad-shim/main.c +++ b/src/cmd/isulad-shim/main.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "common.h" #include "process.h" @@ -91,17 +92,25 @@ int main(int argc, char **argv) char *bundle = NULL; char *rt_name = NULL; char *log_level = NULL; + char *engine_log_path = NULL; int ret = SHIM_ERR; process_t *p = NULL; // execSync timeout uint64_t timeout = 0; pthread_t tid_epoll; - ret = init_shim_log(); + engine_log_path = getenv("ISULAD_SHIIM_LOG_PATH"); + if (engine_log_path == NULL) { + engine_log_path = DEFAULT_ISULAD_SHIIM_LOG_PATH; + } + + log_level = getenv("ISULAD_SHIIM_LOG_LEVEL"); + if (log_level == NULL) { + log_level = DEFAULT_ISULAD_SHIIM_LOG_LEVEL; + } + + ret = isulad_shim_log_init(engine_log_path, log_level); if (ret != SHIM_OK) { - // because shim log init error, print error msg to stderr. - // isulad can obtain the reason why shim exits. - dprintf(STDERR_FILENO, "failed to init shim log"); _exit(EXIT_FAILURE); } @@ -113,22 +122,26 @@ int main(int argc, char **argv) ret = set_subreaper(); if (ret != SHIM_OK) { - write_message(ERR_MSG, "set subreaper failed:%d", ret); - exit(EXIT_FAILURE); + ERROR("set subreaper failed:%d", ret); + error_exit(EXIT_FAILURE); } ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level, &timeout); if (ret != SHIM_OK) { - write_message(ERR_MSG, "parse args failed:%d", ret); - exit(EXIT_FAILURE); + ERROR("parse args failed:%d", ret); + error_exit(EXIT_FAILURE); } p = new_process(container_id, bundle, rt_name); if (p == NULL) { - write_message(ERR_MSG, "new process failed"); - exit(EXIT_FAILURE); + ERROR("new process failed"); + error_exit(EXIT_FAILURE); } + // If isulad-shim is a child process of the isulad process, + // print the log to stderr so that isulad can obtain the exit information of isulad-shim. + log_to_stderr = (p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || p->state->isulad_stderr != NULL); + /* * Open exit pipe * The exit pipe exists only when the container is started, @@ -138,8 +151,9 @@ int main(int argc, char **argv) if (p->state->exit_fifo != NULL) { int efd = open_no_inherit("exit_fifo", O_WRONLY, -1); if (efd < 0) { - write_message(ERR_MSG, "open exit pipe failed:%d", SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); + ERROR("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } p->exit_fd = efd; } @@ -148,22 +162,18 @@ int main(int argc, char **argv) if (p->state->attach_socket != NULL) { ret = prepare_attach_socket(p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "failed to prepare attach socket:%d", ret); - exit(EXIT_FAILURE); - } - - ret = init_attach_log(); - if (ret != SHIM_OK) { - write_message(ERR_MSG, "failed to init shim attach log"); - exit(EXIT_FAILURE); + ERROR("failed to prepare attach socket:%d", ret); + shim_append_error_message("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } } /* start epoll for io copy */ ret = process_io_start(p, &tid_epoll); if (ret != SHIM_OK) { - write_message(ERR_MSG, "process io init failed:%d", ret); - exit(EXIT_FAILURE); + ERROR("process io init failed:%d", ret); + shim_set_error_message("process io init failed:%d", ret); + error_exit(EXIT_FAILURE); } ret = create_process(p); @@ -171,17 +181,17 @@ int main(int argc, char **argv) if (p->console_sock_path != NULL) { (void)unlink(p->console_sock_path); } - exit(EXIT_FAILURE); + error_exit(EXIT_FAILURE); } released_timeout_exit(); ret = process_signal_handle_routine(p, tid_epoll, timeout); if (ret == SHIM_ERR) { - exit(EXIT_FAILURE); + error_exit(EXIT_FAILURE); } if (ret == SHIM_ERR_TIMEOUT) { - exit(SHIM_EXIT_TIMEOUT); + error_exit(SHIM_EXIT_TIMEOUT); } exit(EXIT_SUCCESS); diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c index 97524f1a..5c508173 100644 --- a/src/cmd/isulad-shim/process.c +++ b/src/cmd/isulad-shim/process.c @@ -46,6 +46,7 @@ #include #include #include +#include #include "common.h" #include "terminal.h" @@ -61,7 +62,8 @@ static shim_client_process_state *load_process() p_state = shim_client_process_state_parse_file("process.json", NULL, &err); if (p_state == NULL) { - write_message(ERR_MSG, "parse process state failed: %s", err); + ERROR("parse process state failed: %s", err); + shim_set_error_message("parse process state failed: %s", err); } /* "err" will definitely be allocated memory in the function above */ free(err); @@ -76,7 +78,7 @@ static int open_fifo_noblock(const char *path, mode_t mode) /* By default, We consider that the file has been created by isulad */ fd = open_no_inherit(path, mode | O_NONBLOCK, -1); if (fd < 0) { - write_message(ERR_MSG, "open fifo file failed:%d", SHIM_SYS_ERR(errno)); + ERROR("open fifo file failed:%d", SHIM_SYS_ERR(errno)); return -1; } @@ -115,7 +117,7 @@ static int receive_fd(int sock) */ int ret = recvmsg(sock, &msg, 0); if (ret <= 0) { - write_message(ERR_MSG, "get console fd failed:%d", SHIM_SYS_ERR(errno)); + ERROR("get console fd failed:%d", SHIM_SYS_ERR(errno)); free(cmptr); return -1; } @@ -198,14 +200,14 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && !fifo_exists(err)); if (invalid) { - write_attach_message(ERR_MSG, "File %s or %s or %s does not refer to a FIFO", in, out, err); + ERROR("File %s or %s or %s does not refer to a FIFO", in, out, err); return -1; } if (in != NULL) { fifofd_in = isula_file_open(in, O_RDONLY | O_NONBLOCK | O_CLOEXEC, 0); if (fifofd_in < 0) { - write_attach_message(ERR_MSG, "Failed to open FIFO: %s", in); + ERROR("Failed to open FIFO: %s", in); return -1; } } @@ -213,7 +215,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char if (out != NULL) { fifofd_out = isula_file_open(out, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); if (fifofd_out < 0) { - write_attach_message(ERR_MSG, "Failed to open FIFO: %s", out); + ERROR("Failed to open FIFO: %s", out); return -1; } } @@ -221,14 +223,14 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char if (err != NULL) { fifofd_err = isula_file_open(err, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); if (fifofd_err < 0) { - write_attach_message(ERR_MSG, "Failed to open FIFO: %s", err); + ERROR("Failed to open FIFO: %s", err); return -1; } } fifos = isula_common_calloc_s(sizeof(*fifos)); if (fifos == NULL) { - write_attach_message(ERR_MSG, "Out of memory"); + ERROR("Out of memory"); goto err_out; } @@ -241,7 +243,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char fifos->err_fd = isula_transfer_fd(fifofd_err); node = isula_common_calloc_s(sizeof(struct isula_linked_list)); if (node == NULL) { - write_attach_message(ERR_MSG, "Out of memory"); + ERROR("Out of memory"); goto err_out; } @@ -300,7 +302,7 @@ static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t * w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); if (w_count < 0) { /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); + WARN("write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); close(*fd_to); *fd_to = -1; } @@ -317,7 +319,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de struct isula_linked_list *item; if (events & EPOLLHUP) { - write_message(ERR_MSG, "attach stdin %d received the EPOLLHUP event", fd); + ERROR("attach stdin %d received the EPOLLHUP event", fd); goto err_out; } @@ -329,7 +331,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); if (r_count <= 0) { - write_message(ERR_MSG, "failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); + ERROR("failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); goto err_out; } @@ -345,7 +347,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); if (w_count < 0) { /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); + WARN("write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); close(*fd_to); *fd_to = -1; } @@ -381,7 +383,7 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); if (w_count < 0) { /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); + WARN("write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); close(p->isulad_io->out); p->isulad_io->out = -1; } @@ -427,7 +429,7 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); if (w_count < 0) { /* When any error occurs, set the write fd -1 */ - write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); + WARN("write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); close(p->isulad_io->err); p->isulad_io->err = -1; } @@ -482,37 +484,37 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) char real_path[PATH_MAX] = { 0 }; if (isula_validate_absolute_path(fifopath) != 0) { - write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an valid absolute path", fifopath); + ERROR("attach fifo path \"%s\" must be an valid absolute path", fifopath); return false; } if (realpath(fifopath, real_path) == NULL) { - write_attach_message(ERR_MSG, "Failed to get realpath for '%s': %s.", real_path, SHIM_SYS_ERR(errno)); + ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); return false; } if (!isula_has_prefix(real_path, p->workdir)) { - write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be under the state path", real_path, p->workdir); + ERROR("attach fifo path \"%s\" must be under the state path: %s", real_path, p->workdir); return false; } if (lstat(real_path, &st) != 0) { - write_attach_message(ERR_MSG, "Failed to lstat %s : %s", real_path, SHIM_SYS_ERR(errno)); + ERROR("Failed to lstat %s : %d", real_path, SHIM_SYS_ERR(errno)); return false; } if (!S_ISFIFO(st.st_mode)) { - write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an FIFO", real_path); + ERROR("attach fifo path \"%s\" must be an FIFO", real_path); return false; } if ((st.st_mode & 0777) != ATTACH_FIFOPATH_MODE) { - write_attach_message(ERR_MSG, "attach fifo path \"%s\" permission invalid", real_path); + ERROR("attach fifo path \"%s\" permission invalid", real_path); return false; } if (st.st_uid != 0) { - write_attach_message(ERR_MSG, "attach fifo path \"%s\" uid invalid", real_path); + ERROR("attach fifo path \"%s\" uid invalid", real_path); return false; } @@ -553,20 +555,20 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t r_count = isula_file_read_nointr(fd, tmp_buf, sizeof(tmp_buf) - 1); if (r_count <= 0) { - write_attach_message(ERR_MSG, "Failed to read msg from attach conn fd"); + ERROR("Failed to read msg from attach conn fd"); goto out; } // limit the number of attach connections to MAX_ATTACH_NUM if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { - write_attach_message(ERR_MSG, "The number of attach connections exceeds the limit:%d, and this connection is rejected.", + ERROR("The number of attach connections exceeds the limit:%d, and this connection is rejected.", MAX_ATTACH_NUM); goto out; } tmp_str_array = isula_string_split_to_multi(tmp_buf, ' '); if (tmp_str_array->len != 3) { - write_attach_message(ERR_MSG, "Invalid attach msg from isulad"); + ERROR("Invalid attach msg from isulad"); goto out; } @@ -575,7 +577,7 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t } if (!valid) { - write_attach_message(ERR_MSG, "Invalid attach fifo path from isulad"); + ERROR("Invalid attach fifo path from isulad"); goto out; } @@ -584,14 +586,14 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t err = tmp_str_array->items[2]; if (add_attach_terminal_fifos(in, out, err, &fifofd_in, p) < 0) { - write_attach_message(ERR_MSG, "Failed to add attach terminal fifos"); + ERROR("Failed to add attach terminal fifos"); goto out; } // attach stdin --> container stdin ret = isula_epoll_add_handler(descr, fifofd_in, attach_stdin_cb, p); if (ret != SHIM_OK) { - write_attach_message(ERR_MSG, "add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); + ERROR("add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); struct isula_linked_list *item = get_attach_fifo_item(fd, p->attach_fifos); if (item != NULL && item->elem != NULL) { remove_attach_terminal_fifos(descr, item); @@ -616,13 +618,13 @@ static int do_attach_socket_accept(int fd, uint32_t events, void *cbdata, isula_ conn_fd = accept(p->attach_socket_fd, NULL, NULL); if (conn_fd < 0) { - write_attach_message(ERR_MSG, "accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); + ERROR("accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); return EPOLL_LOOP_HANDLE_CONTINUE; } ret = isula_epoll_add_handler(descr, conn_fd, attach_cb, p); if (ret != SHIM_OK) { - write_attach_message(ERR_MSG, "add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); + ERROR("add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); close(conn_fd); return EPOLL_LOOP_HANDLE_CONTINUE; } @@ -637,13 +639,13 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epol conn_fd = accept(p->listen_fd, NULL, NULL); if (conn_fd < 0) { - write_message(ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); + ERROR("accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); goto out; } p->recv_fd = receive_fd(conn_fd); if (check_fd(p->recv_fd) != true) { - write_message(ERR_MSG, "check console fd failed"); + ERROR("check console fd failed"); goto out; } @@ -652,19 +654,19 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epol // p->isulad_io->in ----> p->recv_fd ret = isula_epoll_add_handler(descr, p->isulad_io->in, stdin_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); + ERROR("add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); goto out; } // p->recv_fd ----> p->isulad_io->out ret = isula_epoll_add_handler(descr, p->recv_fd, stdout_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); + ERROR("add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); goto out; } // p->isulad_io->resize ----> p->recv_fd ret = isula_epoll_add_handler(descr, p->isulad_io->resize, resize_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); + ERROR("add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); goto out; } @@ -724,7 +726,7 @@ static stdio_t *initialize_io(process_t *p) */ if ((pipe2(stdio_fd[0], O_CLOEXEC) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC) != 0) || (pipe2(stdio_fd[2], O_CLOEXEC) != 0)) { - write_message(ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); + ERROR("open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); goto failure; } @@ -789,7 +791,7 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) { - write_message(ERR_MSG, "create socket failed:%d", SHIM_SYS_ERR(errno)); + ERROR("create socket failed:%d", SHIM_SYS_ERR(errno)); goto failure; } @@ -799,13 +801,13 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); if (ret < 0) { - write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); + ERROR("bind console fd failed:%d", SHIM_SYS_ERR(errno)); goto failure; } ret = listen(fd, 2); if (ret < 0) { - write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); + ERROR("listen console fd failed:%d", SHIM_SYS_ERR(errno)); goto failure; } @@ -813,7 +815,7 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) ret = isula_epoll_add_handler(descr, p->listen_fd, task_console_accept, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); + ERROR("add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); goto failure; } @@ -831,7 +833,7 @@ static int open_terminal_io(process_t *p, isula_epoll_descr_t *descr) ret = new_temp_console_path(p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "get temp console sock path failed"); + ERROR("get temp console sock path failed"); return SHIM_ERR; } @@ -853,19 +855,19 @@ static int open_generic_io(process_t *p, isula_epoll_descr_t *descr) // p->isulad_io->in ----> p->shim_io->in ret = isula_epoll_add_handler(descr, p->isulad_io->in, stdin_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); + ERROR("add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); return SHIM_ERR; } // p->shim_io->out ----> p->isulad_io->out ret = isula_epoll_add_handler(descr, p->shim_io->out, stdout_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); + ERROR("add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); return SHIM_ERR; } // p->shim_io->err ----> p->isulad_io->err ret = isula_epoll_add_handler(descr, p->shim_io->err, stderr_cb, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); + ERROR("add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); return SHIM_ERR; } @@ -896,22 +898,25 @@ static void *io_epoll_loop(void *data) ret = isula_epoll_open(&descr); if (ret != 0) { - write_message(ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); + ERROR("epoll loop open failed:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("epoll loop open failed:%d", SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } // sync fd: epoll loop will exit when recive sync fd event. ret = isula_epoll_add_handler(&descr, p->sync_fd, sync_exit_cb, p); if (ret != 0) { - write_message(ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); + ERROR("add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); + shim_set_error_message("add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } if (p->state->attach_socket != NULL) { ret = isula_epoll_add_handler(&descr, p->attach_socket_fd, do_attach_socket_accept, p); if (ret != SHIM_OK) { - write_message(ERR_MSG, "add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); + ERROR("add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); + shim_set_error_message("add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } } @@ -921,8 +926,9 @@ static void *io_epoll_loop(void *data) ret = open_generic_io(p, &descr); } if (ret != SHIM_OK) { - write_message(ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno)); - exit(EXIT_FAILURE); + ERROR("open io failed:%d", SHIM_SYS_ERR(errno)); + shim_append_error_message("open io failed:%d", SHIM_SYS_ERR(errno)); + error_exit(EXIT_FAILURE); } (void)sem_post(&p->sem_mainloop); @@ -933,8 +939,9 @@ static void *io_epoll_loop(void *data) // 3. stdin fd read failed ret = isula_epoll_loop(&descr, -1); if (ret != 0) { - write_message(ERR_MSG, "epoll loop failed"); - exit(EXIT_FAILURE); + ERROR("epoll loop failed"); + shim_set_error_message("epoll loop failed"); + error_exit(EXIT_FAILURE); } // use a timeout epoll loop to ensure complete data reception @@ -943,7 +950,7 @@ static void *io_epoll_loop(void *data) // 2. no event received within 100 milliseconds ret = isula_epoll_loop(&descr, 100); if (ret != 0) { - write_message(ERR_MSG, "Repeat the epoll loop to ensure that all data is transferred"); + ERROR("Repeat the epoll loop to ensure that all data is transferred"); } return NULL; @@ -966,12 +973,12 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s log_term = isula_common_calloc_s(sizeof(log_terminal)); if (log_term == NULL) { - write_message(ERR_MSG, "Failed to calloc log_terminal"); + ERROR("Failed to calloc log_terminal"); goto clean_out; } if (pthread_rwlock_init(&log_term->log_terminal_rwlock, NULL) != 0) { - write_message(ERR_MSG, "Failed to init isulad conf rwlock"); + ERROR("Failed to init isulad conf rwlock"); goto clean_out; } @@ -1047,25 +1054,25 @@ static int init_isulad_stdio(process_t *p) ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in); if (ret != SHIM_OK) { - write_message(ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin); + ERROR("Failed to open in isulad fd: %s", p->state->isulad_stdin); goto failure; } ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out); if (ret != SHIM_OK) { - write_message(ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout); + ERROR("Failed to open out isulad fd: %s", p->state->isulad_stdout); goto failure; } ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err); if (ret != SHIM_OK) { - write_message(ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr); + ERROR("Failed to open err isulad fd: %s", p->state->isulad_stderr); goto failure; } ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize); if (ret != SHIM_OK) { - write_message(ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo); + ERROR("Failed to open resize isulad fd: %s", p->state->resize_fifo); goto failure; } return SHIM_OK; @@ -1105,13 +1112,13 @@ static int init_root_path(process_t *p) // state_path: /run/isulad/runc/{container_id} char *tmp_dir = strrchr(state_path, '/'); if (tmp_dir == NULL) { - write_message(ERR_MSG, "Invalid exec workdir"); + ERROR("Invalid exec workdir"); return SHIM_ERR; } *tmp_dir = '\0'; tmp_dir = strrchr(state_path, '/'); if (tmp_dir == NULL) { - write_message(ERR_MSG, "Invalid exec workdir"); + ERROR("Invalid exec workdir"); return SHIM_ERR; } *tmp_dir = '\0'; @@ -1119,12 +1126,12 @@ static int init_root_path(process_t *p) isula_buffer *buffer = isula_buffer_alloc(PATH_MAX); if (buffer == NULL) { - write_message(ERR_MSG, "Failed to malloc buffer\n"); + ERROR("Failed to malloc buffer\n"); return SHIM_ERR; } if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->runtime) < 0) { - write_message(ERR_MSG, "Failed to append state_path\n"); + ERROR("Failed to append state_path\n"); isula_buffer_free(buffer); return SHIM_ERR; } @@ -1132,7 +1139,7 @@ static int init_root_path(process_t *p) p->root_path = buffer->to_str(buffer); isula_buffer_free(buffer); if (strlen(p->root_path) > PATH_MAX) { - write_message(ERR_MSG, "Root_path is too long\n"); + ERROR("Root_path is too long\n"); return SHIM_ERR; } return SHIM_OK; @@ -1180,7 +1187,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) p->sync_fd = eventfd(0, EFD_CLOEXEC); if (p->sync_fd < 0) { - write_message(ERR_MSG, "Failed to create eventfd: %s", strerror(errno)); + ERROR("Failed to create eventfd: %s", strerror(errno)); goto failure; } @@ -1197,7 +1204,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) // during the execution of isulad-shim, the current working directory will not change. p->workdir = getcwd(NULL, 0); if (p->workdir == NULL) { - write_message(ERR_MSG, "get cwd failed when do create process"); + ERROR("get cwd failed when do create process"); goto failure; } @@ -1432,13 +1439,15 @@ int create_process(process_t *p) int nread = -1; if (pipe2(exec_fd, O_CLOEXEC) != 0) { - write_message(ERR_MSG, "create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); + ERROR("create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); return SHIM_ERR; } pid_t pid = fork(); if (pid == (pid_t) -1) { - write_message(ERR_MSG, "fork failed when create process:%d", SHIM_SYS_ERR(errno)); + ERROR("fork failed when create process:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("fork failed when create process:%d", SHIM_SYS_ERR(errno)); return SHIM_ERR; } @@ -1461,14 +1470,16 @@ int create_process(process_t *p) /* block to wait runtime pid exit */ ret = waitpid(pid, NULL, 0); if (ret != pid) { - write_message(ERR_MSG, "wait runtime failed:%d", SHIM_SYS_ERR(errno)); + ERROR("wait runtime failed:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("wait runtime failed:%d", SHIM_SYS_ERR(errno)); ret = SHIM_ERR; goto out; } // if an error occurs in exec_runtime_process, jump directly to the out branch after waitpid. if (nread > 0) { - write_message(ERR_MSG, "%s", exec_buff); + ERROR("%s", exec_buff); + shim_set_error_message("%s", exec_buff); ret = SHIM_ERR; goto out; } @@ -1476,7 +1487,8 @@ int create_process(process_t *p) /* save runtime pid */ data = read_text_file("pid"); if (data == NULL) { - write_message(ERR_MSG, "read pid of runtime failed"); + ERROR("read pid of runtime failed"); + shim_set_error_message("read pid of runtime failed"); goto out; } int ctr_pid = atoi(data); @@ -1537,12 +1549,12 @@ static int waitpid_with_timeout(int ctr_pid, int *status, const uint64_t timeou if (*status == CONTAINER_ACTION_REBOOT) { nret = setenv("CONTAINER_ACTION", "reboot", 1); if (nret != SHIM_OK) { - write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); + WARN("set reboot action failed:%d", SHIM_SYS_ERR(errno)); } } else if (*status == CONTAINER_ACTION_SHUTDOWN) { nret = setenv("CONTAINER_ACTION", "shutdown", 1); if (nret != SHIM_OK) { - write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); + WARN("set shutdown action failed:%d", SHIM_SYS_ERR(errno)); } } return SHIM_OK; @@ -1565,12 +1577,12 @@ static int wait_container_process_with_timeout(process_t *p, const uint64_t time if (*status == CONTAINER_ACTION_REBOOT) { ret = setenv("CONTAINER_ACTION", "reboot", 1); if (ret != SHIM_OK) { - write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); + WARN("set reboot action failed:%d", SHIM_SYS_ERR(errno)); } } else if (*status == CONTAINER_ACTION_SHUTDOWN) { ret = setenv("CONTAINER_ACTION", "shutdown", 1); if (ret != SHIM_OK) { - write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); + WARN("set shutdown action failed:%d", SHIM_SYS_ERR(errno)); } } return SHIM_OK; @@ -1600,7 +1612,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const // kill container process to ensure process_kill_all effective nret = kill(p->ctr_pid, SIGKILL); if (nret < 0 && errno != ESRCH) { - write_message(ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid); + ERROR("Can not kill process (pid=%d) with SIGKILL, %d", p->ctr_pid, SHIM_SYS_ERR(errno)); return SHIM_ERR; } } @@ -1610,7 +1622,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const // wait atmost 120 seconds DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); if (nret != 0) { - write_message(ERR_MSG, "Failed to wait all child after 120 seconds"); + ERROR("Failed to wait all child after 120 seconds"); } process_delete(p); @@ -1620,13 +1632,13 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const if (p->sync_fd > 0) { if (eventfd_write(p->sync_fd, 1)) { - write_message(ERR_MSG, "Failed to write sync fd"); + ERROR("Failed to write sync fd"); } } nret = pthread_join(tid_epoll, NULL); if (nret != 0) { - write_message(ERR_MSG, "Failed to join epoll loop thread"); + ERROR("Failed to join epoll loop thread"); } close(p->sync_fd); @@ -1638,7 +1650,8 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const } if (ret == SHIM_ERR_TIMEOUT) { - write_message(INFO_MSG, "Wait %d timeout", p->ctr_pid); + ERROR("Wait %d timeout", p->ctr_pid); + shim_set_error_message("Wait %d timeout", p->ctr_pid); return SHIM_ERR_TIMEOUT; } @@ -1653,13 +1666,15 @@ int prepare_attach_socket(process_t *p) int ret = -1; if (strlen(p->state->attach_socket) >= sizeof(addr.sun_path)) { - write_message(ERR_MSG, "Invalid attach socket path: %s", p->state->attach_socket); + ERROR("Invalid attach socket path: %s", p->state->attach_socket); + shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); return SHIM_ERR; } p->attach_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); if (p->attach_socket_fd < 0) { - write_message(ERR_MSG, "Failed to create socket:%d", SHIM_SYS_ERR(errno)); + ERROR("Failed to create socket:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); return SHIM_ERR; } @@ -1669,13 +1684,15 @@ int prepare_attach_socket(process_t *p) ret = bind(p->attach_socket_fd, (struct sockaddr *)&addr, sizeof(addr)); if (ret < 0) { - write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); + ERROR("bind console fd failed:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); return SHIM_ERR; } ret = chmod(p->state->attach_socket, SOCKET_DIRECTORY_MODE); if (ret != 0) { - write_message(ERR_MSG, "Failed to chmod for socket: %s", p->state->attach_socket); + ERROR("Failed to chmod for socket: %s", p->state->attach_socket); + shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); return SHIM_ERR; } @@ -1687,7 +1704,8 @@ int prepare_attach_socket(process_t *p) // The maximum number of attach we allow here is MAX_ATTACH_NUM, so just use it directly ret = listen(p->attach_socket_fd, MAX_ATTACH_NUM); if (ret < 0) { - write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); + ERROR("listen console fd failed:%d", SHIM_SYS_ERR(errno)); + shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); return SHIM_ERR; } return SHIM_OK; diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c index fbb779f7..2b12d173 100644 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c @@ -69,6 +69,16 @@ // handle string from stderr output. typedef int(*handle_output_callback_t)(const char *output); +typedef struct { + bool fg; + const char *id; + char *workdir; + const char *bundle; + const char *runtime_cmd; + int *exit_code; + char *timeout; + int *shim_exit_code; +} shim_create_args; static void copy_process(shim_client_process_state *p, defs_process *dp) { @@ -154,9 +164,10 @@ static void file_read_int(const char *fname, int *val) free(sint); } -static void get_err_message(char *buf, int buf_size, const char *workdir, const char *file) +static int get_err_message(char *buf, int buf_size, const char *workdir, const char *file) { int nret; + int ret = 0; char fname[PATH_MAX] = { 0 }; FILE *fp = NULL; char *pline = NULL; @@ -166,12 +177,12 @@ static void get_err_message(char *buf, int buf_size, const char *workdir, const nret = snprintf(fname, PATH_MAX, "%s/%s", workdir, file); if (nret < 0 || (size_t)nret >= PATH_MAX) { ERROR("failed make full path %s/%s", workdir, file); - return; + return ret; } fp = util_fopen(fname, "r"); if (fp == NULL) { - return; + return ret; } while (getline(&pline, &length, fp) != -1) { @@ -199,48 +210,58 @@ static void get_err_message(char *buf, int buf_size, const char *workdir, const fclose(fp); if (lines[2] != NULL) { - (void)snprintf(buf, buf_size, "%s%s%s", lines[0], lines[1], lines[2]); + nret = snprintf(buf, buf_size, "%s%s%s", lines[0], lines[1], lines[2]); + ret += nret; } else if (lines[1] != NULL) { - (void)snprintf(buf, buf_size, "%s%s", lines[0], lines[1]); + nret = snprintf(buf, buf_size, "%s%s", lines[0], lines[1]); + ret += nret; } else if (lines[0] != NULL) { - (void)snprintf(buf, buf_size, "%s", lines[0]); + nret = snprintf(buf, buf_size, "%s", lines[0]); + ret += nret; } UTIL_FREE_AND_SET_NULL(pline); UTIL_FREE_AND_SET_NULL(lines[0]); UTIL_FREE_AND_SET_NULL(lines[1]); UTIL_FREE_AND_SET_NULL(lines[2]); + return ret; } -static void show_shim_runtime_errlog(const char *workdir) +static void show_runtime_errlog(const char *workdir) { char buf[BUFSIZ] = { 0 }; char buf1[SHIM_LOG_SIZE] = { 0 }; - char buf2[SHIM_LOG_SIZE] = { 0 }; + int nret; if (g_isulad_errmsg != NULL) { return; } - get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); - get_err_message(buf2, sizeof(buf2), workdir, "log.json"); - ERROR("shim-log: %s", buf1); - ERROR("runtime-log: %s", buf2); - (void)snprintf(buf, sizeof(buf), "shim-log error: %s\nruntime-log error: %s\n", buf1, buf2); + nret = get_err_message(buf1, sizeof(buf1), workdir, "log.json"); + if (nret == 0) { + ERROR("empty runtime-log : %s", workdir); + return; + } + ERROR("runtime-log: %s", buf1); + (void)snprintf(buf, sizeof(buf), "runtime-log error: %s\n", buf1); isulad_set_error_message(buf); } -static void show_shim_attach_errlog(const char *workdir) +static void show_shim_errlog(const int fd) { - char buf[SHIM_LOG_SIZE] = { 0 }; + int num; + char buf[BUFSIZ] = { 0 }; if (g_isulad_errmsg != NULL) { return; } - get_err_message(buf, sizeof(buf), workdir, "attach-log.json"); - ERROR("shim-log: %s", buf); - isulad_set_error_message("shim-log error:\n%s\n", buf); + num = util_read_nointr(fd, buf, sizeof(buf) - 1); + if (num < 0) { + SYSERROR("Failed to read err msg from shim stderr"); + return; + } + isulad_set_error_message(buf); } bool rt_isula_detect(const char *runtime) @@ -818,12 +839,48 @@ static int status_to_exit_code(int status) return exit_code; } +static int get_engine_routine_log_info(char **engine_log_path, char **log_level) +{ + int ret = 0; + struct service_arguments *args = NULL; + + if (isulad_server_conf_rdlock() != 0) { + return -1; + } + + args = conf_get_server_conf(); + if (args == NULL) { + ERROR("Failed to get isulad server config"); + ret = -1; + goto unlock_out; + } + + *engine_log_path = conf_get_engine_log_file(); + if (*engine_log_path == NULL) { + ERROR("Log fifo path is NULL"); + ret = -1; + goto unlock_out; + } + + *log_level = util_strdup_s(args->json_confs->log_level); + if (*engine_log_path == NULL) { + ERROR("Log level is NULL"); + ret = -1; + goto unlock_out; + } + +unlock_out: + if (isulad_server_conf_unlock()) { + ret = -1; + } + return ret; +} + /* exit_code records the exit code of the container, obtained by reading the stdout of isulad-shim; shim_exit_code records the exit code of isulad-shim, obtained through waitpid; */ -static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, - int *exit_code, const char *timeout, int *shim_exit_code) +static int shim_create(shim_create_args *args) { pid_t pid = 0; int shim_stderr_pipe[2] = { -1, -1 }; @@ -833,22 +890,29 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char char exec_buff[BUFSIZ + 1] = { 0 }; char fpid[PATH_MAX] = { 0 }; const char *params[PARAM_NUM] = { 0 }; + __isula_auto_free char *engine_log_path = NULL; + __isula_auto_free char *log_level = NULL; int i = 0; int status = 0; int nret = 0; params[i++] = SHIM_BINARY; - params[i++] = id; - params[i++] = bundle; - params[i++] = runtime_cmd; + params[i++] = args->id; + params[i++] = args->bundle; + params[i++] = args->runtime_cmd; params[i++] = "info"; // execSync timeout - if (timeout != NULL) { - params[i++] = timeout; + if (args->timeout != NULL) { + params[i++] = args->timeout; } runtime_exec_param_dump(params); - nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir); + if (get_engine_routine_log_info(&engine_log_path, &log_level) != 0) { + ERROR("failed to get engine log path"); + return -1; + } + + nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", args->workdir); if (nret < 0 || (size_t)nret >= sizeof(fpid)) { ERROR("failed make shim-pid full path"); return -1; @@ -875,12 +939,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char } if (pid == (pid_t)0) { - if (chdir(workdir) < 0) { - (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", id, workdir); + if (chdir(args->workdir) < 0) { + (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); exit(EXIT_FAILURE); } - if (fg) { + if (args->fg) { // child process, dup2 shim_stdout_pipe[1] to STDOUT if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) { (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno)); @@ -896,18 +960,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char // clear NOTIFY_SOCKET from the env to adapt runc create if (unsetenv("NOTIFY_SOCKET") != 0) { - (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); + (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); exit(EXIT_FAILURE); } pid = fork(); if (pid < 0) { - (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", id, strerror(errno)); + (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); _exit(EXIT_FAILURE); } if (pid != 0) { if (file_write_int(fpid, pid) != 0) { - (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", id, fpid, pid); + (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); } _exit(EXIT_SUCCESS); } @@ -918,7 +982,7 @@ realexec: close(shim_stdout_pipe[0]); if (setsid() < 0) { - (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", id, getpid()); + (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); exit(EXIT_FAILURE); } @@ -927,6 +991,16 @@ realexec: exit(EXIT_FAILURE); } + if (setenv("ISULAD_SHIIM_LOG_PATH", engine_log_path, 1) != 0) { + (void)dprintf(shim_stderr_pipe[1], "%s: failed to set ISULAD_SHIIM_LOG_PATH env for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + + if (setenv("ISULAD_SHIIM_LOG_LEVEL", log_level, 1) != 0) { + (void)dprintf(shim_stderr_pipe[1], "%s: failed to set ISULAD_SHIIM_LOG_LEVEL env for process %d", args->id, getpid()); + exit(EXIT_FAILURE); + } + execvp(SHIM_BINARY, (char * const *)params); (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); exit(EXIT_FAILURE); @@ -951,38 +1025,43 @@ realexec: goto out; } - *shim_exit_code = status_to_exit_code(status); - if (*shim_exit_code != 0) { - ERROR("Isulad-shim exit error"); + *(args->shim_exit_code) = status_to_exit_code(status); + if (*(args->shim_exit_code) != 0) { + ERROR("Isulad-shim exit error : %d", *(args->shim_exit_code)); + isulad_set_error_message("Isulad-shim exit error : %d, please get more information from log", *(args->shim_exit_code)); ret = -1; goto out; } // exit_code is NULL when command is create. - if (exit_code == NULL) { + if (args->exit_code == NULL) { goto out; } // when exec in background, exit code is shim exit code - if (!fg) { - *exit_code = *shim_exit_code; + if (!args->fg) { + *(args->exit_code) = *(args->shim_exit_code); goto out; } - ret = util_read_nointr(shim_stdout_pipe[0], exit_code, sizeof(int)); + ret = util_read_nointr(shim_stdout_pipe[0], args->exit_code, sizeof(int)); if (ret <= 0) { - *exit_code = 137; + *(args->exit_code) = 137; } ret = 0; out: - close(shim_stderr_pipe[0]); close(shim_stdout_pipe[0]); if (ret != 0) { - show_shim_runtime_errlog(workdir); - if (timeout != NULL) { + if (*(args->shim_exit_code) == 0) { + show_runtime_errlog(args->workdir); + } else { + show_shim_errlog(shim_stderr_pipe[0]); + } + if (args->timeout != NULL) { kill(pid, SIGKILL); /* can kill other process? */ } } + close(shim_stderr_pipe[0]); return ret; } @@ -1061,6 +1140,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ char workdir[PATH_MAX] = { 0 }; char attach_socket[PATH_MAX] = { 0 }; shim_client_process_state p = { 0 }; + shim_create_args args = { 0 }; int shim_exit_code = 0; int nret = 0; @@ -1108,7 +1188,15 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ } get_runtime_cmd(runtime, &cmd); - ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, NULL, &shim_exit_code); + args.fg = false; + args.id = id; + args.workdir = workdir; + args.bundle = params->bundle; + args.runtime_cmd = cmd; + args.exit_code = NULL; + args.timeout = NULL; + args.shim_exit_code = &shim_exit_code; + ret = shim_create(&args); if (ret != 0) { runtime_call_delete_force(workdir, runtime, id); ERROR("%s: failed create shim process", id); @@ -1185,7 +1273,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t ret = 0; out: if (ret != 0) { - show_shim_runtime_errlog(workdir); + show_runtime_errlog(workdir); shim_kill_force(workdir); } return ret; @@ -1363,6 +1451,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p int pid = 0; char bundle[PATH_MAX] = { 0 }; char workdir[PATH_MAX] = { 0 }; + shim_create_args args = { 0 }; char *timeout = NULL; int shim_exit_code = 0; @@ -1411,7 +1500,15 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p } } - ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, timeout, &shim_exit_code); + args.fg = fg_exec(params); + args.id = id; + args.workdir = workdir; + args.bundle = bundle; + args.runtime_cmd = cmd; + args.exit_code = exit_code; + args.timeout = timeout; + args.shim_exit_code = &shim_exit_code; + ret = shim_create(&args); if (shim_exit_code == SHIM_EXIT_TIMEOUT) { ret = -1; isulad_set_error_message("Exec container error;exec timeout"); @@ -1432,7 +1529,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p errlog_out: if (ret != 0) { - show_shim_runtime_errlog(workdir); + show_runtime_errlog(workdir); } if (timeout != NULL) { @@ -1559,13 +1656,13 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ if (id == NULL || runtime == NULL || params == NULL) { ERROR("Null argument"); - return -1; + goto err_out; } ret = snprintf(workdir, sizeof(workdir), "%s/%s", params->state, id); if (ret < 0 || (size_t)ret >= sizeof(workdir)) { ERROR("Failed join exec full path"); - return -1; + goto err_out; } // the communication format between isulad and isulad-shim attach is: @@ -1573,19 +1670,19 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ len = snprintf(buf, sizeof(buf), "%s %s %s", params->stdin, params->stdout, params->stderr); if (len < 0 || (size_t)len >= sizeof(buf)) { ERROR("Failed to snprintf string"); - return -1; + goto err_out; } ret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); if (ret < 0 || (size_t)ret >= sizeof(attach_socket)) { ERROR("Failed to get full attach socket path"); - return -1; + goto err_out; } ret = get_attach_socketfd(attach_socket, &socket_fd); if (ret < 0) { ERROR("Failed to get attach socketfd"); - return -1; + goto err_out; } DEBUG("write %s to attach fd", buf); @@ -1593,16 +1690,19 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ ret = isula_file_write_nointr(socket_fd, buf, len); if (ret < 0) { SYSERROR("Failed to write attach isulad fd"); - return -1; + goto err_out; } status_code = get_container_attach_statuscode(workdir, socket_fd); if (status_code < 0) { - show_shim_attach_errlog(workdir); - return -1; + ERROR("Failed to attach container io, get more information from log"); + goto err_out; } return 0; +err_out: + isulad_set_error_message("Failed to attach container io, get more information from log"); + return -1; } static int to_engine_resources_unified(const host_config *hostconfig, shim_client_cgroup_resources *cr) -- 2.42.0