diff --git a/0065-refactor-isulad-shim-log.patch b/0065-refactor-isulad-shim-log.patch deleted file mode 100644 index d8042e1..0000000 --- a/0065-refactor-isulad-shim-log.patch +++ /dev/null @@ -1,1590 +0,0 @@ -From ec5736c76acc80b6099c82cd21fdad5b8da79b73 Mon Sep 17 00:00:00 2001 -From: zhongtao -Date: Tue, 19 Dec 2023 19:47:18 +0800 -Subject: [PATCH 65/65] refactor isulad-shim log - -Signed-off-by: zhongtao ---- - src/cmd/isulad-shim/common.c | 179 ++++++++------- - src/cmd/isulad-shim/common.h | 33 ++- - src/cmd/isulad-shim/main.c | 60 ++--- - src/cmd/isulad-shim/process.c | 192 +++++++++------- - .../modules/runtime/isula/isula_rt_ops.c | 212 +++++++++++++----- - 5 files changed, 425 insertions(+), 251 deletions(-) - -diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c -index 3cc7d2a7..d96b6c65 100644 ---- a/src/cmd/isulad-shim/common.c -+++ b/src/cmd/isulad-shim/common.c -@@ -31,34 +31,124 @@ - - #include - #include -+#include -+#include -+#include - --int g_log_fd = -1; --int g_attach_log_fd = -1; -+char *g_shim_errmsg = NULL; -+bool log_to_stderr = false; - --int init_shim_log(void) -+int isulad_shim_log_init(const char *file, const char *priority) - { -- g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); -- if (g_log_fd < 0) { -- return SHIM_ERR; -+ __isula_auto_free char *full_path = NULL; -+ const char *pre_name = "fifo:"; -+ size_t pre_len = 0; -+ struct isula_libutils_log_config lconf = { 0 }; -+ -+ if (file == NULL || priority == NULL || strncmp(file, pre_name, pre_len) != 0) { -+ goto err_out; - } -- return SHIM_OK; -+ -+ pre_len = strlen(pre_name); -+ lconf.name = "engine"; -+ /* File has prefix "fifo:", */ -+ full_path = isula_strdup_s(file + pre_len); -+ lconf.file = full_path; -+ lconf.driver = "fifo"; -+ lconf.priority = priority; -+ if (isula_libutils_log_enable(&lconf) != 0) { -+ goto err_out; -+ } -+ -+ return 0; -+err_out: -+ // because shim log init error, print error msg to stderr. -+ // isulad can obtain the reason why shim exits. -+ dprintf(STDERR_FILENO, "failed to init shim log"); -+ return -1; - } - --int init_attach_log(void) -+void shim_set_error_message(const char *format, ...) - { -- g_attach_log_fd = open_no_inherit(ATTACH_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, LOG_FILE_MODE); -- if (g_attach_log_fd < 0) { -- return SHIM_ERR; -+ int ret = 0; -+ char errbuf[BUFSIZ + 1] = { 0 }; -+ va_list argp; -+ -+ if (format == NULL) { -+ return; - } -- return SHIM_OK; -+ va_start(argp, format); -+ -+ ret = vsnprintf(errbuf, BUFSIZ, format, argp); -+ va_end(argp); -+ if (ret < 0) { -+ return; -+ } -+ -+ SHIM_CLEAR_ERRMSG(); -+ g_shim_errmsg = isula_strdup_s(errbuf); -+} -+ -+void shim_try_set_error_message(const char *format, ...) -+{ -+ int ret = 0; -+ -+ if (g_shim_errmsg != NULL) { -+ return; -+ } -+ char errbuf[BUFSIZ + 1] = { 0 }; -+ -+ va_list argp; -+ va_start(argp, format); -+ -+ ret = vsnprintf(errbuf, BUFSIZ, format, argp); -+ va_end(argp); -+ if (ret < 0) { -+ return; -+ } -+ -+ g_shim_errmsg = isula_strdup_s(errbuf); -+} -+ -+void shim_append_error_message(const char *format, ...) -+{ -+ int ret = 0; -+ char errbuf[BUFSIZ + 1] = { 0 }; -+ char *result = NULL; -+ -+ va_list argp; -+ va_start(argp, format); -+ -+ ret = vsnprintf(errbuf, BUFSIZ, format, argp); -+ va_end(argp); -+ if (ret < 0) { -+ return; -+ } -+ result = isula_string_append(g_shim_errmsg, errbuf); -+ if (result == NULL) { -+ return; -+ } -+ if (g_shim_errmsg != NULL) { -+ free(g_shim_errmsg); -+ } -+ g_shim_errmsg = result; -+} -+ -+void error_exit(int exit_code) -+{ -+ if (log_to_stderr && g_shim_errmsg != NULL) { -+ dprintf(STDERR_FILENO, "shim-error: %s\n", g_shim_errmsg); -+ } -+ exit(exit_code); - } - - void signal_routine(int sig) - { - switch (sig) { - case SIGALRM: -- write_message(ERR_MSG, "runtime timeout"); -- exit(EXIT_FAILURE); -+ ERROR("runtime timeout"); -+ shim_set_error_message("runtime timeout"); -+ error_exit(EXIT_FAILURE); - default: - break; - } -@@ -172,67 +262,6 @@ int generate_random_str(char *id, size_t len) - return SHIM_OK; - } - --#define MAX_MSG_JSON_TEMPLATE 32 --#define MAX_MESSAGE_CONTENT_LEN 128 --#define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN) -- --static void format_log_msg(const char *level, const char *buf, char *msg, int max_message_len) --{ -- time_t current_time = time(NULL); -- struct tm *local_time = localtime(¤t_time); -- char time_str[20]; -- -- strftime(time_str, sizeof(time_str), "%Y-%m-%d %H:%M:%S", local_time); -- -- (void)snprintf(msg, max_message_len - 1, "{\"time\": \"%s\", \"level\": \"%s\", \"msg\": \"%s\"}\n", time_str, level, -- buf); --} -- --void write_message(const char *level, const char *fmt, ...) --{ -- if (g_log_fd < 0) { -- return; -- } -- -- char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; -- char msg[MAX_MESSAGE_LEN] = { 0 }; -- int nwrite = -1; -- -- va_list arg_list; -- va_start(arg_list, fmt); -- nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); -- va_end(arg_list); -- if (nwrite < 0) { -- return; -- } -- -- format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); -- -- (void)isula_file_total_write_nointr(g_log_fd, msg, strlen(msg)); --} -- --void write_attach_message(const char *level, const char *fmt, ...) --{ -- char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 }; -- char msg[MAX_MESSAGE_LEN] = { 0 }; -- int nwrite = -1; -- -- if (g_attach_log_fd < 0) { -- return; -- } -- va_list arg_list; -- va_start(arg_list, fmt); -- nwrite = vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list); -- va_end(arg_list); -- if (nwrite < 0) { -- return; -- } -- -- format_log_msg(level, buf, msg, MAX_MESSAGE_CONTENT_LEN); -- -- (void)isula_file_total_write_nointr(g_attach_log_fd, msg, strlen(msg)); --} -- - /* note: This function can only read small text file. */ - char *read_text_file(const char *path) - { -diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h -index c4f86d24..9ac11b61 100644 ---- a/src/cmd/isulad-shim/common.h -+++ b/src/cmd/isulad-shim/common.h -@@ -55,7 +55,8 @@ extern "C" { - #define MAX_RUNTIME_ARGS 100 - - #define SHIM_BINARY "isulad-shim" --#define SHIM_LOG_NAME "shim-log.json" -+#define DEFAULT_ISULAD_SHIIM_LOG_PATH "fifo:/var/run/isulad/isulad_log_gather_fifo" -+#define DEFAULT_ISULAD_SHIIM_LOG_LEVEL "ERROR" - - #define CONTAINER_ACTION_REBOOT 129 - #define CONTAINER_ACTION_SHUTDOWN 130 -@@ -72,9 +73,7 @@ extern "C" { - #define SOCKET_DIRECTORY_MODE 0600 - #define ATTACH_FIFOPATH_MODE 0600 - --int init_shim_log(void); -- --int init_attach_log(void); -+int isulad_shim_log_init(const char *file, const char *priority); - - void signal_routine(int sig); - -@@ -114,13 +113,31 @@ struct shim_fifos_fd { - int err_fd; - }; - --char *read_text_file(const char *path); -+/* record the isulad-shim errmsg */ -+extern char *g_shim_errmsg; - --int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); -+extern bool log_to_stderr; -+ -+/* clear the g_shim_errmsg */ -+#define SHIM_CLEAR_ERRMSG() \ -+ do { \ -+ if (g_shim_errmsg != NULL) { \ -+ free(g_shim_errmsg); \ -+ g_shim_errmsg = NULL; \ -+ } \ -+ } while (0) -+ -+void shim_set_error_message(const char *format, ...); - --void write_message(const char *level, const char *fmt, ...); -+void shim_try_set_error_message(const char *format, ...); - --void write_attach_message(const char *level, const char *fmt, ...); -+void shim_append_error_message(const char *format, ...); -+ -+void error_exit(int exit_code); -+ -+char *read_text_file(const char *path); -+ -+int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len); - - int generate_random_str(char *id, size_t len); - -diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c -index deb07271..a12e459f 100644 ---- a/src/cmd/isulad-shim/main.c -+++ b/src/cmd/isulad-shim/main.c -@@ -24,6 +24,7 @@ - #include - #include - #include -+#include - - #include "common.h" - #include "process.h" -@@ -91,17 +92,25 @@ int main(int argc, char **argv) - char *bundle = NULL; - char *rt_name = NULL; - char *log_level = NULL; -+ char *engine_log_path = NULL; - int ret = SHIM_ERR; - process_t *p = NULL; - // execSync timeout - uint64_t timeout = 0; - pthread_t tid_epoll; - -- ret = init_shim_log(); -+ engine_log_path = getenv("ISULAD_SHIIM_LOG_PATH"); -+ if (engine_log_path == NULL) { -+ engine_log_path = DEFAULT_ISULAD_SHIIM_LOG_PATH; -+ } -+ -+ log_level = getenv("ISULAD_SHIIM_LOG_LEVEL"); -+ if (log_level == NULL) { -+ log_level = DEFAULT_ISULAD_SHIIM_LOG_LEVEL; -+ } -+ -+ ret = isulad_shim_log_init(engine_log_path, log_level); - if (ret != SHIM_OK) { -- // because shim log init error, print error msg to stderr. -- // isulad can obtain the reason why shim exits. -- dprintf(STDERR_FILENO, "failed to init shim log"); - _exit(EXIT_FAILURE); - } - -@@ -113,22 +122,26 @@ int main(int argc, char **argv) - - ret = set_subreaper(); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "set subreaper failed:%d", ret); -- exit(EXIT_FAILURE); -+ ERROR("set subreaper failed:%d", ret); -+ error_exit(EXIT_FAILURE); - } - - ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level, &timeout); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "parse args failed:%d", ret); -- exit(EXIT_FAILURE); -+ ERROR("parse args failed:%d", ret); -+ error_exit(EXIT_FAILURE); - } - - p = new_process(container_id, bundle, rt_name); - if (p == NULL) { -- write_message(ERR_MSG, "new process failed"); -- exit(EXIT_FAILURE); -+ ERROR("new process failed"); -+ error_exit(EXIT_FAILURE); - } - -+ // If isulad-shim is a child process of the isulad process, -+ // print the log to stderr so that isulad can obtain the exit information of isulad-shim. -+ log_to_stderr = (p->state->exec) && (p->state->isulad_stdin != NULL || p->state->isulad_stdout != NULL || p->state->isulad_stderr != NULL); -+ - /* - * Open exit pipe - * The exit pipe exists only when the container is started, -@@ -138,8 +151,9 @@ int main(int argc, char **argv) - if (p->state->exit_fifo != NULL) { - int efd = open_no_inherit("exit_fifo", O_WRONLY, -1); - if (efd < 0) { -- write_message(ERR_MSG, "open exit pipe failed:%d", SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -+ ERROR("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - p->exit_fd = efd; - } -@@ -148,22 +162,18 @@ int main(int argc, char **argv) - if (p->state->attach_socket != NULL) { - ret = prepare_attach_socket(p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "failed to prepare attach socket:%d", ret); -- exit(EXIT_FAILURE); -- } -- -- ret = init_attach_log(); -- if (ret != SHIM_OK) { -- write_message(ERR_MSG, "failed to init shim attach log"); -- exit(EXIT_FAILURE); -+ ERROR("failed to prepare attach socket:%d", ret); -+ shim_append_error_message("open exit pipe failed:%d", SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - } - - /* start epoll for io copy */ - ret = process_io_start(p, &tid_epoll); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "process io init failed:%d", ret); -- exit(EXIT_FAILURE); -+ ERROR("process io init failed:%d", ret); -+ shim_set_error_message("process io init failed:%d", ret); -+ error_exit(EXIT_FAILURE); - } - - ret = create_process(p); -@@ -171,17 +181,17 @@ int main(int argc, char **argv) - if (p->console_sock_path != NULL) { - (void)unlink(p->console_sock_path); - } -- exit(EXIT_FAILURE); -+ error_exit(EXIT_FAILURE); - } - - released_timeout_exit(); - - ret = process_signal_handle_routine(p, tid_epoll, timeout); - if (ret == SHIM_ERR) { -- exit(EXIT_FAILURE); -+ error_exit(EXIT_FAILURE); - } - if (ret == SHIM_ERR_TIMEOUT) { -- exit(SHIM_EXIT_TIMEOUT); -+ error_exit(SHIM_EXIT_TIMEOUT); - } - - exit(EXIT_SUCCESS); -diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c -index 97524f1a..5c508173 100644 ---- a/src/cmd/isulad-shim/process.c -+++ b/src/cmd/isulad-shim/process.c -@@ -46,6 +46,7 @@ - #include - #include - #include -+#include - - #include "common.h" - #include "terminal.h" -@@ -61,7 +62,8 @@ static shim_client_process_state *load_process() - - p_state = shim_client_process_state_parse_file("process.json", NULL, &err); - if (p_state == NULL) { -- write_message(ERR_MSG, "parse process state failed: %s", err); -+ ERROR("parse process state failed: %s", err); -+ shim_set_error_message("parse process state failed: %s", err); - } - /* "err" will definitely be allocated memory in the function above */ - free(err); -@@ -76,7 +78,7 @@ static int open_fifo_noblock(const char *path, mode_t mode) - /* By default, We consider that the file has been created by isulad */ - fd = open_no_inherit(path, mode | O_NONBLOCK, -1); - if (fd < 0) { -- write_message(ERR_MSG, "open fifo file failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("open fifo file failed:%d", SHIM_SYS_ERR(errno)); - return -1; - } - -@@ -115,7 +117,7 @@ static int receive_fd(int sock) - */ - int ret = recvmsg(sock, &msg, 0); - if (ret <= 0) { -- write_message(ERR_MSG, "get console fd failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("get console fd failed:%d", SHIM_SYS_ERR(errno)); - free(cmptr); - return -1; - } -@@ -198,14 +200,14 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char - bool invalid = (in != NULL && !fifo_exists(in)) || (out != NULL && !fifo_exists(out)) || (err != NULL && - !fifo_exists(err)); - if (invalid) { -- write_attach_message(ERR_MSG, "File %s or %s or %s does not refer to a FIFO", in, out, err); -+ ERROR("File %s or %s or %s does not refer to a FIFO", in, out, err); - return -1; - } - - if (in != NULL) { - fifofd_in = isula_file_open(in, O_RDONLY | O_NONBLOCK | O_CLOEXEC, 0); - if (fifofd_in < 0) { -- write_attach_message(ERR_MSG, "Failed to open FIFO: %s", in); -+ ERROR("Failed to open FIFO: %s", in); - return -1; - } - } -@@ -213,7 +215,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char - if (out != NULL) { - fifofd_out = isula_file_open(out, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); - if (fifofd_out < 0) { -- write_attach_message(ERR_MSG, "Failed to open FIFO: %s", out); -+ ERROR("Failed to open FIFO: %s", out); - return -1; - } - } -@@ -221,14 +223,14 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char - if (err != NULL) { - fifofd_err = isula_file_open(err, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0); - if (fifofd_err < 0) { -- write_attach_message(ERR_MSG, "Failed to open FIFO: %s", err); -+ ERROR("Failed to open FIFO: %s", err); - return -1; - } - } - - fifos = isula_common_calloc_s(sizeof(*fifos)); - if (fifos == NULL) { -- write_attach_message(ERR_MSG, "Out of memory"); -+ ERROR("Out of memory"); - goto err_out; - } - -@@ -241,7 +243,7 @@ static int add_attach_terminal_fifos(const char *in, const char *out, const char - fifos->err_fd = isula_transfer_fd(fifofd_err); - node = isula_common_calloc_s(sizeof(struct isula_linked_list)); - if (node == NULL) { -- write_attach_message(ERR_MSG, "Out of memory"); -+ ERROR("Out of memory"); - goto err_out; - } - -@@ -300,7 +302,7 @@ static int stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t * - w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ -- write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); -+ WARN("write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); - close(*fd_to); - *fd_to = -1; - } -@@ -317,7 +319,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de - struct isula_linked_list *item; - - if (events & EPOLLHUP) { -- write_message(ERR_MSG, "attach stdin %d received the EPOLLHUP event", fd); -+ ERROR("attach stdin %d received the EPOLLHUP event", fd); - goto err_out; - } - -@@ -329,7 +331,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de - - r_count = isula_file_read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF); - if (r_count <= 0) { -- write_message(ERR_MSG, "failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); -+ ERROR("failed to read from attach stdin %d, error:%d", fd, SHIM_SYS_ERR(errno)); - goto err_out; - } - -@@ -345,7 +347,7 @@ static int attach_stdin_cb(int fd, uint32_t events, void *cbdata, isula_epoll_de - w_count = isula_file_total_write_nointr(*fd_to, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ -- write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); -+ WARN("write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno)); - close(*fd_to); - *fd_to = -1; - } -@@ -381,7 +383,7 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t - w_count = isula_file_total_write_nointr(p->isulad_io->out, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ -- write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); -+ WARN("write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno)); - close(p->isulad_io->out); - p->isulad_io->out = -1; - } -@@ -427,7 +429,7 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t - w_count = isula_file_total_write_nointr(p->isulad_io->err, p->buf, r_count); - if (w_count < 0) { - /* When any error occurs, set the write fd -1 */ -- write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); -+ WARN("write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno)); - close(p->isulad_io->err); - p->isulad_io->err = -1; - } -@@ -482,37 +484,37 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath) - char real_path[PATH_MAX] = { 0 }; - - if (isula_validate_absolute_path(fifopath) != 0) { -- write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an valid absolute path", fifopath); -+ ERROR("attach fifo path \"%s\" must be an valid absolute path", fifopath); - return false; - } - - if (realpath(fifopath, real_path) == NULL) { -- write_attach_message(ERR_MSG, "Failed to get realpath for '%s': %s.", real_path, SHIM_SYS_ERR(errno)); -+ ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno)); - return false; - } - - if (!isula_has_prefix(real_path, p->workdir)) { -- write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be under the state path", real_path, p->workdir); -+ ERROR("attach fifo path \"%s\" must be under the state path: %s", real_path, p->workdir); - return false; - } - - if (lstat(real_path, &st) != 0) { -- write_attach_message(ERR_MSG, "Failed to lstat %s : %s", real_path, SHIM_SYS_ERR(errno)); -+ ERROR("Failed to lstat %s : %d", real_path, SHIM_SYS_ERR(errno)); - return false; - } - - if (!S_ISFIFO(st.st_mode)) { -- write_attach_message(ERR_MSG, "attach fifo path \"%s\" must be an FIFO", real_path); -+ ERROR("attach fifo path \"%s\" must be an FIFO", real_path); - return false; - } - - if ((st.st_mode & 0777) != ATTACH_FIFOPATH_MODE) { -- write_attach_message(ERR_MSG, "attach fifo path \"%s\" permission invalid", real_path); -+ ERROR("attach fifo path \"%s\" permission invalid", real_path); - return false; - } - - if (st.st_uid != 0) { -- write_attach_message(ERR_MSG, "attach fifo path \"%s\" uid invalid", real_path); -+ ERROR("attach fifo path \"%s\" uid invalid", real_path); - return false; - } - -@@ -553,20 +555,20 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t - - r_count = isula_file_read_nointr(fd, tmp_buf, sizeof(tmp_buf) - 1); - if (r_count <= 0) { -- write_attach_message(ERR_MSG, "Failed to read msg from attach conn fd"); -+ ERROR("Failed to read msg from attach conn fd"); - goto out; - } - - // limit the number of attach connections to MAX_ATTACH_NUM - if (isula_linked_list_len(p->attach_fifos) >= MAX_ATTACH_NUM) { -- write_attach_message(ERR_MSG, "The number of attach connections exceeds the limit:%d, and this connection is rejected.", -+ ERROR("The number of attach connections exceeds the limit:%d, and this connection is rejected.", - MAX_ATTACH_NUM); - goto out; - } - - tmp_str_array = isula_string_split_to_multi(tmp_buf, ' '); - if (tmp_str_array->len != 3) { -- write_attach_message(ERR_MSG, "Invalid attach msg from isulad"); -+ ERROR("Invalid attach msg from isulad"); - goto out; - } - -@@ -575,7 +577,7 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t - } - - if (!valid) { -- write_attach_message(ERR_MSG, "Invalid attach fifo path from isulad"); -+ ERROR("Invalid attach fifo path from isulad"); - goto out; - } - -@@ -584,14 +586,14 @@ static int attach_cb(int fd, uint32_t events, void *cbdata, isula_epoll_descr_t - err = tmp_str_array->items[2]; - - if (add_attach_terminal_fifos(in, out, err, &fifofd_in, p) < 0) { -- write_attach_message(ERR_MSG, "Failed to add attach terminal fifos"); -+ ERROR("Failed to add attach terminal fifos"); - goto out; - } - - // attach stdin --> container stdin - ret = isula_epoll_add_handler(descr, fifofd_in, attach_stdin_cb, p); - if (ret != SHIM_OK) { -- write_attach_message(ERR_MSG, "add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); -+ ERROR("add fifofd_in fd %d to epoll loop failed:%d", fifofd_in, SHIM_SYS_ERR(errno)); - struct isula_linked_list *item = get_attach_fifo_item(fd, p->attach_fifos); - if (item != NULL && item->elem != NULL) { - remove_attach_terminal_fifos(descr, item); -@@ -616,13 +618,13 @@ static int do_attach_socket_accept(int fd, uint32_t events, void *cbdata, isula_ - - conn_fd = accept(p->attach_socket_fd, NULL, NULL); - if (conn_fd < 0) { -- write_attach_message(ERR_MSG, "accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); -+ ERROR("accept from fd %d failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); - return EPOLL_LOOP_HANDLE_CONTINUE; - } - - ret = isula_epoll_add_handler(descr, conn_fd, attach_cb, p); - if (ret != SHIM_OK) { -- write_attach_message(ERR_MSG, "add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); -+ ERROR("add recv_fd %d to epoll loop failed:%d", conn_fd, SHIM_SYS_ERR(errno)); - close(conn_fd); - return EPOLL_LOOP_HANDLE_CONTINUE; - } -@@ -637,13 +639,13 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epol - - conn_fd = accept(p->listen_fd, NULL, NULL); - if (conn_fd < 0) { -- write_message(ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); -+ ERROR("accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); - goto out; - } - - p->recv_fd = receive_fd(conn_fd); - if (check_fd(p->recv_fd) != true) { -- write_message(ERR_MSG, "check console fd failed"); -+ ERROR("check console fd failed"); - goto out; - } - -@@ -652,19 +654,19 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, isula_epol - // p->isulad_io->in ----> p->recv_fd - ret = isula_epoll_add_handler(descr, p->isulad_io->in, stdin_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); -+ ERROR("add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); - goto out; - } - // p->recv_fd ----> p->isulad_io->out - ret = isula_epoll_add_handler(descr, p->recv_fd, stdout_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); -+ ERROR("add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno)); - goto out; - } - // p->isulad_io->resize ----> p->recv_fd - ret = isula_epoll_add_handler(descr, p->isulad_io->resize, resize_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); -+ ERROR("add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno)); - goto out; - } - -@@ -724,7 +726,7 @@ static stdio_t *initialize_io(process_t *p) - */ - if ((pipe2(stdio_fd[0], O_CLOEXEC) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC) != 0) || - (pipe2(stdio_fd[2], O_CLOEXEC) != 0)) { -- write_message(ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); -+ ERROR("open pipe failed when init io:%d", SHIM_SYS_ERR(errno)); - goto failure; - } - -@@ -789,7 +791,7 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) - - fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd < 0) { -- write_message(ERR_MSG, "create socket failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("create socket failed:%d", SHIM_SYS_ERR(errno)); - goto failure; - } - -@@ -799,13 +801,13 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) - - ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); - if (ret < 0) { -- write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("bind console fd failed:%d", SHIM_SYS_ERR(errno)); - goto failure; - } - - ret = listen(fd, 2); - if (ret < 0) { -- write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("listen console fd failed:%d", SHIM_SYS_ERR(errno)); - goto failure; - } - -@@ -813,7 +815,7 @@ static int console_init(process_t *p, isula_epoll_descr_t *descr) - - ret = isula_epoll_add_handler(descr, p->listen_fd, task_console_accept, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); -+ ERROR("add listen_fd fd %d to epoll loop failed:%d", p->listen_fd, SHIM_SYS_ERR(errno)); - goto failure; - } - -@@ -831,7 +833,7 @@ static int open_terminal_io(process_t *p, isula_epoll_descr_t *descr) - - ret = new_temp_console_path(p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "get temp console sock path failed"); -+ ERROR("get temp console sock path failed"); - return SHIM_ERR; - } - -@@ -853,19 +855,19 @@ static int open_generic_io(process_t *p, isula_epoll_descr_t *descr) - // p->isulad_io->in ----> p->shim_io->in - ret = isula_epoll_add_handler(descr, p->isulad_io->in, stdin_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); -+ ERROR("add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - // p->shim_io->out ----> p->isulad_io->out - ret = isula_epoll_add_handler(descr, p->shim_io->out, stdout_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); -+ ERROR("add out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - // p->shim_io->err ----> p->isulad_io->err - ret = isula_epoll_add_handler(descr, p->shim_io->err, stderr_cb, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); -+ ERROR("add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - -@@ -896,22 +898,25 @@ static void *io_epoll_loop(void *data) - - ret = isula_epoll_open(&descr); - if (ret != 0) { -- write_message(ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -+ ERROR("epoll loop open failed:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("epoll loop open failed:%d", SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - - // sync fd: epoll loop will exit when recive sync fd event. - ret = isula_epoll_add_handler(&descr, p->sync_fd, sync_exit_cb, p); - if (ret != 0) { -- write_message(ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -+ ERROR("add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); -+ shim_set_error_message("add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - - if (p->state->attach_socket != NULL) { - ret = isula_epoll_add_handler(&descr, p->attach_socket_fd, do_attach_socket_accept, p); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -+ ERROR("add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); -+ shim_set_error_message("add attach_socket_fd %d to epoll loop failed:%d", p->attach_socket_fd, SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - } - -@@ -921,8 +926,9 @@ static void *io_epoll_loop(void *data) - ret = open_generic_io(p, &descr); - } - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno)); -- exit(EXIT_FAILURE); -+ ERROR("open io failed:%d", SHIM_SYS_ERR(errno)); -+ shim_append_error_message("open io failed:%d", SHIM_SYS_ERR(errno)); -+ error_exit(EXIT_FAILURE); - } - - (void)sem_post(&p->sem_mainloop); -@@ -933,8 +939,9 @@ static void *io_epoll_loop(void *data) - // 3. stdin fd read failed - ret = isula_epoll_loop(&descr, -1); - if (ret != 0) { -- write_message(ERR_MSG, "epoll loop failed"); -- exit(EXIT_FAILURE); -+ ERROR("epoll loop failed"); -+ shim_set_error_message("epoll loop failed"); -+ error_exit(EXIT_FAILURE); - } - - // use a timeout epoll loop to ensure complete data reception -@@ -943,7 +950,7 @@ static void *io_epoll_loop(void *data) - // 2. no event received within 100 milliseconds - ret = isula_epoll_loop(&descr, 100); - if (ret != 0) { -- write_message(ERR_MSG, "Repeat the epoll loop to ensure that all data is transferred"); -+ ERROR("Repeat the epoll loop to ensure that all data is transferred"); - } - - return NULL; -@@ -966,12 +973,12 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s - - log_term = isula_common_calloc_s(sizeof(log_terminal)); - if (log_term == NULL) { -- write_message(ERR_MSG, "Failed to calloc log_terminal"); -+ ERROR("Failed to calloc log_terminal"); - goto clean_out; - } - - if (pthread_rwlock_init(&log_term->log_terminal_rwlock, NULL) != 0) { -- write_message(ERR_MSG, "Failed to init isulad conf rwlock"); -+ ERROR("Failed to init isulad conf rwlock"); - goto clean_out; - } - -@@ -1047,25 +1054,25 @@ static int init_isulad_stdio(process_t *p) - - ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "Failed to open in isulad fd: %s", p->state->isulad_stdin); -+ ERROR("Failed to open in isulad fd: %s", p->state->isulad_stdin); - goto failure; - } - - ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "Failed to open out isulad fd: %s", p->state->isulad_stdout); -+ ERROR("Failed to open out isulad fd: %s", p->state->isulad_stdout); - goto failure; - } - - ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "Failed to open err isulad fd: %s", p->state->isulad_stderr); -+ ERROR("Failed to open err isulad fd: %s", p->state->isulad_stderr); - goto failure; - } - - ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize); - if (ret != SHIM_OK) { -- write_message(ERR_MSG, "Failed to open resize isulad fd: %s", p->state->resize_fifo); -+ ERROR("Failed to open resize isulad fd: %s", p->state->resize_fifo); - goto failure; - } - return SHIM_OK; -@@ -1105,13 +1112,13 @@ static int init_root_path(process_t *p) - // state_path: /run/isulad/runc/{container_id} - char *tmp_dir = strrchr(state_path, '/'); - if (tmp_dir == NULL) { -- write_message(ERR_MSG, "Invalid exec workdir"); -+ ERROR("Invalid exec workdir"); - return SHIM_ERR; - } - *tmp_dir = '\0'; - tmp_dir = strrchr(state_path, '/'); - if (tmp_dir == NULL) { -- write_message(ERR_MSG, "Invalid exec workdir"); -+ ERROR("Invalid exec workdir"); - return SHIM_ERR; - } - *tmp_dir = '\0'; -@@ -1119,12 +1126,12 @@ static int init_root_path(process_t *p) - - isula_buffer *buffer = isula_buffer_alloc(PATH_MAX); - if (buffer == NULL) { -- write_message(ERR_MSG, "Failed to malloc buffer\n"); -+ ERROR("Failed to malloc buffer\n"); - return SHIM_ERR; - } - - if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->runtime) < 0) { -- write_message(ERR_MSG, "Failed to append state_path\n"); -+ ERROR("Failed to append state_path\n"); - isula_buffer_free(buffer); - return SHIM_ERR; - } -@@ -1132,7 +1139,7 @@ static int init_root_path(process_t *p) - p->root_path = buffer->to_str(buffer); - isula_buffer_free(buffer); - if (strlen(p->root_path) > PATH_MAX) { -- write_message(ERR_MSG, "Root_path is too long\n"); -+ ERROR("Root_path is too long\n"); - return SHIM_ERR; - } - return SHIM_OK; -@@ -1180,7 +1187,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) - - p->sync_fd = eventfd(0, EFD_CLOEXEC); - if (p->sync_fd < 0) { -- write_message(ERR_MSG, "Failed to create eventfd: %s", strerror(errno)); -+ ERROR("Failed to create eventfd: %s", strerror(errno)); - goto failure; - } - -@@ -1197,7 +1204,7 @@ process_t *new_process(char *id, char *bundle, char *runtime) - // during the execution of isulad-shim, the current working directory will not change. - p->workdir = getcwd(NULL, 0); - if (p->workdir == NULL) { -- write_message(ERR_MSG, "get cwd failed when do create process"); -+ ERROR("get cwd failed when do create process"); - goto failure; - } - -@@ -1432,13 +1439,15 @@ int create_process(process_t *p) - int nread = -1; - - if (pipe2(exec_fd, O_CLOEXEC) != 0) { -- write_message(ERR_MSG, "create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); -+ ERROR("create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("create pipe failed when create process:%d", SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - - pid_t pid = fork(); - if (pid == (pid_t) -1) { -- write_message(ERR_MSG, "fork failed when create process:%d", SHIM_SYS_ERR(errno)); -+ ERROR("fork failed when create process:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("fork failed when create process:%d", SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - -@@ -1461,14 +1470,16 @@ int create_process(process_t *p) - /* block to wait runtime pid exit */ - ret = waitpid(pid, NULL, 0); - if (ret != pid) { -- write_message(ERR_MSG, "wait runtime failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("wait runtime failed:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("wait runtime failed:%d", SHIM_SYS_ERR(errno)); - ret = SHIM_ERR; - goto out; - } - - // if an error occurs in exec_runtime_process, jump directly to the out branch after waitpid. - if (nread > 0) { -- write_message(ERR_MSG, "%s", exec_buff); -+ ERROR("%s", exec_buff); -+ shim_set_error_message("%s", exec_buff); - ret = SHIM_ERR; - goto out; - } -@@ -1476,7 +1487,8 @@ int create_process(process_t *p) - /* save runtime pid */ - data = read_text_file("pid"); - if (data == NULL) { -- write_message(ERR_MSG, "read pid of runtime failed"); -+ ERROR("read pid of runtime failed"); -+ shim_set_error_message("read pid of runtime failed"); - goto out; - } - int ctr_pid = atoi(data); -@@ -1537,12 +1549,12 @@ static int waitpid_with_timeout(int ctr_pid, int *status, const uint64_t timeou - if (*status == CONTAINER_ACTION_REBOOT) { - nret = setenv("CONTAINER_ACTION", "reboot", 1); - if (nret != SHIM_OK) { -- write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); -+ WARN("set reboot action failed:%d", SHIM_SYS_ERR(errno)); - } - } else if (*status == CONTAINER_ACTION_SHUTDOWN) { - nret = setenv("CONTAINER_ACTION", "shutdown", 1); - if (nret != SHIM_OK) { -- write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); -+ WARN("set shutdown action failed:%d", SHIM_SYS_ERR(errno)); - } - } - return SHIM_OK; -@@ -1565,12 +1577,12 @@ static int wait_container_process_with_timeout(process_t *p, const uint64_t time - if (*status == CONTAINER_ACTION_REBOOT) { - ret = setenv("CONTAINER_ACTION", "reboot", 1); - if (ret != SHIM_OK) { -- write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno)); -+ WARN("set reboot action failed:%d", SHIM_SYS_ERR(errno)); - } - } else if (*status == CONTAINER_ACTION_SHUTDOWN) { - ret = setenv("CONTAINER_ACTION", "shutdown", 1); - if (ret != SHIM_OK) { -- write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno)); -+ WARN("set shutdown action failed:%d", SHIM_SYS_ERR(errno)); - } - } - return SHIM_OK; -@@ -1600,7 +1612,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const - // kill container process to ensure process_kill_all effective - nret = kill(p->ctr_pid, SIGKILL); - if (nret < 0 && errno != ESRCH) { -- write_message(ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid); -+ ERROR("Can not kill process (pid=%d) with SIGKILL, %d", p->ctr_pid, SHIM_SYS_ERR(errno)); - return SHIM_ERR; - } - } -@@ -1610,7 +1622,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const - // wait atmost 120 seconds - DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child); - if (nret != 0) { -- write_message(ERR_MSG, "Failed to wait all child after 120 seconds"); -+ ERROR("Failed to wait all child after 120 seconds"); - } - - process_delete(p); -@@ -1620,13 +1632,13 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const - - if (p->sync_fd > 0) { - if (eventfd_write(p->sync_fd, 1)) { -- write_message(ERR_MSG, "Failed to write sync fd"); -+ ERROR("Failed to write sync fd"); - } - } - - nret = pthread_join(tid_epoll, NULL); - if (nret != 0) { -- write_message(ERR_MSG, "Failed to join epoll loop thread"); -+ ERROR("Failed to join epoll loop thread"); - } - - close(p->sync_fd); -@@ -1638,7 +1650,8 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const - } - - if (ret == SHIM_ERR_TIMEOUT) { -- write_message(INFO_MSG, "Wait %d timeout", p->ctr_pid); -+ ERROR("Wait %d timeout", p->ctr_pid); -+ shim_set_error_message("Wait %d timeout", p->ctr_pid); - return SHIM_ERR_TIMEOUT; - } - -@@ -1653,13 +1666,15 @@ int prepare_attach_socket(process_t *p) - int ret = -1; - - if (strlen(p->state->attach_socket) >= sizeof(addr.sun_path)) { -- write_message(ERR_MSG, "Invalid attach socket path: %s", p->state->attach_socket); -+ ERROR("Invalid attach socket path: %s", p->state->attach_socket); -+ shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); - return SHIM_ERR; - } - - p->attach_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (p->attach_socket_fd < 0) { -- write_message(ERR_MSG, "Failed to create socket:%d", SHIM_SYS_ERR(errno)); -+ ERROR("Failed to create socket:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); - return SHIM_ERR; - } - -@@ -1669,13 +1684,15 @@ int prepare_attach_socket(process_t *p) - - ret = bind(p->attach_socket_fd, (struct sockaddr *)&addr, sizeof(addr)); - if (ret < 0) { -- write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("bind console fd failed:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); - return SHIM_ERR; - } - - ret = chmod(p->state->attach_socket, SOCKET_DIRECTORY_MODE); - if (ret != 0) { -- write_message(ERR_MSG, "Failed to chmod for socket: %s", p->state->attach_socket); -+ ERROR("Failed to chmod for socket: %s", p->state->attach_socket); -+ shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); - return SHIM_ERR; - } - -@@ -1687,7 +1704,8 @@ int prepare_attach_socket(process_t *p) - // The maximum number of attach we allow here is MAX_ATTACH_NUM, so just use it directly - ret = listen(p->attach_socket_fd, MAX_ATTACH_NUM); - if (ret < 0) { -- write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno)); -+ ERROR("listen console fd failed:%d", SHIM_SYS_ERR(errno)); -+ shim_set_error_message("Invalid attach socket path: %s", p->state->attach_socket); - return SHIM_ERR; - } - return SHIM_OK; -diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c -index fbb779f7..2b12d173 100644 ---- a/src/daemon/modules/runtime/isula/isula_rt_ops.c -+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c -@@ -69,6 +69,16 @@ - - // handle string from stderr output. - typedef int(*handle_output_callback_t)(const char *output); -+typedef struct { -+ bool fg; -+ const char *id; -+ char *workdir; -+ const char *bundle; -+ const char *runtime_cmd; -+ int *exit_code; -+ char *timeout; -+ int *shim_exit_code; -+} shim_create_args; - - static void copy_process(shim_client_process_state *p, defs_process *dp) - { -@@ -154,9 +164,10 @@ static void file_read_int(const char *fname, int *val) - free(sint); - } - --static void get_err_message(char *buf, int buf_size, const char *workdir, const char *file) -+static int get_err_message(char *buf, int buf_size, const char *workdir, const char *file) - { - int nret; -+ int ret = 0; - char fname[PATH_MAX] = { 0 }; - FILE *fp = NULL; - char *pline = NULL; -@@ -166,12 +177,12 @@ static void get_err_message(char *buf, int buf_size, const char *workdir, const - nret = snprintf(fname, PATH_MAX, "%s/%s", workdir, file); - if (nret < 0 || (size_t)nret >= PATH_MAX) { - ERROR("failed make full path %s/%s", workdir, file); -- return; -+ return ret; - } - - fp = util_fopen(fname, "r"); - if (fp == NULL) { -- return; -+ return ret; - } - - while (getline(&pline, &length, fp) != -1) { -@@ -199,48 +210,58 @@ static void get_err_message(char *buf, int buf_size, const char *workdir, const - fclose(fp); - - if (lines[2] != NULL) { -- (void)snprintf(buf, buf_size, "%s%s%s", lines[0], lines[1], lines[2]); -+ nret = snprintf(buf, buf_size, "%s%s%s", lines[0], lines[1], lines[2]); -+ ret += nret; - } else if (lines[1] != NULL) { -- (void)snprintf(buf, buf_size, "%s%s", lines[0], lines[1]); -+ nret = snprintf(buf, buf_size, "%s%s", lines[0], lines[1]); -+ ret += nret; - } else if (lines[0] != NULL) { -- (void)snprintf(buf, buf_size, "%s", lines[0]); -+ nret = snprintf(buf, buf_size, "%s", lines[0]); -+ ret += nret; - } - - UTIL_FREE_AND_SET_NULL(pline); - UTIL_FREE_AND_SET_NULL(lines[0]); - UTIL_FREE_AND_SET_NULL(lines[1]); - UTIL_FREE_AND_SET_NULL(lines[2]); -+ return ret; - } - --static void show_shim_runtime_errlog(const char *workdir) -+static void show_runtime_errlog(const char *workdir) - { - char buf[BUFSIZ] = { 0 }; - char buf1[SHIM_LOG_SIZE] = { 0 }; -- char buf2[SHIM_LOG_SIZE] = { 0 }; -+ int nret; - - if (g_isulad_errmsg != NULL) { - return; - } - -- get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json"); -- get_err_message(buf2, sizeof(buf2), workdir, "log.json"); -- ERROR("shim-log: %s", buf1); -- ERROR("runtime-log: %s", buf2); -- (void)snprintf(buf, sizeof(buf), "shim-log error: %s\nruntime-log error: %s\n", buf1, buf2); -+ nret = get_err_message(buf1, sizeof(buf1), workdir, "log.json"); -+ if (nret == 0) { -+ ERROR("empty runtime-log : %s", workdir); -+ return; -+ } -+ ERROR("runtime-log: %s", buf1); -+ (void)snprintf(buf, sizeof(buf), "runtime-log error: %s\n", buf1); - isulad_set_error_message(buf); - } - --static void show_shim_attach_errlog(const char *workdir) -+static void show_shim_errlog(const int fd) - { -- char buf[SHIM_LOG_SIZE] = { 0 }; -+ int num; -+ char buf[BUFSIZ] = { 0 }; - - if (g_isulad_errmsg != NULL) { - return; - } - -- get_err_message(buf, sizeof(buf), workdir, "attach-log.json"); -- ERROR("shim-log: %s", buf); -- isulad_set_error_message("shim-log error:\n%s\n", buf); -+ num = util_read_nointr(fd, buf, sizeof(buf) - 1); -+ if (num < 0) { -+ SYSERROR("Failed to read err msg from shim stderr"); -+ return; -+ } -+ isulad_set_error_message(buf); - } - - bool rt_isula_detect(const char *runtime) -@@ -818,12 +839,48 @@ static int status_to_exit_code(int status) - return exit_code; - } - -+static int get_engine_routine_log_info(char **engine_log_path, char **log_level) -+{ -+ int ret = 0; -+ struct service_arguments *args = NULL; -+ -+ if (isulad_server_conf_rdlock() != 0) { -+ return -1; -+ } -+ -+ args = conf_get_server_conf(); -+ if (args == NULL) { -+ ERROR("Failed to get isulad server config"); -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ *engine_log_path = conf_get_engine_log_file(); -+ if (*engine_log_path == NULL) { -+ ERROR("Log fifo path is NULL"); -+ ret = -1; -+ goto unlock_out; -+ } -+ -+ *log_level = util_strdup_s(args->json_confs->log_level); -+ if (*engine_log_path == NULL) { -+ ERROR("Log level is NULL"); -+ ret = -1; -+ goto unlock_out; -+ } -+ -+unlock_out: -+ if (isulad_server_conf_unlock()) { -+ ret = -1; -+ } -+ return ret; -+} -+ - /* - exit_code records the exit code of the container, obtained by reading the stdout of isulad-shim; - shim_exit_code records the exit code of isulad-shim, obtained through waitpid; - */ --static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd, -- int *exit_code, const char *timeout, int *shim_exit_code) -+static int shim_create(shim_create_args *args) - { - pid_t pid = 0; - int shim_stderr_pipe[2] = { -1, -1 }; -@@ -833,22 +890,29 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - char exec_buff[BUFSIZ + 1] = { 0 }; - char fpid[PATH_MAX] = { 0 }; - const char *params[PARAM_NUM] = { 0 }; -+ __isula_auto_free char *engine_log_path = NULL; -+ __isula_auto_free char *log_level = NULL; - int i = 0; - int status = 0; - int nret = 0; - - params[i++] = SHIM_BINARY; -- params[i++] = id; -- params[i++] = bundle; -- params[i++] = runtime_cmd; -+ params[i++] = args->id; -+ params[i++] = args->bundle; -+ params[i++] = args->runtime_cmd; - params[i++] = "info"; - // execSync timeout -- if (timeout != NULL) { -- params[i++] = timeout; -+ if (args->timeout != NULL) { -+ params[i++] = args->timeout; - } - runtime_exec_param_dump(params); - -- nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir); -+ if (get_engine_routine_log_info(&engine_log_path, &log_level) != 0) { -+ ERROR("failed to get engine log path"); -+ return -1; -+ } -+ -+ nret = snprintf(fpid, sizeof(fpid), "%s/shim-pid", args->workdir); - if (nret < 0 || (size_t)nret >= sizeof(fpid)) { - ERROR("failed make shim-pid full path"); - return -1; -@@ -875,12 +939,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - } - - if (pid == (pid_t)0) { -- if (chdir(workdir) < 0) { -- (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", id, workdir); -+ if (chdir(args->workdir) < 0) { -+ (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", args->id, args->workdir); - exit(EXIT_FAILURE); - } - -- if (fg) { -+ if (args->fg) { - // child process, dup2 shim_stdout_pipe[1] to STDOUT - if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) { - (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno)); -@@ -896,18 +960,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char - - // clear NOTIFY_SOCKET from the env to adapt runc create - if (unsetenv("NOTIFY_SOCKET") != 0) { -- (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); -+ (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno)); - exit(EXIT_FAILURE); - } - - pid = fork(); - if (pid < 0) { -- (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", id, strerror(errno)); -+ (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno)); - _exit(EXIT_FAILURE); - } - if (pid != 0) { - if (file_write_int(fpid, pid) != 0) { -- (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", id, fpid, pid); -+ (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid); - } - _exit(EXIT_SUCCESS); - } -@@ -918,7 +982,7 @@ realexec: - close(shim_stdout_pipe[0]); - - if (setsid() < 0) { -- (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", id, getpid()); -+ (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", args->id, getpid()); - exit(EXIT_FAILURE); - } - -@@ -927,6 +991,16 @@ realexec: - exit(EXIT_FAILURE); - } - -+ if (setenv("ISULAD_SHIIM_LOG_PATH", engine_log_path, 1) != 0) { -+ (void)dprintf(shim_stderr_pipe[1], "%s: failed to set ISULAD_SHIIM_LOG_PATH env for process %d", args->id, getpid()); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (setenv("ISULAD_SHIIM_LOG_LEVEL", log_level, 1) != 0) { -+ (void)dprintf(shim_stderr_pipe[1], "%s: failed to set ISULAD_SHIIM_LOG_LEVEL env for process %d", args->id, getpid()); -+ exit(EXIT_FAILURE); -+ } -+ - execvp(SHIM_BINARY, (char * const *)params); - (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno)); - exit(EXIT_FAILURE); -@@ -951,38 +1025,43 @@ realexec: - goto out; - } - -- *shim_exit_code = status_to_exit_code(status); -- if (*shim_exit_code != 0) { -- ERROR("Isulad-shim exit error"); -+ *(args->shim_exit_code) = status_to_exit_code(status); -+ if (*(args->shim_exit_code) != 0) { -+ ERROR("Isulad-shim exit error : %d", *(args->shim_exit_code)); -+ isulad_set_error_message("Isulad-shim exit error : %d, please get more information from log", *(args->shim_exit_code)); - ret = -1; - goto out; - } - - // exit_code is NULL when command is create. -- if (exit_code == NULL) { -+ if (args->exit_code == NULL) { - goto out; - } - - // when exec in background, exit code is shim exit code -- if (!fg) { -- *exit_code = *shim_exit_code; -+ if (!args->fg) { -+ *(args->exit_code) = *(args->shim_exit_code); - goto out; - } -- ret = util_read_nointr(shim_stdout_pipe[0], exit_code, sizeof(int)); -+ ret = util_read_nointr(shim_stdout_pipe[0], args->exit_code, sizeof(int)); - if (ret <= 0) { -- *exit_code = 137; -+ *(args->exit_code) = 137; - } - ret = 0; - - out: -- close(shim_stderr_pipe[0]); - close(shim_stdout_pipe[0]); - if (ret != 0) { -- show_shim_runtime_errlog(workdir); -- if (timeout != NULL) { -+ if (*(args->shim_exit_code) == 0) { -+ show_runtime_errlog(args->workdir); -+ } else { -+ show_shim_errlog(shim_stderr_pipe[0]); -+ } -+ if (args->timeout != NULL) { - kill(pid, SIGKILL); /* can kill other process? */ - } - } -+ close(shim_stderr_pipe[0]); - - return ret; - } -@@ -1061,6 +1140,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ - char workdir[PATH_MAX] = { 0 }; - char attach_socket[PATH_MAX] = { 0 }; - shim_client_process_state p = { 0 }; -+ shim_create_args args = { 0 }; - int shim_exit_code = 0; - int nret = 0; - -@@ -1108,7 +1188,15 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_ - } - - get_runtime_cmd(runtime, &cmd); -- ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, NULL, &shim_exit_code); -+ args.fg = false; -+ args.id = id; -+ args.workdir = workdir; -+ args.bundle = params->bundle; -+ args.runtime_cmd = cmd; -+ args.exit_code = NULL; -+ args.timeout = NULL; -+ args.shim_exit_code = &shim_exit_code; -+ ret = shim_create(&args); - if (ret != 0) { - runtime_call_delete_force(workdir, runtime, id); - ERROR("%s: failed create shim process", id); -@@ -1185,7 +1273,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t - ret = 0; - out: - if (ret != 0) { -- show_shim_runtime_errlog(workdir); -+ show_runtime_errlog(workdir); - shim_kill_force(workdir); - } - return ret; -@@ -1363,6 +1451,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - int pid = 0; - char bundle[PATH_MAX] = { 0 }; - char workdir[PATH_MAX] = { 0 }; -+ shim_create_args args = { 0 }; - char *timeout = NULL; - int shim_exit_code = 0; - -@@ -1411,7 +1500,15 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - } - } - -- ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, timeout, &shim_exit_code); -+ args.fg = fg_exec(params); -+ args.id = id; -+ args.workdir = workdir; -+ args.bundle = bundle; -+ args.runtime_cmd = cmd; -+ args.exit_code = exit_code; -+ args.timeout = timeout; -+ args.shim_exit_code = &shim_exit_code; -+ ret = shim_create(&args); - if (shim_exit_code == SHIM_EXIT_TIMEOUT) { - ret = -1; - isulad_set_error_message("Exec container error;exec timeout"); -@@ -1432,7 +1529,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p - - errlog_out: - if (ret != 0) { -- show_shim_runtime_errlog(workdir); -+ show_runtime_errlog(workdir); - } - - if (timeout != NULL) { -@@ -1559,13 +1656,13 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ - - if (id == NULL || runtime == NULL || params == NULL) { - ERROR("Null argument"); -- return -1; -+ goto err_out; - } - - ret = snprintf(workdir, sizeof(workdir), "%s/%s", params->state, id); - if (ret < 0 || (size_t)ret >= sizeof(workdir)) { - ERROR("Failed join exec full path"); -- return -1; -+ goto err_out; - } - - // the communication format between isulad and isulad-shim attach is: -@@ -1573,19 +1670,19 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ - len = snprintf(buf, sizeof(buf), "%s %s %s", params->stdin, params->stdout, params->stderr); - if (len < 0 || (size_t)len >= sizeof(buf)) { - ERROR("Failed to snprintf string"); -- return -1; -+ goto err_out; - } - - ret = snprintf(attach_socket, sizeof(attach_socket), "%s/%s", workdir, ATTACH_SOCKET); - if (ret < 0 || (size_t)ret >= sizeof(attach_socket)) { - ERROR("Failed to get full attach socket path"); -- return -1; -+ goto err_out; - } - - ret = get_attach_socketfd(attach_socket, &socket_fd); - if (ret < 0) { - ERROR("Failed to get attach socketfd"); -- return -1; -+ goto err_out; - } - - DEBUG("write %s to attach fd", buf); -@@ -1593,16 +1690,19 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_ - ret = isula_file_write_nointr(socket_fd, buf, len); - if (ret < 0) { - SYSERROR("Failed to write attach isulad fd"); -- return -1; -+ goto err_out; - } - - status_code = get_container_attach_statuscode(workdir, socket_fd); - if (status_code < 0) { -- show_shim_attach_errlog(workdir); -- return -1; -+ ERROR("Failed to attach container io, get more information from log"); -+ goto err_out; - } - - return 0; -+err_out: -+ isulad_set_error_message("Failed to attach container io, get more information from log"); -+ return -1; - } - - static int to_engine_resources_unified(const host_config *hostconfig, shim_client_cgroup_resources *cr) --- -2.42.0 -