From f084483fae4e051b872a4bc812ffa3dc73039c76 Mon Sep 17 00:00:00 2001 From: wo_cow Date: Fri, 31 Mar 2023 19:07:09 +0800 Subject: [PATCH 29/30] stackprobe: fix inaccurate call stack count. add sample_perioud param --- .../src/stackprobe/conf/stackprobe.conf | 1 + .../src/stackprobe/conf/stackprobe_conf.h | 3 + .../src/stackprobe/conf/stackprobe_config.c | 11 ++ .../ebpf.probe/src/stackprobe/flame_graph.c | 77 +------- .../ebpf.probe/src/stackprobe/flame_graph.h | 5 +- .../ebpf.probe/src/stackprobe/java_support.c | 2 +- .../src/stackprobe/jvmti/jvm_agent.c | 2 +- .../extends/ebpf.probe/src/stackprobe/stack.h | 3 +- .../src/stackprobe/stack_bpf/stackprobe_bpf.h | 4 +- .../ebpf.probe/src/stackprobe/stackprobe.c | 177 +++++++++++++----- .../ebpf.probe/src/stackprobe/stackprobe.h | 2 + .../extends/ebpf.probe/src/stackprobe/svg.h | 9 + .../ebpf.probe/src/stackprobe/symbol.c | 1 - 13 files changed, 175 insertions(+), 122 deletions(-) diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf index 0b3d0ac..4d50eb2 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf +++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf @@ -2,6 +2,7 @@ general = { whitelist_enable = false; period = 180; # unit is second + sample_period = 10; # unit is ms log_dir = "/var/log/gala-gopher/stacktrace/logs"; svg_dir = "/var/log/gala-gopher/stacktrace"; flame_dir = "/var/log/gala-gopher/flamegraph"; diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h index d872b4d..a1b9913 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h @@ -23,6 +23,8 @@ #define PROC_MAX_RANGE 64 #define PERIOD_MAX 600 #define PERIOD_MIN 30 +#define SAMPLE_PERIOD_MAX 1000 +#define SAMPLE_PERIOD_MIN 10 typedef enum { SWITCH_ON = 0, @@ -31,6 +33,7 @@ typedef enum { typedef struct { int period; + int samplePeriod; char logDir[PATH_LEN]; char svgDir[PATH_LEN]; char flameDir[PATH_LEN]; diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c index 282cbbd..ed6ae19 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c @@ -184,6 +184,17 @@ static int configLoadGeneral(void *config, config_setting_t *settings) } generalConfig->period = intVal; + ret = config_setting_lookup_int(settings, "sample_period", &intVal); + if (ret == 0) { + ERROR("[STACKPROBE]: load config for general sample_period failed.\n"); + return -1; + } + if (intVal < SAMPLE_PERIOD_MIN || intVal > SAMPLE_PERIOD_MAX) { + ERROR("[STACKPROBE]: Please check config for general sample_period, val shold inside 10~1000.\n"); + return -1; + } + generalConfig->samplePeriod = intVal; + ret = config_setting_lookup_string(settings, "log_dir", &strVal); if (ret == 0) { ERROR("[STACKPROBE]: load config for general log_dir failed.\n"); diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c index fd03699..cb70ec0 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c @@ -25,7 +25,6 @@ #include #include #include -#include #ifdef BPF_PROG_KERN #undef BPF_PROG_KERN @@ -38,16 +37,7 @@ #include "bpf.h" #include "flame_graph.h" -#define POST_MAX_STEP_SIZE 1048576 // 1M -static int g_post_max = POST_MAX_STEP_SIZE; - -struct post_info_s { - int post_flag; - int remain_size; - char *buf_start; - char *buf; - CURL *curl; -}; +extern int g_post_max; struct MemoryStruct { char *memory; @@ -108,13 +98,6 @@ static FILE *__open_flame_graph_fp(struct stack_svg_mng_s *svg_mng) return sfg->fp; } -static FILE *__get_flame_graph_fp(struct stack_svg_mng_s *svg_mng) -{ - struct stack_flamegraph_s *sfg; - - sfg = &(svg_mng->flame_graph); - return sfg->fp; -} static void __mkdir_flame_graph_path(struct stack_svg_mng_s *svg_mng) { @@ -192,51 +175,7 @@ static void __reopen_flame_graph_file(struct stack_svg_mng_s *svg_mng) __set_flame_graph_flags(svg_mng, FLAME_GRAPH_NEW); } -#define HISTO_TMP_LEN (2 * STACK_SYMBS_LEN) -static char __histo_tmp_str[HISTO_TMP_LEN]; - -static int __do_wr_stack_histo(struct stack_svg_mng_s *svg_mng, - struct stack_trace_histo_s *stack_trace_histo, int first, struct post_info_s *post_info) -{ - FILE *fp = __get_flame_graph_fp(svg_mng); - if (!fp) { - ERROR("[FLAMEGRAPH]: Invalid fp.\n"); - return -1; - } - __histo_tmp_str[0] = 0; - - if (first) { - (void)snprintf(__histo_tmp_str, HISTO_TMP_LEN, "%s %llu", - stack_trace_histo->stack_symbs_str, stack_trace_histo->count); - } else { - (void)snprintf(__histo_tmp_str, HISTO_TMP_LEN, "\n%s %llu", - stack_trace_histo->stack_symbs_str, stack_trace_histo->count); - } - - if (post_info->post_flag) { - int written = post_info->buf - post_info->buf_start; - int ret = __snprintf(&post_info->buf, post_info->remain_size, &post_info->remain_size, "%s", __histo_tmp_str); - if (ret < 0) { - int new_post_max = g_post_max + POST_MAX_STEP_SIZE; - char *temp = (char *)realloc(post_info->buf_start, new_post_max); - if(temp == NULL) { - ERROR("[FLAMEGRAPH]: Not enough post memory (realloc failed), current capacity is %d.\n", - g_post_max); - } else { - post_info->buf_start = temp; - post_info->buf = post_info->buf_start + written; - post_info->remain_size += POST_MAX_STEP_SIZE; - g_post_max = new_post_max; - INFO("[FLAMEGRAPH]: post memory realloc to %d\n", g_post_max); - (void)__snprintf(&post_info->buf, post_info->remain_size, &post_info->remain_size, "%s", __histo_tmp_str); - } - } - } - - (void)fputs(__histo_tmp_str, fp); - return 0; -} static size_t __write_memory_cb(void *contents, size_t size, size_t nmemb, void *userp) { @@ -362,8 +301,7 @@ static void __init_curl_handle(struct post_server_s *post_server, struct post_in } } -static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, - struct post_server_s *post_server, int en_type) +static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct post_server_s *post_server, int en_type) { int first_flag = 0; struct post_info_s post_info = {.remain_size = g_post_max, .post_flag = 0}; @@ -374,11 +312,8 @@ static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_tra __init_curl_handle(post_server, &post_info); - struct stack_trace_histo_s *item, *tmp; - H_ITER(head, item, tmp) { - (void)__do_wr_stack_histo(svg_mng, item, first_flag, &post_info); - first_flag = 0; - } + iter_histo_tbl(svg_mng, en_type, &first_flag, &post_info); + if (post_info.post_flag) { __curl_post(post_server, &post_info, en_type); } @@ -389,10 +324,10 @@ static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_tra #endif -void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type, +void wr_flamegraph(struct stack_svg_mng_s *svg_mng, int en_type, struct post_server_s *post_server) { - __do_wr_flamegraph(svg_mng, head, post_server, en_type); + __do_wr_flamegraph(svg_mng, post_server, en_type); if (is_svg_tmout(svg_mng)) { (void)create_svg_file(svg_mng, diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h index ea29107..d596328 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h @@ -20,8 +20,9 @@ #include "svg.h" #include "stackprobe.h" -void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type, - struct post_server_s *post_server); +int do_wr_stack_histo(struct stack_svg_mng_s *svg_mng, + struct stack_trace_histo_s *stack_trace_histo, int first, struct post_info_s *post_info); +void wr_flamegraph(struct stack_svg_mng_s *svg_mng, int en_type, struct post_server_s *post_server); int set_flame_graph_path(struct stack_svg_mng_s *svg_mng, const char* path, const char *flame_name); int set_post_server(struct post_server_s *post_server, const char *pyroscopeServer); void clean_post_server(); diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/java_support.c b/src/probes/extends/ebpf.probe/src/stackprobe/java_support.c index 48b4386..e7088fb 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/java_support.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/java_support.c @@ -63,7 +63,7 @@ static struct jvm_agent_hash_t *jvm_agent_head = NULL; #define FIND_JAVA_PROC_COMM "ps -e -o pid,comm | grep java | awk '{print $1}'" #define PROC_COMM "/usr/bin/cat /proc/%u/comm 2> /dev/null" #define ATTACH_BIN_PATH "/opt/gala-gopher/extend_probes/jvm_attach" -#define ATTACH_CMD "%s %u %u load %s true %s" // jvm_attach load /tmp/jvm_agent.so true /tmp/java-symbolization-123 +#define ATTACH_CMD "%s %u %u load %s true %s" // jvm_attach load /tmp/jvm_agent.so true /tmp/java-sym-123 #define HOST_SO_DIR "/opt/gala-gopher/extend_probes" #define AGENT_SO_FILE "jvm_agent.so" #define HOST_JAVA_SYM_PATH "/proc/%u/root/tmp/java-sym-%u/%s" // /proc//root/tmp/java-sym-/java-symbols.bin diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/jvmti/jvm_agent.c b/src/probes/extends/ebpf.probe/src/stackprobe/jvmti/jvm_agent.c index 21b1780..f8450ac 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/jvmti/jvm_agent.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/jvmti/jvm_agent.c @@ -103,7 +103,7 @@ void get_class_name_from_csig(char *dest, size_t dest_size, const char *sig) { for(i = 0; i < (dest_size - 1) && src[i]; i++) { char c = src[i]; if (c == '/') c = '.'; - if (c == ';') break; + if (c == ';' || c == '$') break; dest[i] = c; } dest[i] = 0; diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stack.h b/src/probes/extends/ebpf.probe/src/stackprobe/stack.h index 82fda3f..5b6f7e3 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/stack.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/stack.h @@ -20,12 +20,11 @@ #include "common.h" #define AGGRE_PERIOD (1 * 30 * 1000) // 30s -#define SAMPLE_PERIOD (10) // 10ms #define TMOUT_PERIOD (AGGRE_PERIOD / 1000) // Second as unit #define PROC_CACHE_MAX_COUNT 100 // Cache 100 proc symbols #define DIV_ROUND_UP(NUM, DEN) ((NUM + DEN - 1) / DEN) -#define PERCPU_SAMPLE_COUNT (2 * DIV_ROUND_UP(AGGRE_PERIOD, SAMPLE_PERIOD)) +#define MAX_PERCPU_SAMPLE_COUNT (2 * DIV_ROUND_UP(AGGRE_PERIOD, 10)) // samplePeriod as 10ms struct convert_data_t { u32 whitelist_enable; diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stack_bpf/stackprobe_bpf.h b/src/probes/extends/ebpf.probe/src/stackprobe/stack_bpf/stackprobe_bpf.h index 5626bd5..1ab3b13 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/stack_bpf/stackprobe_bpf.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/stack_bpf/stackprobe_bpf.h @@ -54,7 +54,7 @@ struct { __uint(type, BPF_MAP_TYPE_STACK_TRACE); __uint(key_size, sizeof(u32)); __uint(value_size, PERF_MAX_STACK_DEPTH * sizeof(u64)); - __uint(max_entries, PERCPU_SAMPLE_COUNT); + __uint(max_entries, MAX_PERCPU_SAMPLE_COUNT); } stackmap_a SEC(".maps"); @@ -63,7 +63,7 @@ struct { __uint(type, BPF_MAP_TYPE_STACK_TRACE); __uint(key_size, sizeof(u32)); __uint(value_size, PERF_MAX_STACK_DEPTH * sizeof(u64)); - __uint(max_entries, PERCPU_SAMPLE_COUNT); + __uint(max_entries, MAX_PERCPU_SAMPLE_COUNT); } stackmap_b SEC(".maps"); diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c index eafe5ad..dcc26c7 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c @@ -64,6 +64,8 @@ #define IS_IEG_ADDR(addr) ((addr) != 0xcccccccccccccccc && (addr) != 0xffffffffffffffff) #define MEMLEAK_SEC_NUM 4 +#define HISTO_TMP_LEN (2 * STACK_SYMBS_LEN) +#define POST_MAX_STEP_SIZE 1048576 // 1M #define BPF_GET_MAP_FD(obj, map_name) \ ({ \ @@ -85,7 +87,7 @@ __ret; \ }) -typedef int (*AttachFunc)(struct svg_stack_trace_s *svg_st); +typedef int (*AttachFunc)(struct svg_stack_trace_s *svg_st, StackprobeConfig *conf); typedef int (*PerfProcessFunc)(void *ctx, int cpu, void *data, u32 size); enum pid_state_t { @@ -116,6 +118,9 @@ struct bpf_link_hash_t { struct bpf_link_hash_value v; // value }; + +static char __histo_tmp_str[HISTO_TMP_LEN]; +int g_post_max = POST_MAX_STEP_SIZE; static struct probe_params params = {.period = DEFAULT_PERIOD}; static volatile sig_atomic_t g_stop; static struct stack_trace_s *g_st = NULL; @@ -198,12 +203,10 @@ static void destroy_proc_cache_tbl(struct stack_trace_s *st) return; } - struct proc_cache_s *proc_hash_tbl = st->proc_cache; struct proc_cache_s *item, *tmp; - - H_ITER(proc_hash_tbl, item, tmp) { + H_ITER(st->proc_cache, item, tmp) { __destroy_proc_cache(item); - H_DEL(proc_hash_tbl, item); + H_DEL(st->proc_cache, item); (void)free(item); } st->proc_cache = NULL; @@ -322,7 +325,7 @@ static struct raw_stack_trace_s *create_raw_stack_trace(struct stack_trace_s *st { struct raw_stack_trace_s *raw_stack_trace; - size_t stack_size = st->cpus_num * PERCPU_SAMPLE_COUNT; + size_t stack_size = st->cpus_num * MAX_PERCPU_SAMPLE_COUNT; size_t mem_size = sizeof(struct raw_stack_trace_s); mem_size += (stack_size * sizeof(struct raw_trace_s)); @@ -353,9 +356,19 @@ static int add_raw_stack_id(struct raw_stack_trace_s *raw_st, struct raw_trace_s #endif +#define STACK_LAYER_ELSE 0 +#define STACK_LAYER_1ST 1 +#define STACK_LAYER_2ND 2 // only for Java +#define STACK_LAYER_3RD 3 // only for Java + +// For deep call stacks (especially prone to Java programs), it is easy to sample incomplete call stacks. +// If the function name at the first layer of the call stack contains ".", +// it means that this is must be an incomplete call stack. +// We query whether the first two layers of this call stack are contained in other call stacks (eg. A), +// and then count this call on the A call stack. #if 1 static int __stack_addrsymbs2string(struct proc_symbs_s *proc_symbs, struct addr_symb_s *addr_symb, - int first, char *p, int size) + int *layer, char *p, int size) { int ret; char *symb; @@ -366,21 +379,33 @@ static int __stack_addrsymbs2string(struct proc_symbs_s *proc_symbs, struct addr char *cur_p = p; int len = size; -#if 1 - symb = addr_symb->sym ?: addr_symb->mod; - if (first) { - if (proc_symbs->pod[0] != 0) { - ret = __snprintf(&cur_p, len, &len, "[Pod]%s; ", proc_symbs->pod); - } - if (proc_symbs->container_name[0] != 0) { - ret = __snprintf(&cur_p, len, &len, "[Con]%s; ", proc_symbs->container_name); + if (addr_symb->sym == NULL) { + return 0; + } + symb = addr_symb->sym; + + if (*layer == STACK_LAYER_1ST) { + if (strstr(symb, ".") != NULL) { + ret = __snprintf(&cur_p, len, &len, "; %s", symb); + *layer = STACK_LAYER_2ND; + } else { + if (proc_symbs->pod[0] != 0) { + ret = __snprintf(&cur_p, len, &len, "[Pod]%s; ", proc_symbs->pod); + } + if (proc_symbs->container_name[0] != 0) { + ret = __snprintf(&cur_p, len, &len, "[Con]%s; ", proc_symbs->container_name); + } + ret = __snprintf(&cur_p, len, &len, "[%d]%s; %s", proc_symbs->proc_id, proc_symbs->comm, symb); + *layer = STACK_LAYER_ELSE; } - ret = __snprintf(&cur_p, len, &len, "[%d]%s; %s", proc_symbs->proc_id, proc_symbs->comm, symb); + } else if (*layer == STACK_LAYER_2ND) { + ret = __snprintf(&cur_p, len, &len, "; %s", symb); + *layer = STACK_LAYER_3RD; } else { ret = __snprintf(&cur_p, len, &len, "; %s", symb); + *layer = STACK_LAYER_ELSE; } -#endif if (ret < 0) { return -1; } @@ -391,7 +416,7 @@ static int __stack_symbs2string(struct stack_symbs_s *stack_symbs, struct proc_s char symbos_str[], size_t size) { int len; - int first_flag = 1; + int layer = STACK_LAYER_1ST; int remain_len = size; char *pos = symbos_str; struct addr_symb_s *addr_symb; @@ -399,28 +424,19 @@ static int __stack_symbs2string(struct stack_symbs_s *stack_symbs, struct proc_s for (int i = 0; i < PERF_MAX_STACK_DEPTH; i++) { addr_symb = &(stack_symbs->user_stack_symbs[i]); if (addr_symb->orign_addr != 0) { - len = __stack_addrsymbs2string(proc_symbs, addr_symb, first_flag, pos, remain_len); - if (len < 0) { + len = __stack_addrsymbs2string(proc_symbs, addr_symb, &layer, pos, remain_len); + if (layer == STACK_LAYER_3RD) { return -1; } - remain_len -= len; - pos += len; - first_flag = 0; - } - } - - for (int i = 0; i < PERF_MAX_STACK_DEPTH; i++) { - addr_symb = &(stack_symbs->kern_stack_symbs[i]); - if (addr_symb->orign_addr != 0) { - len = __stack_addrsymbs2string(proc_symbs, addr_symb, first_flag, pos, remain_len); if (len < 0) { return -1; } remain_len -= len; pos += len; - first_flag = 0; } } + + symbos_str[size - 1] = 0; return 0; } @@ -428,7 +444,8 @@ static int add_stack_histo(struct stack_trace_s *st, struct stack_symbs_s *stack struct proc_symbs_s *proc_symbs, enum stack_svg_type_e en_type, s64 count) { char str[STACK_SYMBS_LEN]; - struct stack_trace_histo_s *item = NULL, *new_item; + struct stack_trace_histo_s *item = NULL, *new_item = NULL; + struct stack_trace_histo_s *tmp; str[0] = 0; if (__stack_symbs2string(stack_symbs, proc_symbs, str, STACK_SYMBS_LEN)) { @@ -448,6 +465,20 @@ static int add_stack_histo(struct stack_trace_s *st, struct stack_symbs_s *stack return 0; } + // Java incomplete call stack merge + if (str[0] == ';') { + char tmp_str[__FUNC_NAME_LEN] = {0}; + (void)snprintf(tmp_str, __FUNC_NAME_LEN, "[%d]", proc_symbs->proc_id); + H_ITER(st->svg_stack_traces[en_type]->histo_tbl, item, tmp) { + if (strstr(item->stack_symbs_str, tmp_str) && strstr(item->stack_symbs_str, str)) { + st->stats.count[STACK_STATS_HISTO_FOLDED]++; + item->count = item->count + count; + return 0; + } + } + return -1; + } + H_FIND_S(st->svg_stack_traces[en_type]->histo_tbl, str, item); if (item) { st->stats.count[STACK_STATS_HISTO_FOLDED]++; @@ -461,6 +492,7 @@ static int add_stack_histo(struct stack_trace_s *st, struct stack_symbs_s *stack } new_item->stack_symbs_str[0] = 0; (void)strncpy(new_item->stack_symbs_str, str, STACK_SYMBS_LEN - 1); + new_item->stack_symbs_str[STACK_SYMBS_LEN - 1] = 0; new_item->count = count < 0 ? 0 : count; H_ADD_S(st->svg_stack_traces[en_type]->histo_tbl, stack_symbs_str, new_item); @@ -473,11 +505,9 @@ static void clear_stack_histo(struct svg_stack_trace_s *svg_st) return; } - struct stack_trace_histo_s *stack_trace_histo_tbl = svg_st->histo_tbl; struct stack_trace_histo_s *item, *tmp; - - H_ITER(stack_trace_histo_tbl, item, tmp) { - H_DEL(stack_trace_histo_tbl, item); + H_ITER(svg_st->histo_tbl, item, tmp) { + H_DEL(svg_st->histo_tbl, item); (void)free(item); } svg_st->histo_tbl = NULL; @@ -1035,12 +1065,13 @@ err: return -1; } -static int attach_oncpu_bpf_prog(struct svg_stack_trace_s *svg_st) +static int attach_oncpu_bpf_prog(struct svg_stack_trace_s *svg_st, StackprobeConfig *conf) { int ret; + int samplePeriod = conf->generalConfig->samplePeriod; struct perf_event_attr attr_type_sw = { - .sample_freq = SAMPLE_PERIOD, + .sample_freq = samplePeriod, // default 10ms .freq = 1, .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_CPU_CLOCK, @@ -1270,7 +1301,7 @@ static void *__uprobe_attach_check(void *arg) } -static int attach_memleak_bpf_prog(struct svg_stack_trace_s *svg_st) +static int attach_memleak_bpf_prog(struct svg_stack_trace_s *svg_st, StackprobeConfig *conf) { int err; #if 0 @@ -1393,6 +1424,66 @@ static void *__running(void *arg) return NULL; } +static FILE *__get_flame_graph_fp(struct stack_svg_mng_s *svg_mng) +{ + struct stack_flamegraph_s *sfg; + + sfg = &(svg_mng->flame_graph); + return sfg->fp; +} + +int __do_wr_stack_histo(struct stack_svg_mng_s *svg_mng, + struct stack_trace_histo_s *stack_trace_histo, int first, struct post_info_s *post_info) +{ + FILE *fp = __get_flame_graph_fp(svg_mng); + if (!fp) { + ERROR("[STACKPROBE]: Invalid fp.\n"); + return -1; + } + + __histo_tmp_str[0] = 0; + + if (first) { + (void)snprintf(__histo_tmp_str, HISTO_TMP_LEN, "%s %llu", + stack_trace_histo->stack_symbs_str, stack_trace_histo->count); + } else { + (void)snprintf(__histo_tmp_str, HISTO_TMP_LEN, "\n%s %llu", + stack_trace_histo->stack_symbs_str, stack_trace_histo->count); + } + if (post_info->post_flag) { + int written = post_info->buf - post_info->buf_start; + int ret = __snprintf(&post_info->buf, post_info->remain_size, &post_info->remain_size, "%s", __histo_tmp_str); + if (ret < 0) { + int new_post_max = g_post_max + POST_MAX_STEP_SIZE; + char *temp = (char *)realloc(post_info->buf_start, new_post_max); + if(temp == NULL) { + ERROR("[STACKPROBE]: Not enough post memory (realloc failed), current capacity is %d.\n", + g_post_max); + } else { + post_info->buf_start = temp; + post_info->buf = post_info->buf_start + written; + post_info->remain_size += POST_MAX_STEP_SIZE; + g_post_max = new_post_max; + INFO("[STACKPROBE]: post memory realloc to %d\n", g_post_max); + (void)__snprintf(&post_info->buf, post_info->remain_size, &post_info->remain_size, "%s", __histo_tmp_str); + } + } + } + + (void)fputs(__histo_tmp_str, fp); + return 0; +} + +void iter_histo_tbl(struct stack_svg_mng_s *svg_mng, int en_type, int *first_flag, struct post_info_s *post_info) +{ + struct stack_trace_histo_s *item, *tmp; + H_ITER(g_st->svg_stack_traces[en_type]->histo_tbl, item, tmp) { + (void)__do_wr_stack_histo(svg_mng, item, *first_flag, post_info); + *first_flag = 0; + } + return; +} + static void switch_stackmap() { struct stack_trace_s *st = g_st; @@ -1410,8 +1501,10 @@ static void switch_stackmap() if (st->svg_stack_traces[i] == NULL) { continue; } - (void)stack_id2histogram(st, i, st->is_stackmap_a); - wr_flamegraph(st->svg_stack_traces[i]->svg_mng, st->svg_stack_traces[i]->histo_tbl, i, &st->post_server); + if (stack_id2histogram(st, i, st->is_stackmap_a) != 0) { + continue; + } + wr_flamegraph(st->svg_stack_traces[i]->svg_mng , i, &st->post_server); clear_raw_stack_trace(st->svg_stack_traces[i], st->is_stackmap_a); } record_running_ctx(st); @@ -1468,7 +1561,7 @@ static int init_enabled_svg_stack_traces(StackprobeConfig *conf) } if (flameProcs[i].func) { - if (flameProcs[i].func(svg_st)) { + if (flameProcs[i].func(svg_st, conf)) { goto err; } } diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h index 2e3cfcc..b09cdba 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h @@ -141,4 +141,6 @@ struct stack_trace_s { int pmu_fd[]; // It must be put to the last. }; +void iter_histo_tbl(struct stack_svg_mng_s *svg_mng, int en_type, int *first_flag, struct post_info_s *post_info); + #endif diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/svg.h b/src/probes/extends/ebpf.probe/src/stackprobe/svg.h index d763e7b..a707342 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/svg.h +++ b/src/probes/extends/ebpf.probe/src/stackprobe/svg.h @@ -19,6 +19,15 @@ #include #include "stack.h" +#include + +struct post_info_s { + int post_flag; + int remain_size; + char *buf_start; + char *buf; + CURL *curl; +}; #define DAYS_TIME (24 * 60 *60) // 1 DAY #define WEEKS_TIME (DAYS_TIME * 7) // 1 WEEK diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/symbol.c b/src/probes/extends/ebpf.probe/src/stackprobe/symbol.c index 2f7e06b..0c4ffaf 100644 --- a/src/probes/extends/ebpf.probe/src/stackprobe/symbol.c +++ b/src/probes/extends/ebpf.probe/src/stackprobe/symbol.c @@ -1049,7 +1049,6 @@ int proc_search_addr_symb(struct proc_symbs_s *proc_symbs, } continue; } - // search debug symbs ret = search_elf_symb(proc_symbs->mods[i]->debug_symbs, addr, addr, comm, addr_symb); -- 2.33.0