packages/etmem
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
etmem/0026-add-scan-library.patch
YangXin ce6533c4bd Update etmem. 
Signed-off-by: YangXin <245051644@qq.com>
2021-09-30 22:57:41 +08:00

525 lines
18 KiB
Diff
Raw Blame History

From dcf4760df185d5e75f522914e160d85d4f3543ce Mon Sep 17 00:00:00 2001
From: Kemeng Shi <shikemeng@huawei.com>
Date: Tue, 11 May 2021 15:53:10 +0800
Subject: [PATCH 26/50] add scan library
Signed-off-by: Kemeng Shi <shikemeng@huawei.com>
---
CMakeLists.txt | 25 ++++++++++
inc/etmemd_inc/etmemd.h | 23 +---------
inc/etmemd_inc/etmemd_exp.h | 42 +++++++++++++++++
inc/etmemd_inc/etmemd_scan.h | 43 ++---------------
inc/etmemd_inc/etmemd_scan_exp.h | 71 +++++++++++++++++++++++++++++
inc/etmemd_inc/etmemd_scan_export.h | 22 +++++++++
src/etmemd_src/etmemd_common.c | 13 +++++-
src/etmemd_src/etmemd_scan.c | 67 +++++++++++++++++++++++++--
src/etmemd_src/etmemd_scan.version | 4 ++
9 files changed, 244 insertions(+), 66 deletions(-)
create mode 100644 inc/etmemd_inc/etmemd_exp.h
create mode 100644 inc/etmemd_inc/etmemd_scan_exp.h
create mode 100644 inc/etmemd_inc/etmemd_scan_export.h
create mode 100644 src/etmemd_src/etmemd_scan.version
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9ce4724..6d11da9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,8 +55,13 @@ add_executable(etmemd
add_executable(etmem
${ETMEM_SRC})
+add_library(etmemd_scan SHARED
+ ${ETMEMD_SRC})
+
set(EXECUTABLE_OUTPUT_PATH ${BUILD_DIR}/bin)
+set(LIBRARY_OUTPUT_PATH ${BUILD_DIR}/lib)
+
include(FindPkgConfig)
pkg_search_module(GLIB2 REQUIRED glib-2.0)
@@ -67,6 +72,10 @@ target_include_directories(etmemd PRIVATE
target_include_directories(etmem PRIVATE
${PROJECT_SOURCE_DIR}/inc/etmem_inc)
+target_include_directories(etmemd_scan PRIVATE
+ ${PROJECT_SOURCE_DIR}/inc/etmemd_inc
+ ${GLIB2_INCLUDE_DIRS})
+
target_compile_options(etmemd PRIVATE -fsigned-char -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing -Werror -Wformat -Wformat-security -D_GNU_SOURCE -fPIE -pie -fPIC -fstack-protector-strong -fno-common -DNDEBUG -O2 -D_FORTIFY_SOURCE=2 -Wall -Werror -Wl,-z,relro,-z,now,-z,noexecstack -Wtrampolines -pthread -Wno-pointer-sign -Wstrict-prototypes -Wold-style-definition -std=gnu99)
@@ -97,3 +106,19 @@ if( ${ARCHITECTURE} STREQUAL "aarch64" )
else()
target_compile_options(etmem PRIVATE -march=core-avx-i -m64)
endif()
+
+target_compile_options(etmemd_scan PRIVATE -fsigned-char -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing -Werror -Wformat -Wformat-security -D_GNU_SOURCE -fPIE -pie -fPIC -fstack-protector-strong -fno-common -DNDEBUG -O2 -D_FORTIFY_SOURCE=2 -Wall -Werror -Wl,-z,relro,-z,now,-z,noexecstack -Wtrampolines -pthread -Wno-pointer-sign -Wstrict-prototypes -Wold-style-definition -std=gnu99 -fPIC -shared)
+
+
+if(CONFIG_DEBUG STREQUAL "y")
+ target_compile_options(etmemd_scan PRIVATE -g)
+endif()
+
+set_target_properties(etmemd_scan PROPERTIES LINK_FLAGS "-s -fPIE -pie -fPIC -Wl,-z,relro,-z,now,-z,noexecstack -Wtrampolines -Wl,--version-script=${ETMEMD_SRC_DIR}/etmemd_scan.version")
+target_link_libraries(etmemd_scan PRIVATE pthread dl rt boundscheck numa ${GLIB2_LIBRARIES})
+
+if( ${ARCHITECTURE} STREQUAL "aarch64" )
+ target_compile_options(etmemd_scan PRIVATE -march=armv8-a)
+else()
+ target_compile_options(etmemd_scan PRIVATE -march=core-avx-i -m64)
+endif()
diff --git a/inc/etmemd_inc/etmemd.h b/inc/etmemd_inc/etmemd.h
index 797049e..357ea4a 100644
--- a/inc/etmemd_inc/etmemd.h
+++ b/inc/etmemd_inc/etmemd.h
@@ -16,34 +16,13 @@
#ifndef ETMEMD_H
#define ETMEMD_H
-#include <stdint.h>
#include <stdbool.h>
+#include "etmemd_exp.h"
#define PTE_SIZE_SHIFT 12
#define PMD_SIZE_SHIFT 21
#define PUD_SIZE_SHIFT 30
-/*
- * page type specified by size
- * */
-enum page_type {
- PTE_TYPE = 0,
- PMD_TYPE,
- PUD_TYPE,
- PAGE_TYPE_INVAL,
-};
-
-/*
- * page struct after scan and parse
- * */
-struct page_refs {
- uint64_t addr; /* page address */
- int count; /* page count */
- enum page_type type; /* page type including PTE/PMD/PUD */
-
- struct page_refs *next; /* point to next page */
-};
-
/* memory grade is the result that judged by policy function after pagerefs come into it,
* every policy fucntion has its own rule to make the choice which page is hot grade or
* the other grades */
diff --git a/inc/etmemd_inc/etmemd_exp.h b/inc/etmemd_inc/etmemd_exp.h
new file mode 100644
index 0000000..8c57d9f
--- /dev/null
+++ b/inc/etmemd_inc/etmemd_exp.h
@@ -0,0 +1,42 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019-2021. All rights reserved.
+ * etmem is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: shikemeng
+ * Create: 2021-04-30
+ * Description: This is a header file of the export data structure definition for page.
+ ******************************************************************************/
+
+#ifndef ETMEMD_EXP_H
+#define ETMEMD_EXP_H
+
+#include <stdint.h>
+
+/*
+ * page type specified by size
+ * */
+enum page_type {
+ PTE_TYPE = 0,
+ PMD_TYPE,
+ PUD_TYPE,
+ PAGE_TYPE_INVAL,
+};
+
+/*
+ * page struct after scan and parse
+ * */
+struct page_refs {
+ uint64_t addr; /* page address */
+ int count; /* page count */
+ enum page_type type; /* page type including PTE/PMD/PUD */
+
+ struct page_refs *next; /* point to next page */
+};
+
+#endif
diff --git a/inc/etmemd_inc/etmemd_scan.h b/inc/etmemd_inc/etmemd_scan.h
index ed72e1a..09ad51c 100644
--- a/inc/etmemd_inc/etmemd_scan.h
+++ b/inc/etmemd_inc/etmemd_scan.h
@@ -19,9 +19,8 @@
#include <fcntl.h>
#include "etmemd.h"
#include "etmemd_task.h"
+#include "etmemd_scan_exp.h"
-#define VMA_PATH_STR_LEN 256
-#define VMA_MAJOR_MINOR_LEN 8
#define VMA_SEG_CNT_MAX 6
#define VMA_PERMS_STR_LEN 5
#define VMA_ADDR_STR_LEN 17
@@ -35,15 +34,7 @@
#define SMAPS_FILE "/smaps"
#define VMFLAG_HEAD "VmFlags"
-#define SCAN_AS_HUGE O_LARGEFILE
-
-enum {
- VMA_STAT_READ = 0,
- VMA_STAT_WRITE,
- VMA_STAT_EXEC,
- VMA_STAT_MAY_SHARE,
- VMA_STAT_INIT,
-};
+#define ALL_SCAN_FLAGS (SCAN_AS_HUGE | SCAN_IGN_HOST)
enum page_idle_type {
PTE_ACCESS = 0, /* 4k page */
@@ -66,40 +57,12 @@ enum access_type_weight {
WRITE_TYPE_WEIGHT = 3,
};
-/*
- * vma struct
- * */
-struct vma {
- uint64_t start; /* address start */
- uint64_t end; /* address end */
- bool stat[VMA_STAT_INIT]; /* vm area permissions */
- uint64_t offset; /* vm area offset */
- uint64_t inode; /* vm area inode */
- char path[VMA_PATH_STR_LEN]; /* path name */
- char major[VMA_MAJOR_MINOR_LEN]; /* device number major part */
- char minor[VMA_MAJOR_MINOR_LEN]; /* device number minor part */
-
- struct vma *next; /* point to next vma */
-};
-
struct walk_address {
uint64_t walk_start; /* walk address start */
uint64_t walk_end; /* walk address end */
uint64_t last_walk_end; /* last walk address end */
};
-/*
- * vmas struct
- * */
-struct vmas {
- uint64_t vma_cnt; /* number of vm area */
-
- struct vma *vma_list; /* vm area list */
-};
-
-/* etmemd_free_page_refs need to be called by the handler who called etmemd_do_scan() successfully */
-void etmemd_free_page_refs(struct page_refs *pf);
-
/* the caller need to judge value returned by etmemd_do_scan(), NULL means fail. */
struct page_refs *etmemd_do_scan(const struct task_pid *tpid, const struct task *tk);
@@ -107,7 +70,7 @@ struct page_refs *etmemd_do_scan(const struct task_pid *tpid, const struct task
void free_vmas(struct vmas *vmas);
struct page_refs **walk_vmas(int fd, struct walk_address *walk_address, struct page_refs **pf, unsigned long *use_rss);
-int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, unsigned long *use_rss);
+int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, unsigned long *use_rss, int flags);
int split_vmflags(char ***vmflags_array, char *vmflags);
struct vmas *get_vmas_with_flags(const char *pid, char **vmflags_array, int vmflags_num, bool is_anon_only);
diff --git a/inc/etmemd_inc/etmemd_scan_exp.h b/inc/etmemd_inc/etmemd_scan_exp.h
new file mode 100644
index 0000000..1fd4379
--- /dev/null
+++ b/inc/etmemd_inc/etmemd_scan_exp.h
@@ -0,0 +1,71 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019-2021. All rights reserved.
+ * etmem is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: shikemeng
+ * Create: 2021-04-30
+ * Description: This is a header file of the function declaration for export scan function.
+ ******************************************************************************/
+
+#ifndef ETMEMD_SCAN_EXP_H
+#define ETMEMD_SCAN_EXP_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <fcntl.h>
+
+#define VMA_PATH_STR_LEN 256
+#define VMA_MAJOR_MINOR_LEN 8
+
+#define SCAN_AS_HUGE 0100000000 /* treat normal vm page as vm hugepage */
+#define SCAN_IGN_HOST 0200000000 /* ignore host access when scan vm */
+
+enum {
+ VMA_STAT_READ = 0,
+ VMA_STAT_WRITE,
+ VMA_STAT_EXEC,
+ VMA_STAT_MAY_SHARE,
+ VMA_STAT_INIT,
+};
+
+/*
+ * vma struct
+ * */
+struct vma {
+ uint64_t start; /* address start */
+ uint64_t end; /* address end */
+ bool stat[VMA_STAT_INIT]; /* vm area permissions */
+ uint64_t offset; /* vm area offset */
+ uint64_t inode; /* vm area inode */
+ char path[VMA_PATH_STR_LEN]; /* path name */
+ char major[VMA_MAJOR_MINOR_LEN]; /* device number major part */
+ char minor[VMA_MAJOR_MINOR_LEN]; /* device number minor part */
+
+ struct vma *next; /* point to next vma */
+};
+
+/*
+ * vmas struct
+ * */
+struct vmas {
+ uint64_t vma_cnt; /* number of vm area */
+
+ struct vma *vma_list; /* vm area list */
+};
+
+int etmemd_scan_init(void);
+void etmemd_scan_exit(void);
+
+struct vmas *etmemd_get_vmas(const char *pid, char **vmflags_array, int vmflags_num, bool is_anon_only);
+void etmemd_free_vmas(struct vmas *vmas);
+
+int etmemd_get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, int flags);
+void etmemd_free_page_refs(struct page_refs *page_refs);
+
+#endif
diff --git a/inc/etmemd_inc/etmemd_scan_export.h b/inc/etmemd_inc/etmemd_scan_export.h
new file mode 100644
index 0000000..7ddc097
--- /dev/null
+++ b/inc/etmemd_inc/etmemd_scan_export.h
@@ -0,0 +1,22 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019-2021. All rights reserved.
+ * etmem is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: shikemeng
+ * Create: 2021-4-30
+ * Description: This is a header file of the export scan library.
+ ******************************************************************************/
+
+#ifndef ETMEMD_SCAN_EXPORT_H
+#define ETMEMD_SCAN_EXPORT_H
+
+#include "etmemd_exp.h"
+#include "etmemd_scan_exp.h"
+
+#endif
diff --git a/src/etmemd_src/etmemd_common.c b/src/etmemd_src/etmemd_common.c
index 4b9c4cb..59933c4 100644
--- a/src/etmemd_src/etmemd_common.c
+++ b/src/etmemd_src/etmemd_common.c
@@ -24,12 +24,18 @@
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
#include "securec.h"
#include "etmemd_common.h"
#include "etmemd_rpc.h"
#include "etmemd_log.h"
+#define IDLE_SCAN_MAGIC 0X66
+#define IDLE_SCAN_ADD_FLAGS _IOW(IDLE_SCAN_MAGIC, 0x0, unsigned int)
+#define IDLE_SCAN_REMOVE_FLAGS _IOW(IDLE_SCAN_MAGIC, 0x1, unsigned int)
+
static void usage(void)
{
printf("\nusage of etmemd:\n"
@@ -228,11 +234,16 @@ FILE *etmemd_get_proc_file(const char *pid, const char *file, int flags, const c
return NULL;
}
- fd = open(file_name, flags);
+ fd = open(file_name, 0);
if (fd < 0) {
etmemd_log(ETMEMD_LOG_ERR, "open file %s fail\n", file_name);
goto free_file_name;
}
+ if (flags != 0 && ioctl(fd, IDLE_SCAN_ADD_FLAGS, &flags) != 0) {
+ etmemd_log(ETMEMD_LOG_ERR, "set idle flags for %s fail with %s\n", pid, strerror(errno));
+ close(fd);
+ goto free_file_name;
+ }
fp = fdopen(fd, mode);
free_file_name:
diff --git a/src/etmemd_src/etmemd_scan.c b/src/etmemd_src/etmemd_scan.c
index fb4dd33..ba0cf5e 100644
--- a/src/etmemd_src/etmemd_scan.c
+++ b/src/etmemd_src/etmemd_scan.c
@@ -35,6 +35,8 @@
#define VMFLAG_MAX_LEN 100
#define VMFLAG_MAX_NUM 30
+static bool g_exp_scan_inited = false;
+
static const enum page_type g_page_type_by_idle_kind[] = {
PTE_TYPE,
PMD_TYPE,
@@ -403,6 +405,25 @@ struct vmas *get_vmas(const char *pid)
return get_vmas_with_flags(pid, NULL, 0, true);
}
+struct vmas *etmemd_get_vmas(const char *pid, char *vmflags_array[], int vmflags_num, bool is_anon_only)
+{
+ int i;
+
+ if (pid == NULL) {
+ etmemd_log(ETMEMD_LOG_ERR, "etmemd_get_vmas pid param is NULL\n");
+ return NULL;
+ }
+
+ for (i = 0; i < vmflags_num; i++) {
+ if (vmflags_array[i] == NULL) {
+ etmemd_log(ETMEMD_LOG_ERR, "etmemd_get_vmas vmflags_array[%d] is NULL\n", i);
+ return NULL;
+ }
+ }
+
+ return get_vmas_with_flags(pid, vmflags_array, vmflags_num, is_anon_only);
+}
+
static u_int64_t get_address_from_buf(const unsigned char *buf, u_int64_t index)
{
u_int64_t address;
@@ -633,7 +654,7 @@ struct page_refs **walk_vmas(int fd,
* this parameter is used only in the dynamic engine to calculate the swap-in rate.
* In other policies, NULL can be directly transmitted.
* */
-int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, unsigned long *use_rss)
+int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, unsigned long *use_rss, int flags)
{
u_int64_t i;
FILE *scan_fp = NULL;
@@ -642,7 +663,7 @@ int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **p
struct page_refs **tmp_page_refs = NULL;
struct walk_address walk_address = {0, 0, 0};
- scan_fp = etmemd_get_proc_file(pid, IDLE_SCAN_FILE, 0, "r");
+ scan_fp = etmemd_get_proc_file(pid, IDLE_SCAN_FILE, flags, "r");
if (scan_fp == NULL) {
etmemd_log(ETMEMD_LOG_ERR, "open %s file fail\n", IDLE_SCAN_FILE);
return -1;
@@ -683,6 +704,21 @@ int get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **p
return 0;
}
+int etmemd_get_page_refs(const struct vmas *vmas, const char *pid, struct page_refs **page_refs, int flags)
+{
+ if (!g_exp_scan_inited) {
+ etmemd_log(ETMEMD_LOG_ERR, "scan module is not inited before etmemd_get_page_refs\n");
+ return -1;
+ }
+
+ if (vmas == NULL || pid == NULL || page_refs == NULL) {
+ etmemd_log(ETMEMD_LOG_ERR, "NULL param is found in etmemd_get_page_refs\n");
+ return -1;
+ }
+
+ return get_page_refs(vmas, pid, page_refs, NULL, flags & ALL_SCAN_FLAGS);
+}
+
void etmemd_free_page_refs(struct page_refs *pf)
{
struct page_refs *tmp_pf = NULL;
@@ -721,7 +757,7 @@ struct page_refs *etmemd_do_scan(const struct task_pid *tpid, const struct task
/* loop for scanning idle_pages to get result of memory access. */
for (i = 0; i < tk->eng->proj->loop; i++) {
- ret = get_page_refs(vmas, pid, &page_refs, NULL);
+ ret = get_page_refs(vmas, pid, &page_refs, NULL, 0);
if (ret != 0) {
etmemd_log(ETMEMD_LOG_ERR, "scan operation failed\n");
/* free page_refs nodes already exist */
@@ -737,6 +773,11 @@ struct page_refs *etmemd_do_scan(const struct task_pid *tpid, const struct task
return page_refs;
}
+void etmemd_free_vmas(struct vmas *vmas)
+{
+ free_vmas(vmas);
+}
+
void clean_page_refs_unexpected(void *arg)
{
struct page_refs **pf = (struct page_refs **)arg;
@@ -773,3 +814,23 @@ struct page_refs *add_page_refs_into_memory_grade(struct page_refs *page_refs, s
/* return the next page_refs of the one that passed in */
return tmp;
}
+
+int etmemd_scan_init(void)
+{
+ if (g_exp_scan_inited) {
+ etmemd_log(ETMEMD_LOG_ERR, "scan module already inited\n");
+ return -1;
+ }
+
+ if (init_g_page_size() == -1) {
+ return -1;
+ }
+
+ g_exp_scan_inited = true;
+ return 0;
+}
+
+void etmemd_scan_exit(void)
+{
+ g_exp_scan_inited = false;
+}
diff --git a/src/etmemd_src/etmemd_scan.version b/src/etmemd_src/etmemd_scan.version
new file mode 100644
index 0000000..576c96f
--- /dev/null
+++ b/src/etmemd_src/etmemd_scan.version
@@ -0,0 +1,4 @@
+libetmemd_scan {
+ global: etmemd_scan_init; etmemd_scan_exit; etmemd_get_vmas; etmemd_free_vmas; etmemd_get_page_refs; etmemd_free_page_refs;
+ local:*;
+};
--
2.27.0
Reference in New Issue View Git Blame Copy Permalink