packages/A-FOT
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: packages:ca6e0c65ea0071e6216e7f04a27ed149eab11515
Branches Tags
packages:main
..
compare: packages:955d07188f51180fddd919cc0fa6ad62e4917390
Branches Tags
packages:main

No commits in common. "ca6e0c65ea0071e6216e7f04a27ed149eab11515" and "955d07188f51180fddd919cc0fa6ad62e4917390" have entirely different histories.
ca6e0c65ea ... 955d07188f

7 changed files with 0 additions and 2310 deletions
Show Stats Expand all files Collapse all files

426
0001-Add-Bear-to-A-FOT.patch
View File

@ -1,426 +0,0 @@
diff --git a/a-fot b/a-fot
index 1520a9e..c633098 100644
--- a/a-fot
+++ b/a-fot
@@ -74,6 +74,10 @@ function parse_input_params() {
check_success=$2
shift 2
;;
+ --build_mode)
+ build_mode=$2
+ shift 2
+ ;;
*)
suggest_info
exit 1
@@ -124,6 +128,10 @@ function check_config_item() {
echo "[ERROR] The configuration item 'check_success' is missing, please check!"
exit 1
fi
+ if [[ -z ${build_mode} ]]; then
+ echo "[ERROR] The configuration item 'build_mode' is missing, please check!"
+ exit 1
+ fi
}
function suggest_info() {
@@ -141,6 +149,7 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...]
--run_script Script path for run application
--max_waiting_time Maximum binary startup time (unit: seconds)
--check_success Check optimization result
+--build_mode Execute build scrip mode (Wrapper/Bear)
"""
}
@@ -181,11 +190,27 @@ function check_common_dependency() {
is_file_exist "${gcc_path}/bin/gcc"
}
+# 拆分编译数据库
+function split_option() {
+ if [ "$bear_prefix" ];then
+ python3 $current_path/split_json.py -i $PWD/compile_commands.json
+ mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
+ mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
+ fi
+}
+
# 使用原始编译选项进行编译
function first_compilation() {
echo "[INFO] Start raw compilation"
is_file_exist ${build_script} "build_script"
- /bin/bash ${build_script} >> ${log_file} 2>&1
+ if [[ $build_mode =~ "Bear" ]]; then
+ bear_prefix="bear -- "
+ echo "[INFO] Build in Bear mode"
+ else
+ echo "[INFO] Build in Wrapper mode"
+ fi
+ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+ split_option first
is_file_exist ${bin_file}
is_success $?
}
@@ -237,7 +262,7 @@ function detect_process() {
function second_compilation() {
echo "[INFO] Try compiling with the new compilation options"
if [[ ${check_success} -eq 1 ]]; then
- /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
+ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
echo "[INFO] Found build id: ${build_id}"
add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}')
build_status=`ps -p ${build_id} | grep -c ${build_id}`
@@ -254,9 +279,10 @@ function second_compilation() {
done
wait
else
- /bin/bash ${build_script} >> ${log_file} 2>&1
+ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
fi
echo "[INFO] Finish compiling with new compilation options"
+ split_option second
is_success $?
}
@@ -308,10 +334,12 @@ function is_opt_success() {
if [[ ${opt_success} -eq 0 ]]; then
echo "[WARNING] Optimization may fail or the build process is too short, please check!"
echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler"
+ exit 1
else
echo "[INFO] Optimization may success!"
fi
fi
+ exit 0
}
#执行入口，部分函数为加载不同优化脚本中得到
@@ -324,15 +352,17 @@ function main() {
load_script
check_dependency
- create_wrapper
+ prepare_env
first_compilation
execute_run_script
detect_process
perf_record
- create_new_wrapper
+ prepare_new_env
second_compilation
is_opt_success
+ exit "$?"
}
main "$@"
+exit "$?"
diff --git a/a-fot.ini b/a-fot.ini
index 03a4702..b395504 100644
--- a/a-fot.ini
+++ b/a-fot.ini
@@ -18,4 +18,6 @@ perf_time=100
gcc_path=/usr/
# 检测是否优化成功(1=启用，0=禁用)
check_success=0
+# 构建模式 （Bear、Wrapper）
+build_mode=Bear
# 结束行勿删
\ No newline at end of file
diff --git a/auto_bolt.sh b/auto_bolt.sh
index 625d1d5..ebcb2e5 100644
--- a/auto_bolt.sh
+++ b/auto_bolt.sh
@@ -9,11 +9,29 @@ function check_dependency() {
fi
}
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-Wl,-q"
+ export LINK_OPTIONS="-q"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
+
# 创建原始wrapper
function create_wrapper() {
echo "[INFO] Start generating the original wrapper"
- echo "${gcc_path}/bin/gcc -Wl,-q \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -Wl,-q \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
}
@@ -28,9 +46,28 @@ function perf_record() {
pkill ${application_name}
}
+
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_new_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_new_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q"
+ export LINK_OPTIONS="-q"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
+
#生成新的wrapper
function create_new_wrapper() {
echo "[INFO] Start to generate a new wrapper"
- echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_fdo.sh b/auto_fdo.sh
index 3d7bbb8..8426b30 100644
--- a/auto_fdo.sh
+++ b/auto_fdo.sh
@@ -9,11 +9,28 @@ function check_dependency() {
fi
}
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-g"
+ export LINK_OPTIONS="-g"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
# 创建原始wrapper
function create_wrapper() {
echo "[INFO] Start generating the original wrapper"
- echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
}
@@ -39,9 +56,25 @@ function perf_record() {
pkill ${application_name}
}
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_new_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_new_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
#生成新的wrapper
function create_new_wrapper() {
echo "[INFO] Start to generate a new wrapper"
- echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_prefetch.sh b/auto_prefetch.sh
index 2562dd6..265828a 100644
--- a/auto_prefetch.sh
+++ b/auto_prefetch.sh
@@ -9,11 +9,28 @@ function check_dependency() {
fi
}
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-g"
+ export LINK_OPTIONS="-g"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
# 创建原始wrapper
function create_wrapper() {
echo "[INFO] Start generating the original wrapper"
- echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
}
@@ -40,9 +57,25 @@ function perf_record() {
pkill ${application_name}
}
+# 根据模式选择Wrapper或者Bear模式构建
+function prepare_new_env() {
+ case ${build_mode} in
+ "Wrapper")
+ create_new_wrapper
+ ;;
+ "Bear")
+ export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2"
+ ;;
+ *)
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ exit 1
+ ;;
+ esac
+}
+
#生成新的wrapper
function create_new_wrapper() {
echo "[INFO] Start to generate a new wrapper"
- echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/gcc
- echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/g++
+ echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc
+ echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/split_json.py b/split_json.py
new file mode 100644
index 0000000..bb6547f
--- /dev/null
+++ b/split_json.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# _*_ coding:utf-8 _*_
+
+
+"""
+ Copyright@2022-2022, All rights reserved, Powered by: Huawei Tech.Co.,Ltd.
+
+split_json is a interface to split compile commands according to its execution result.
+
+It will generate a new JSON file for the failed compile commands and remove these commands from source file.
+
+Input:
+ compile commands json
+Output:
+ compile commands json including commands which has been successfully executed.
+ compile commands json including commands which executed unsuccessfully.
+
+"""
+__author__ = 'z00500762'
+
+import argparse
+import os
+import json
+import sys
+import logging
+
+
+class SplitJson:
+ compile_commands_success = list()
+ compile_commands_fail = list()
+
+ def __init__(self, input_json):
+ self.input = input_json
+
+ @staticmethod
+ def validate(execution):
+ if "arguments" not in execution:
+ return False
+ if "directory" not in execution:
+ return False
+ if "exec_result" not in execution:
+ return False
+ return True
+
+ def get_compile_commands(self):
+ compile_commands = list()
+ try:
+ with open(self.input, "r", encoding='utf-8', errors='ignore') as json_file:
+ compile_commands = json.load(json_file)
+ if len(compile_commands) == 0:
+ logging.info("compile commands json file is empty: %s", self.input)
+ except IOError as exception:
+ logging.error("open compile commands json file failed: %s", exception)
+ except json.decoder.JSONDecodeError as exception:
+ logging.error("json decode file failed: %s", exception)
+
+ return compile_commands
+
+ def split_commands(self):
+ compile_commands = self.get_compile_commands()
+ for item in compile_commands:
+ if not self.validate(item):
+ logging.info("discard invalid commands: %s", str(item))
+ if not item.get("rebuild"):
+ self.compile_commands_success.append(item)
+ else:
+ self.compile_commands_fail.append(item)
+ self.write_json()
+
+ def write_json(self):
+ compile_commands_success_file = os.path.splitext(self.input)[0] + ".json"
+ compile_commands_fail_file = os.path.splitext(self.input)[0] + ".fail.json"
+ with open(compile_commands_success_file, 'w+') as fw:
+ json.dump(self.compile_commands_success, fw, sort_keys=False, indent=4)
+
+ with open(compile_commands_fail_file, 'w+') as fw:
+ json.dump(self.compile_commands_fail, fw, sort_keys=False, indent=4)
+
+
+def main(input_json):
+ if not os.path.isabs(input_json):
+ input_json = os.path.join(os.getcwd(), input_json)
+ if not os.path.exists(input_json):
+ logging.error("compile_command_file not exists : %s", input_json)
+ return -1
+
+ sj = SplitJson(input_json)
+ sj.split_commands()
+
+
+if __name__ == "__main__":
+ cmd_parser = argparse.ArgumentParser(description="split compile commands json")
+ cmd_parser.add_argument(
+ '-i', '--input', dest='input_json', metavar='store', action='store',
+ help='json to split'
+ )
+ args = cmd_parser.parse_args()
+ sys.exit(main(args.input_json))

13
0002-Bugfix-Remove-Backslash-in-Options.patch
View File

@ -1,13 +0,0 @@
diff --git a/auto_prefetch.sh b/auto_prefetch.sh
index 265828a..b7f5932 100644
--- a/auto_prefetch.sh
+++ b/auto_prefetch.sh
@@ -64,7 +64,7 @@ function prepare_new_env() {
create_new_wrapper
;;
"Bear")
- export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2"
+ export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses:u -fprefetch-loop-arrays=2"
;;
*)
echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"

807
0003-PGO-kernel-Add-PGO-kernel-mode.patch
View File

@ -1,807 +0,0 @@
From 2beb891a3e87e0fa61509d3746f0e216374f6f23 Mon Sep 17 00:00:00 2001
From: xiongzhou4 <xiongzhou4@huawei.com>
Date: Mon, 29 May 2023 17:14:22 +0800
Subject: [PATCH] [PGO kernel] Add PGO kernel mode.
---
GcovSummaryAddTool.cpp | 176 +++++++++++++++++++++++
a-fot | 129 ++++++++++++++---
a-fot.ini | 45 ++++--
auto_kernel_pgo.sh | 310 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 630 insertions(+), 30 deletions(-)
create mode 100644 GcovSummaryAddTool.cpp
create mode 100644 auto_kernel_pgo.sh
diff --git a/GcovSummaryAddTool.cpp b/GcovSummaryAddTool.cpp
new file mode 100644
index 0000000..656f3d5
--- /dev/null
+++ b/GcovSummaryAddTool.cpp
@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved.
+ * This tool is used to add GCC 9 and GCC 10 summary info into gcov file.
+ */
+
+#include <cstdio>
+#include <string>
+#include <vector>
+#include <cassert>
+
+using namespace std;
+
+constexpr long long GCOV_DATA_MAGIC = 0x67636461u;
+constexpr long long GCOV_TAG_FUNCTION = 0x01000000u;
+constexpr long long GCOV_TAG_COUNTER_BASE = 0x01a10000u;
+constexpr long long GCOV_TAG_OBJECT_SUMMARY = 0xa1000000u;
+
+/* value profile counter */
+constexpr long long GCOV_TAG_COUNTER_INTERVAL = 0x01a30000u;
+constexpr long long GCOV_TAG_COUNTER_POW2 = 0x01a50000u;
+constexpr long long GCOV_TAG_COUNTER_TOPN = 0x01a70000u;
+constexpr long long GCOV_TAG_COUNTER_IC = 0x01a90000u; // indirect call profiler
+constexpr long long GCOV_TAG_COUNTER_AVERAGE = 0x01ab0000u;
+constexpr long long GCOV_TAG_COUNTER_IOR = 0x01ad0000u;
+constexpr long long GCOV_TAG_COUNTER_TP = 0x01af0000u; // time profiler
+
+static int ReadFile(const string filename, vector<char>& out)
+{
+ FILE* fp = fopen(filename.c_str(), "rb");
+ if (!fp) {
+ fprintf(stderr, "[!] Fail to read: %s\n", filename.c_str());
+ return 1;
+ }
+ constexpr int bufSize = 4096;
+ char buf[bufSize];
+ size_t sz;
+ while (sz = fread(buf, 1, bufSize, fp)) {
+ out.insert(out.end(), buf, buf + sz);
+ }
+ fclose(fp);
+ return 0;
+}
+
+static void SplitLines(const vector<char>& in, vector<string>& out)
+{
+ size_t pos = 0;
+ for (size_t i = 0; i < in.size(); ++i) {
+ if (in[i] != '\r' && in[i] != '\n') {
+ continue;
+ }
+ out.push_back(string(in.begin() + pos, in.begin() + i));
+ if (in[i] == '\r' && i + 1 < in.size() && in[i + 1] == '\n') {
+ i++;
+ }
+ pos = i + 1;
+ }
+ if (pos < in.size()) {
+ out.push_back(string(in.begin() + pos, in.end()));
+ }
+}
+
+static int WriteFile(string fileName, const vector<char> in, unsigned int valMax)
+{
+ if (in.size() < 12) {
+ fprintf(stderr, "[!] Not enough size to write\n");
+ return 1;
+ }
+ FILE* fp = fopen(fileName.c_str(), "wb");
+ if (!fp) {
+ fprintf(stderr, "[!] Fail to write: %s \n", fileName.c_str());
+ return 1;
+ }
+ fwrite(in.data(), 1, 12, fp);
+ unsigned int title[4] = {
+ GCOV_TAG_OBJECT_SUMMARY,
+ 2,
+ 1,
+ valMax
+ };
+ fwrite(title, 1, 16, fp);
+ fwrite(in.data() + 12, 1, in.size() - 12, fp);
+ fclose(fp);
+ return 0;
+}
+
+static int ProcessFile(const string fileName)
+{
+ vector<char> source;
+ if (ReadFile(fileName, source)) {
+ fprintf(stderr, "[!] Fail to read file: %s \n", fileName.c_str());
+ return 1;
+ }
+ int state = 1;
+ unsigned int valMax = 0;
+ unsigned int count = 0;
+ unsigned int n = source.size() / 4;
+ auto vData = (const unsigned int*) source.data();
+ for (int i = 0; i < n; ++i) {
+ unsigned int val = vData[i];
+ switch (state) {
+ case 1:
+ if (val != GCOV_DATA_MAGIC) {
+ fprintf(stderr, "[!] GCOV_DATA_MAGIC mismatches: 0x%x\n", val);
+ return 1;
+ }
+ i += 2;
+ state = 2;
+ break;
+ case 2:
+ if (i == n - 1 && val) {
+ fprintf(stderr, "[!] Single last tag: 0x%x\n", val);
+ return 1;
+ }
+ if (val == GCOV_TAG_FUNCTION) {
+ i += 1 + vData[i + 1];
+ } else if (val == GCOV_TAG_COUNTER_BASE) {
+ if (vData[i + 1] % 2) {
+ fprintf(stderr, "[!] Invalid length: %d\n", vData[i + 1]);
+ return 1;
+ }
+ count = vData[++i];
+ if (count) {
+ state = 3;
+ }
+ } else if (val) {
+ switch (val) {
+ case GCOV_TAG_COUNTER_INTERVAL:
+ case GCOV_TAG_COUNTER_POW2:
+ case GCOV_TAG_COUNTER_TOPN:
+ case GCOV_TAG_COUNTER_IC:
+ case GCOV_TAG_COUNTER_AVERAGE:
+ case GCOV_TAG_COUNTER_IOR:
+ case GCOV_TAG_COUNTER_TP:
+ i += 1 + vData[i + 1];
+ break;
+ default:
+ fprintf(stderr, "[!] Unknown tag: 0x%x\n", val);
+ return 1;
+ }
+ }
+ break;
+ case 3:
+ valMax = valMax < val ? val : valMax;
+ if (--count == 0) {
+ state = 2;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ if (WriteFile(fileName, source, valMax)) {
+ return 1;
+ }
+ return 0;
+}
+
+int main(int argc, char** argv)
+{
+ vector<char> fileNameList;
+ if (argc != 2 || ReadFile(argv[1], fileNameList)) {
+ fprintf(stderr, "USAGE:\n %s <Input File List>\n", argv[0]);
+ return 1;
+ }
+ vector<string> fileNames;
+ SplitLines(fileNameList, fileNames);
+ for (size_t i = 0; i < fileNames.size(); ++i) {
+ string fileName = fileNames[i];
+ fprintf(stderr, "[.] Processing %s \n", fileName.c_str());
+ if (ProcessFile(fileName)) {
+ return 1;
+ }
+ }
+ fprintf(stderr, "[.] File procesed: %d \n", (int) fileNames.size());
+ return 0;
+}
diff --git a/a-fot b/a-fot
index c633098..1e7d08e 100755
--- a/a-fot
+++ b/a-fot
@@ -1,11 +1,13 @@
#!/bin/bash
-current_path=$(cd "$(dirname "$0")";pwd)
-config_file=${current_path}/a-fot.ini
+afot_path=$(cd "$(dirname "$0")";pwd)
+config_file=${afot_path}/a-fot.ini
# Profile名字
profile_name="profile.data"
# gcov名字
gcov_name="profile.gcov"
+# 内核编译选项
+kernel_configs=()
# 解析配置文件配置项
function parse_config() {
@@ -13,21 +15,21 @@ function parse_config() {
echo "[ERROR] Could not load config file at: ${config_file}, please check!"
exit 1
fi
- while read line; do
- if [[ ! ${line} =~ "#" ]]; then
+ while read line || [[ -n ${line} ]]; do
+ if [[ ! ${line} =~ "#" ]] && [[ ${line} != "" ]]; then
key=$(echo ${line} | awk -F "=" '{print $1}')
value=$(echo ${line} | awk -F "=" '{print $2}')
- eval "${key}=${value}"
+ if [[ ${key} =~ "CONFIG_" ]]; then
+ kernel_configs+="${key}=${value}"
+ else
+ eval "${key}=${value}"
+ fi
fi
done <${config_file}
}
# 解析输入的参数
function parse_input_params() {
- if [[ $# == 1 ]]; then
- suggest_info
- exit 1
- fi
while [ $# -ge 2 ]; do
case $1 in
--opt_mode)
@@ -78,12 +80,71 @@ function parse_input_params() {
build_mode=$2
shift 2
;;
+ --pgo_mode)
+ pgo_mode=$2
+ shift 2
+ ;;
+ --pgo_phase)
+ pgo_phase=$2
+ shift 2
+ ;;
+ --kernel_src)
+ kernel_src=$2
+ shift 2
+ ;;
+ --kernel_name)
+ kernel_name=$2
+ shift 2
+ ;;
+ --run_time)
+ run_time=$2
+ shift 2
+ ;;
+ --CONFIG_*)
+ kernel_configs+="${1:2}=$2"
+ shift 2
+ ;;
+ --last_time)
+ last_time=$2
+ shift 2
+ ;;
+ -s)
+ silent=1
+ shift
+ ;;
+ -n)
+ disable_compilation=1
+ shift
+ ;;
+ --makefile)
+ makefile=$2
+ shift 2
+ ;;
+ --kernel_config)
+ kernel_config=$2
+ shift 2
+ ;;
+ --data_dir)
+ data_dir=$2
+ shift 2
+ ;;
*)
suggest_info
exit 1
;;
esac
done
+
+ if [[ $# == 1 ]]; then
+ if [[ $1 == "-s" ]]; then
+ silent=1
+ elif [[ $1 == "-n" ]]; then
+ disable_compilation=1
+ else
+ suggest_info
+ exit 1
+ fi
+ fi
}
function check_config_item() {
@@ -138,6 +199,7 @@ function suggest_info() {
echo """
Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...]
+For perf mode:
--config_file Path of configuration file
--opt_mode Optimization modes (AutoFDO/AutoPrefetch/AutoBOLT)
--perf_time Perf sampling duration (unit: seconds)
@@ -146,10 +208,28 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...]
--bin_file Executable binary file path
--build_script Application build script path
--work_path Script working directory (used to compile the application and store the profile)
---run_script Script path for run application
+--run_script Script path for running application
--max_waiting_time Maximum binary startup time (unit: seconds)
--check_success Check optimization result
---build_mode Execute build scrip mode (Wrapper/Bear)
+--build_mode Execute build script mode (Wrapper/Bear)
+
+For kernel PGO mode:
+--config_file Path of configuration file
+--opt_mode Optimization mode (Auto_kernel_PGO)
+--pgo_mode PGO mode (arc/all)
+--pgo_phase Phase of kernel PGO (1/2)
+--kernel_src Kernel source directory
+--kernel_name Kernel local version name (will be appended with "-pgoing" or "-pgoed")
+--work_path Script working directory (used to store the profile and the log)
+--run_script Script path for running application
+--gcc_path Compiler gcc path
+--CONFIG_... Kernel building
+--last_time Last time directory before rebooting (used to put log infos together)
+-s Silent mode (reboot automatically after kernel installation)
+-n Do not compile kernel automatically
+--makefile Makefile path of kernel
+--kernel_config Config file path of kernel
+--data_dir Profile path generated by kernel
"""
}
@@ -157,13 +237,13 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...]
function load_script() {
case ${opt_mode} in
"AutoFDO")
- source ${current_path}/auto_fdo.sh
+ source ${afot_path}/auto_fdo.sh
;;
"AutoPrefetch")
- source ${current_path}/auto_prefetch.sh
+ source ${afot_path}/auto_prefetch.sh
;;
"AutoBOLT")
- source ${current_path}/auto_bolt.sh
+ source ${afot_path}/auto_bolt.sh
;;
*)
echo "[ERROR] Optimization mode ${opt_mode} is not supported, Check the configuration item: opt_mode"
@@ -193,7 +273,7 @@ function check_common_dependency() {
# 拆分编译数据库
function split_option() {
if [ "$bear_prefix" ];then
- python3 $current_path/split_json.py -i $PWD/compile_commands.json
+ python3 $afot_path/split_json.py -i $PWD/compile_commands.json
mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
fi
@@ -289,7 +369,7 @@ function second_compilation() {
# 判断上一步执行是否成功
function is_success() {
pre_result=$1
- if [[ ${pre_result} -eq 1 ]]; then
+ if [[ ${pre_result} -ne 0 ]]; then
echo "[ERROR] Execution failed, please check the log: ${log_file}"
exit 1
fi
@@ -342,11 +422,10 @@ function is_opt_success() {
exit 0
}
-#执行入口，部分函数为加载不同优化脚本中得到
-function main() {
- parse_input_params "$@"
- parse_config
- parse_input_params "$@"
+function do_optimization() {
+ if [[ ${opt_mode} == Auto_kernel_PGO ]]; then
+ source ${afot_path}/auto_kernel_pgo.sh
+ fi
check_config_item
init_profile_and_log
load_script
@@ -361,6 +440,14 @@ function main() {
prepare_new_env
second_compilation
is_opt_success
+}
+
+#执行入口，部分函数为加载不同优化脚本中得到
+function main() {
+ parse_input_params "$@"
+ parse_config
+ parse_input_params "$@"
+ do_optimization
exit "$?"
}
diff --git a/a-fot.ini b/a-fot.ini
index b395504..c5da1b0 100644
--- a/a-fot.ini
+++ b/a-fot.ini
@@ -1,23 +1,50 @@
+# 文件和目录请使用绝对路径
+
+# 优化模式（AutoFDO、AutoPrefetch、AutoBOLT、Auto_kernel_PGO）
+opt_mode=Auto_kernel_PGO
+# 脚本工作目录（用来编译应用程序/存放profile、日志）
+work_path=/opt
+# 应用运行脚本路径
+run_script=/root/run.sh
+# GCC路径（bin、lib的父目录）
+gcc_path=/usr
+
+# AutoFDO、AutoPrefetch、AutoBOLT
+# 针对应用的三种优化模式，请填写此部分配置
+
# 应用进程名
application_name=test
# 二进制安装后可执行文件
bin_file=/tmp/test
-# 脚本工作目录（用来编译应用程序/存放profile）
-work_path=/tmp/
# 应用构建脚本路径
build_script=/root/build.sh
-# Benhmark路径及运行命令
-run_script=/root/benchmark.sh
# 最大二进制启动时间(单位：秒)
max_waiting_time=600
-# 优化模式（AutoFDO、AutoPrefetch、AutoBOLT）
-opt_mode=AutoPrefetch
# Perf采样时长(单位：秒)
perf_time=100
-# gcc路径
-gcc_path=/usr/
# 检测是否优化成功(1=启用，0=禁用)
check_success=0
# 构建模式 （Bear、Wrapper）
build_mode=Bear
-# 结束行勿删
\ No newline at end of file
+
+# auto_kernel_PGO
+# 针对内核的优化模式，请填写此部分配置
+
+# 内核PGO模式（arc=只启用arc profile，all=启用完整的PGO优化）
+pgo_mode=all
+# 执行阶段（1=编译插桩内核阶段，2=编译优化内核阶段）
+pgo_phase=1
+# 内核源码目录（不指定则自动下载）
+kernel_src=/opt/kernel
+# 内核构建的本地名（将根据阶段添加"-pgoing"或"-pgoed"后缀）
+kernel_name=kernel
+# 内核编译选项（请确保选项修改正确合法，不会造成内核编译失败）
+#CONFIG_...=y
+# 重启前的时间目录（用于将同一套流程的日志存放在一起）
+last_time=
+# 内核源码的Makefile地址（用于不自动编译内核的场景）
+makefile=
+# 内核配置文件路径（用于不自动编译内核的场景）
+kernel_config=
+# 内核生成的原始profile目录（用于不自动编译内核的场景）
+data_dir=
\ No newline at end of file
diff --git a/auto_kernel_pgo.sh b/auto_kernel_pgo.sh
new file mode 100644
index 0000000..0b62220
--- /dev/null
+++ b/auto_kernel_pgo.sh
@@ -0,0 +1,310 @@
+#!/bin/bash
+
+parallel=$(cat /proc/cpuinfo | grep "processor" | wc -l)
+
+# 检查某个配置项是否存在
+function check_item() {
+ if [[ -z $1 ]]; then
+ echo "[ERROR] The configuration item '$2' is missing, please check!"
+ exit 1
+ fi
+}
+
+# 检查所有配置项
+function check_config_items() {
+ check_item ${pgo_phase} pgo_phase
+ if [[ ${pgo_phase} -ne 1 ]] && [[ ${pgo_phase} -ne 2 ]]; then
+ echo "[ERROR] The value of configuration item 'pgo_phase' is invalid, please check!"
+ exit 1
+ fi
+ if [[ ${pgo_phase} -eq 1 ]]; then
+ check_item ${pgo_mode} pgo_mode
+ if [[ ${pgo_mode} != "arc" ]] && [[ ${pgo_mode} != "all" ]]; then
+ echo "[ERROR] The value of configuration item 'pgo_mode' is invalid, please check!"
+ exit 1
+ fi
+ fi
+
+ check_item ${kernel_name} kernel_name
+ check_item ${work_path} work_path
+ if [[ ${work_path} =~ "/tmp" ]]; then
+ echo "[ERROR] Do not put work path under /tmp, or it will be cleaned after rebooting."
+ fi
+ check_item ${run_script} run_script
+ check_item ${gcc_path} gcc_path
+
+ if [[ -n ${disable_compilation} ]]; then
+ check_item ${makefile} makefile
+ is_file_exist ${makefile} "makefile"
+ if [[ ${pgo_phase} -eq 1 ]]; then
+ check_item ${kernel_config} kernel_config
+ is_file_exist ${kernel_config} "kernel_config"
+ fi
+ if [[ ${pgo_phase} -eq 2 ]]; then
+ check_item ${data_dir} data_dir
+ if [[ ! -d ${data_dir} ]]; then
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ exit 1
+ fi
+ fi
+ fi
+}
+
+# 初始化文件和目录
+function init() {
+ if [[ -z ${last_time} ]]; then
+ now_time=$(date '+%Y%m%d-%H%M%S')
+ else
+ now_time=${last_time}
+ fi
+ time_dir=${work_path}/${now_time}
+ log_file=${time_dir}/log
+
+ if [[ -d ${time_dir} ]]; then
+ echo "[INFO] Use last time directory: ${time_dir}"
+ else
+ mkdir -p ${time_dir}
+ echo "[INFO] Create time directory: ${time_dir}"
+ fi
+
+ if [[ -f ${log_file} ]]; then
+ echo "[INFO] Use last log: ${log_file}"
+ else
+ touch ${log_file}
+ echo "[INFO] Init log file: ${log_file}"
+ fi
+}
+
+# 检测环境
+function check_dependency() {
+ sed -i '/a-fot/d' /etc/rc.d/rc.local
+
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]]; then
+ echo "[INFO] Current arch: ${get_arch}"
+ else
+ echo "[ERROR] Unsupport arch: ${get_arch}"
+ exit 1
+ fi
+ is_file_exist ${run_script} "run_script"
+ is_file_exist "${gcc_path}/bin/gcc" "gcc_path"
+
+ export CC=${gcc_path}/bin/gcc
+ export PATH=${gcc_path}/bin:${PATH}
+ export LD_LIBRARY_PATH=${gcc_path}/lib64:${LD_LIBRARY_PATH}
+
+ if [[ ${pgo_phase} -eq 1 ]] && [[ ${pgo_mode} == "all" ]] && [[ -z ${disable_compilation} ]]; then
+ if echo 'int main() { return 0; }' | gcc -x c - -o /dev/null -Werror -fkernel-pgo >/dev/null 2>&1; then
+ echo "[INFO] Current GCC supports kernel PGO in runtime, use -fkernel-pgo option."
+ option="-fkernel-pgo"
+ elif gcc -v 2>&1 | grep -- "--disable-tls" >/dev/null; then
+ echo "[INFO] Current GCC is recompiled with --disable-tls, does support kernel PGO in runtime."
+ else
+ echo "[ERROR] Current GCC does not support kernel PGO, you may use openeuler GCC or recompile GCC with --disable-tls --disable-libsanitizer."
+ fi
+ fi
+}
+
+# 修改内核配置文件
+function modify_kernel_config() {
+ if [[ ${pgo_mode} == "all" ]]; then
+ if ! grep "PGO" $1 >/dev/null; then
+ echo "[ERROR] Current version of kernel does not support PGO, please use newer ones or choose arc mode."
+ exit 1
+ fi
+ sed -i 's/.*CONFIG_PGO_KERNEL.*/CONFIG_PGO_KERNEL=y/' $1
+ fi
+ for config in ${kernel_configs[@]}; do
+ sed -i "s/.*${config%%=*}.*/${config}/" $1
+ done
+ sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-${kernel_name}-pgoing\"/" $1
+ sed -i 's/.*CONFIG_GCOV_KERNEL.*/CONFIG_GCOV_KERNEL=y/' $1
+ sed -i '/CONFIG_ARCH_HAS_GCOV_PROFILE_ALL/ a\CONFIG_GCOV_PROFILE_ALL=y' $1
+ sed -i '/CONFIG_CC_HAS_ASM_INLINE/ a\CONFIG_CONSTRUCTORS=y' $1
+ sed -i '/CONFIG_TRACE_EVAL_MAP_FILE/ a\# CONFIG_GCOV_PROFILE_FTRACE is not set' $1
+}
+
+# 编译插桩版本的内核
+function first_compilation() {
+ if [[ -n ${disable_compilation} ]]; then
+ modify_kernel_config ${kernel_config}
+ echo "[INFO] Kernel configuration has been generated, the path is: ${kernel_config}"
+ if [[ -z ${option} ]]; then
+ sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += fkernel-pgo" ${makefile}
+ fi
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ exit 0
+ fi
+
+ if [[ -z ${kernel_src} ]]; then
+ echo "[WARNING] ${kernel_src} is not specified, A-FOT will download and build kernel source code automatically in 10 seconds."
+ sleep 10
+ oe_version=$(grep 'openeulerversion' /etc/openEuler-latest | cut -d '=' -f 2)
+ echo "[INFO] Current openEuler version: ${oe_version}"
+ url="https://repo.huaweicloud.com/openeuler/${oe_version}/source/Packages/"
+ echo "[INFO] Download the kernel source rpm package from ${url}"
+ cd ${work_path}
+ wget --no-parent --no-directories -r -A 'kernel-[0-9]*.rpm' ${url} --no-check-certificate >>${log_file} 2>&1
+ is_success $?
+ srpm=$(ls -t kernel-[0-9]*.rpm 2>/dev/null | head -n 1)
+ echo "[INFO] Successfully downloaded kernel source rpm package: ${srpm}"
+ echo "[INFO] Build kernel source code."
+ rpm -ivh ${srpm} >>${log_file} 2>&1
+ cd - >/dev/null
+ rpmbuild -bp ~/rpmbuild/SPECS/kernel.spec >>${log_file} 2>&1
+ is_success $?
+ src_dir=$(ls -td ~/rpmbuild/BUILD/kernel-*/*-source/ | head -n 1)
+ cp -r ${src_dir} ${work_path}/kernel
+ kernel_src=${work_path}/kernel
+ echo "[INFO] Successfully builded kernel source code: ${kernel_src}"
+ else
+ echo "[INFO] Build kernel in an existing source directory."
+ fi
+
+ if [[ ! -d ${kernel_src} ]]; then
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ exit 1
+ fi
+
+ cd ${kernel_src}
+ make openeuler_defconfig
+ modify_kernel_config .config
+ echo "[INFO] Start PGO kernel compilation."
+ Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="${option}" ARCH=${Arch} -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ is_success ${PIPESTATUS[0]}
+ cd - >/dev/null
+}
+
+# 第一次重启操作
+function first_reboot() {
+ grub2-set-default 0
+ next_cmd="${afot_path}/a-fot --opt_mode Auto_kernel_PGO --pgo_phase 2 --kernel_src ${kernel_src} --kernel_name ${kernel_name} --work_path ${work_path} --run_script ${run_script} --gcc_path ${gcc_path} --last_time ${now_time} -s"
+ if [[ -z ${silent} ]]; then
+ read -p $'PGO kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
+ if [[ ${reboot_p} == "y" ]]; then
+ echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ sleep 10
+ reboot
+ else
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ fi
+ else
+ echo ${next_cmd} >>/etc/rc.d/rc.local
+ chmod u+x /etc/rc.d/rc.local
+ echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ sleep 10
+ reboot
+ fi
+ exit 0
+}
+
+# 执行应用程序执行脚本
+function execute_run_script() {
+ echo "[INFO] Start to execute the run_script: ${run_script}"
+ /bin/bash ${run_script} >>${log_file} 2>&1
+ is_success $?
+}
+
+# 收集和处理profile文件
+function process_profiles() {
+ if [[ -z ${disable_compilation} ]]; then
+ data_dir=/sys/kernel/debug/gcov
+ fi
+ if [[ ! -d ${data_dir} ]]; then
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ exit 1
+ fi
+
+ temp_dir=$(mktemp -d)
+ cd ${work_path}
+ echo "[INFO] Start collecting the profiles."
+ find ${data_dir} -type d -exec mkdir -p ${temp_dir}/\{\} \;
+ find ${data_dir} -name '*.gcda' -exec sh -c 'cat < $0 > '${temp_dir}'/$0' {} \;
+ find ${data_dir} -name '*.gcno' -exec sh -c 'cp -d $0 '${temp_dir}'/$0' {} \;
+
+ echo "[INFO] Start post-processing the profiles."
+ find ${temp_dir} -name '*.gcda' >list.txt
+ /usr/bin/g++ ${afot_path}/GcovSummaryAddTool.cpp -o calcsum
+ ./calcsum list.txt
+ rm -f list.txt
+
+ profile_dir=${time_dir}/gcovdata
+ rm -rf ${profile_dir}
+ mkdir ${profile_dir}
+ for file in $(find ${temp_dir} -name '*.gcda'); do
+ hash_path=$(echo ${file//\//\#})
+ name=$(echo ${hash_path#*gcov})
+ mv $file ${profile_dir}/$name
+ done
+ rm -rf ${temp_dir}
+ cd - >/dev/null
+ echo "[INFO] Profiles have been successfully processed, the path is: ${profile_dir}"
+}
+
+# 使用profile编译优化的内核
+function second_compilation() {
+ if [[ -n ${disable_compilation} ]]; then
+ sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += -fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" ${makefile}
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ exit 0
+ fi
+
+ if [[ -z ${kernel_src} ]]; then
+ kernel_src=${work_path}/kernel
+ fi
+ if [[ ! -d ${kernel_src} ]]; then
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ exit 1
+ fi
+
+ cd ${kernel_src}
+ make openeuler_defconfig
+ for config in ${kernel_configs[@]}; do
+ sed -i 's/.*${config%%=*}.*/${config}/' .config
+ done
+ sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-${kernel_name}-pgoed\"/" .config
+ echo "[INFO] Start optimized kernel compilation."
+ Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ is_success ${PIPESTATUS[0]}
+ cd - >/dev/null
+}
+
+# 第二次重启操作
+function second_reboot() {
+ grub2-set-default 0
+ if [[ -z ${silent} ]]; then
+ read -p $'Optimized kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
+ fi
+ if [[ -n ${silent} ]] || [[ ${reboot_p} == "y" ]]; then
+ echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ sleep 10
+ reboot
+ exit 0
+ fi
+}
+
+# 执行入口
+function main() {
+ check_config_items
+ if [[ ${pgo_phase} -eq 1 ]]; then
+ check_dependency
+ init
+ first_compilation
+ first_reboot
+ fi
+ if [[ ${pgo_phase} -eq 2 ]]; then
+ check_dependency
+ init
+ execute_run_script
+ process_profiles
+ second_compilation
+ second_reboot
+ fi
+ exit "$?"
+}
+
+main
+exit "$?"
--
2.33.0

889
0004-Add-reboot-action-to-fasten-the-second-compilation.patch
View File

@ -1,889 +0,0 @@
From 71ea86a3c08caaa65472f66f70dd550ead05aa9f Mon Sep 17 00:00:00 2001
From: xiongzhou4 <xiongzhou4@huawei.com>
Date: Tue, 13 Jun 2023 14:36:32 +0800
Subject: [PATCH] [perf] [fix] [style] Add reboot action to fasten the second
compilation, fix bugs and format codes.
---
a-fot | 128 +++++++++++++++++------------------------
a-fot.ini | 6 +-
auto_bolt.sh | 19 +++----
auto_fdo.sh | 24 ++++----
auto_kernel_pgo.sh | 139 ++++++++++++++++++++++++++-------------------
auto_prefetch.sh | 22 +++----
6 files changed, 164 insertions(+), 174 deletions(-)
diff --git a/a-fot b/a-fot
index 1e7d08e..bc9e38f 100755
--- a/a-fot
+++ b/a-fot
@@ -1,8 +1,8 @@
#!/bin/bash
-afot_path=$(cd "$(dirname "$0")";pwd)
+afot_path=$(cd "$(dirname "$0")" && pwd)
config_file=${afot_path}/a-fot.ini
-# Profile名字
+# profile名字
profile_name="profile.data"
# gcov名字
gcov_name="profile.gcov"
@@ -20,7 +20,7 @@ function parse_config() {
key=$(echo ${line} | awk -F "=" '{print $1}')
value=$(echo ${line} | awk -F "=" '{print $2}')
if [[ ${key} =~ "CONFIG_" ]]; then
- kernel_configs+="${key}=${value}"
+ kernel_configs+=("${key}=${value}")
else
eval "${key}=${value}"
fi
@@ -101,7 +101,7 @@ function parse_input_params() {
shift 2
;;
--CONFIG_*)
- kernel_configs+="${1:2}=$2"
+ kernel_configs+=("${1:2}=$2")
shift 2
;;
--last_time)
@@ -147,52 +147,26 @@ function parse_input_params() {
fi
}
-function check_config_item() {
- if [[ -z ${application_name} ]]; then
- echo "[ERROR] The configuration item 'application_name' is missing, please check!"
- exit 1
- fi
- if [[ -z ${bin_file} ]]; then
- echo "[ERROR] The configuration item 'bin_file' is missing, please check!"
+# 检查某个配置项是否存在
+function check_item() {
+ if [[ -z $1 ]]; then
+ echo "[ERROR] The configuration item '$2' is missing, please check!"
exit 1
fi
+}
- if [[ -z ${work_path} ]]; then
- echo "[ERROR] The configuration item 'work_path' is missing, please check!"
- exit 1
- fi
- if [[ -z ${build_script} ]]; then
- echo "[ERROR] The configuration item 'build_script' is missing, please check!"
- exit 1
- fi
- if [[ -z ${run_script} ]]; then
- echo "[ERROR] The configuration item 'run_script' is missing, please check!"
- exit 1
- fi
- if [[ -z ${max_waiting_time} ]]; then
- echo "[ERROR] The configuration item 'max_waiting_time' is missing, please check!"
- exit 1
- fi
- if [[ -z ${opt_mode} ]]; then
- echo "[ERROR] The configuration item 'opt_mode' is missing, please check!"
- exit 1
- fi
- if [[ -z ${perf_time} ]]; then
- echo "[ERROR] The configuration item 'perf_time' is missing, please check!"
- exit 1
- fi
- if [[ -z ${gcc_path} ]]; then
- echo "[ERROR] The configuration item 'gcc_path' is missing, please check!"
- exit 1
- fi
- if [[ -z ${check_success} ]]; then
- echo "[ERROR] The configuration item 'check_success' is missing, please check!"
- exit 1
- fi
- if [[ -z ${build_mode} ]]; then
- echo "[ERROR] The configuration item 'build_mode' is missing, please check!"
- exit 1
- fi
+function check_config_items() {
+ check_item ${application_name} application_name
+ check_item ${bin_file} bin_file
+ check_item ${work_path} work_path
+ check_item ${build_script} build_script
+ check_item ${run_script} run_script
+ check_item ${max_waiting_time} max_waiting_time
+ check_item ${opt_mode} opt_mode
+ check_item ${perf_time} perf_time
+ check_item ${gcc_path} gcc_path
+ check_item ${check_success} check_success
+ check_item ${build_mode} build_mode
}
function suggest_info() {
@@ -246,7 +220,7 @@ function load_script() {
source ${afot_path}/auto_bolt.sh
;;
*)
- echo "[ERROR] Optimization mode ${opt_mode} is not supported, Check the configuration item: opt_mode"
+ echo "[ERROR] Optimization mode ${opt_mode} is not supported, check the configuration item: opt_mode"
exit 1
;;
esac
@@ -254,15 +228,15 @@ function load_script() {
# 公共依赖检查项
function check_common_dependency() {
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]]; then
echo "[INFO] Current arch: ${get_arch}"
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
if ! type perf &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but perf is missing, try 'yum install perf'"
+ echo "[ERROR] Optimization mode ${opt_mode} but perf is missing, try 'yum install perf'."
exit 1
fi
is_file_exist ${build_script}
@@ -272,24 +246,24 @@ function check_common_dependency() {
# 拆分编译数据库
function split_option() {
- if [ "$bear_prefix" ];then
- python3 $afot_path/split_json.py -i $PWD/compile_commands.json
- mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
- mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
+ if [ "$bear_prefix" ]; then
+ python3 $afot_path/split_json.py -i $PWD/compile_commands.json
+ mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
+ mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
fi
}
# 使用原始编译选项进行编译
function first_compilation() {
- echo "[INFO] Start raw compilation"
+ echo "[INFO] Start raw compilation."
is_file_exist ${build_script} "build_script"
if [[ $build_mode =~ "Bear" ]]; then
- bear_prefix="bear -- "
- echo "[INFO] Build in Bear mode"
+ bear_prefix="bear -- "
+ echo "[INFO] Build in Bear mode."
else
- echo "[INFO] Build in Wrapper mode"
+ echo "[INFO] Build in Wrapper mode."
fi
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1
split_option first
is_file_exist ${bin_file}
is_success $?
@@ -312,24 +286,24 @@ function execute_run_script() {
echo "[INFO] Start to execute the run_script: ${run_script}"
process_id=$(pidof ${application_name})
if [[ -n ${process_id} ]]; then
- echo "[ERROR] Application: ${application_name} process already exists. The run_script will not be executed. Please check"
+ echo "[ERROR] Application: ${application_name} process already exists. The run_script will not be executed. Please check!"
exit 1
fi
is_file_exist ${run_script} "run_script"
- /bin/bash ${run_script} >> ${log_file} 2>&1 &
+ /bin/bash ${run_script} >>${log_file} 2>&1 &
is_success $?
}
# 探测应用进程是否存在
function detect_process() {
- echo "[INFO] Start to detect whether process ${application_name} is started"
+ echo "[INFO] Start to detect whether process ${application_name} is started."
detect_time=0
while [ -z $(pidof ${application_name}) ]; do
sleep 1
((detect_time = ${detect_time} + 1))
- echo "[INFO] Finding ${application_name}"
+ echo "[INFO] Finding ${application_name}."
if [[ ${detect_time} -gt ${max_waiting_time} ]]; then
- echo "[ERROR] Process ${application_name} is not found after ${max_waiting_time}. Please check"
+ echo "[ERROR] Process ${application_name} is not found after ${max_waiting_time}. Please check!"
exit 1
fi
done
@@ -340,12 +314,12 @@ function detect_process() {
# 需注意此检查依赖wrapper中编译器后添加第一个编译选项，
# 因此需保证编译器后添加第一个编译选项为优化选项而非通用选项
function second_compilation() {
- echo "[INFO] Try compiling with the new compilation options"
+ echo "[INFO] Try compiling with the new compilation options."
if [[ ${check_success} -eq 1 ]]; then
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1 & build_id=$!
echo "[INFO] Found build id: ${build_id}"
add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}')
- build_status=`ps -p ${build_id} | grep -c ${build_id}`
+ build_status=$(ps -p ${build_id} | grep -c ${build_id})
opt_success=0
while [[ ${build_status} -ne 0 ]]; do
if [[ ${opt_success} -eq 0 ]]; then
@@ -355,13 +329,13 @@ function second_compilation() {
break
fi
fi
- build_status=`ps -p ${build_id} | grep -c ${build_id}`
+ build_status=$(ps -p ${build_id} | grep -c ${build_id})
done
wait
else
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1
fi
- echo "[INFO] Finish compiling with new compilation options"
+ echo "[INFO] Finish compiling with new compilation options."
split_option second
is_success $?
}
@@ -381,9 +355,9 @@ function is_file_exist() {
config_item=$2
if [[ ! -f ${file} ]]; then
if [[ -n ${config_item} ]]; then
- echo "[ERROR] The file ${file} does not exist. Check the configuration item: ${config_item}"
+ echo "[ERROR] The file ${file} does not exist! Check the configuration item: ${config_item}"
else
- echo "[ERROR] The file ${file} does not exist"
+ echo "[ERROR] The file ${file} does not exist!"
fi
exit 1
fi
@@ -391,7 +365,7 @@ function is_file_exist() {
#初始化profile文件夹和log文件
function init_profile_and_log() {
- # Profile和log所在路径
+ # profile和log所在路径
now_time=$(date '+%Y%m%d-%H%M%S')
profile_data_path=${work_path}/${now_time}
log_file=${work_path}/${now_time}/opt.log
@@ -413,10 +387,10 @@ function is_opt_success() {
if [[ ${check_success} -eq 1 ]]; then
if [[ ${opt_success} -eq 0 ]]; then
echo "[WARNING] Optimization may fail or the build process is too short, please check!"
- echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler"
+ echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler."
exit 1
else
- echo "[INFO] Optimization may success!"
+ echo "[INFO] Optimization success!"
fi
fi
exit 0
@@ -426,7 +400,7 @@ function do_optimization() {
if [[ ${opt_mode} == Auto_kernel_PGO ]]; then
source ${afot_path}/auto_kernel_pgo.sh
fi
- check_config_item
+ check_config_items
init_profile_and_log
load_script
diff --git a/a-fot.ini b/a-fot.ini
index c5da1b0..859ab92 100644
--- a/a-fot.ini
+++ b/a-fot.ini
@@ -1,7 +1,7 @@
# 文件和目录请使用绝对路径
# 优化模式（AutoFDO、AutoPrefetch、AutoBOLT、Auto_kernel_PGO）
-opt_mode=Auto_kernel_PGO
+opt_mode=AutoPrefetch
# 脚本工作目录（用来编译应用程序/存放profile、日志）
work_path=/opt
# 应用运行脚本路径
@@ -25,7 +25,7 @@ perf_time=100
# 检测是否优化成功(1=启用，0=禁用)
check_success=0
# 构建模式 （Bear、Wrapper）
-build_mode=Bear
+build_mode=Wrapper
# auto_kernel_PGO
# 针对内核的优化模式，请填写此部分配置
@@ -47,4 +47,4 @@ makefile=
# 内核配置文件路径（用于不自动编译内核的场景）
kernel_config=
# 内核生成的原始profile目录（用于不自动编译内核的场景）
-data_dir=
\ No newline at end of file
+data_dir=
diff --git a/auto_bolt.sh b/auto_bolt.sh
index ebcb2e5..2102541 100644
--- a/auto_bolt.sh
+++ b/auto_bolt.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type llvm-bolt &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but llvm-bolt is missing, try 'yum install llvm-bolt'"
+ echo "[ERROR] Optimization mode ${opt_mode} but llvm-bolt is missing, try 'yum install llvm-bolt'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -26,10 +26,9 @@ function prepare_env() {
esac
}
-
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -37,18 +36,17 @@ function create_wrapper() {
# 执行perf采样，生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- perf record -e cycles:u -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e cycles:u -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- perf2bolt -p=${profile_data_path}/${profile_name} ${bin_file} -o ${profile_data_path}/${gcov_name} -nl >> ${log_file} 2>&1
+ perf2bolt -p=${profile_data_path}/${profile_name} ${bin_file} -o ${profile_data_path}/${gcov_name} -nl >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${gcov_name}"
pkill ${application_name}
}
-
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -64,10 +62,9 @@ function prepare_new_env() {
esac
}
-
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_fdo.sh b/auto_fdo.sh
index 8426b30..b8cc8cd 100644
--- a/auto_fdo.sh
+++ b/auto_fdo.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type create_gcov &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'"
+ echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -28,7 +28,7 @@ function prepare_env() {
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -36,28 +36,28 @@ function create_wrapper() {
# 执行perf采样，生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" ]]; then
perf_event="br_inst_retired.near_taken:u"
use_lbr=1
- elif [[ ${get_arch} =~ "aarch64" ]];then
+ elif [[ ${get_arch} =~ "aarch64" ]]; then
perf_event="inst_retired:u"
use_lbr=0
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
- perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=${use_lbr} >> ${log_file} 2>&1
+ create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=${use_lbr} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${gcov_name}"
pkill ${application_name}
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -66,7 +66,7 @@ function prepare_new_env() {
export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}"
;;
*)
- echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear."
exit 1
;;
esac
@@ -74,7 +74,7 @@ function prepare_new_env() {
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_kernel_pgo.sh b/auto_kernel_pgo.sh
index 0b62220..d7814e8 100644
--- a/auto_kernel_pgo.sh
+++ b/auto_kernel_pgo.sh
@@ -2,25 +2,22 @@
parallel=$(cat /proc/cpuinfo | grep "processor" | wc -l)
-# 检查某个配置项是否存在
-function check_item() {
- if [[ -z $1 ]]; then
- echo "[ERROR] The configuration item '$2' is missing, please check!"
- exit 1
- fi
+# 取消A-FOT脚本的自动执行
+cancel_afot_rc() {
+ sed -i '/a-fot/d' /etc/rc.d/rc.local
}
# 检查所有配置项
function check_config_items() {
check_item ${pgo_phase} pgo_phase
if [[ ${pgo_phase} -ne 1 ]] && [[ ${pgo_phase} -ne 2 ]]; then
- echo "[ERROR] The value of configuration item 'pgo_phase' is invalid, please check!"
+ echo "[ERROR] The value of configuration item 'pgo_phase' is invalid, please check!" | tee -a ${log_file}
exit 1
fi
if [[ ${pgo_phase} -eq 1 ]]; then
check_item ${pgo_mode} pgo_mode
if [[ ${pgo_mode} != "arc" ]] && [[ ${pgo_mode} != "all" ]]; then
- echo "[ERROR] The value of configuration item 'pgo_mode' is invalid, please check!"
+ echo "[ERROR] The value of configuration item 'pgo_mode' is invalid, please check!" | tee -a ${log_file}
exit 1
fi
fi
@@ -28,7 +25,8 @@ function check_config_items() {
check_item ${kernel_name} kernel_name
check_item ${work_path} work_path
if [[ ${work_path} =~ "/tmp" ]]; then
- echo "[ERROR] Do not put work path under /tmp, or it will be cleaned after rebooting."
+ echo "[ERROR] Do not put work path under /tmp, or it will be cleaned after rebooting." | tee -a ${log_file}
+ exit 1
fi
check_item ${run_script} run_script
check_item ${gcc_path} gcc_path
@@ -43,7 +41,7 @@ function check_config_items() {
if [[ ${pgo_phase} -eq 2 ]]; then
check_item ${data_dir} data_dir
if [[ ! -d ${data_dir} ]]; then
- echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist." | tee -a ${log_file}
exit 1
fi
fi
@@ -52,7 +50,7 @@ function check_config_items() {
# 初始化文件和目录
function init() {
- if [[ -z ${last_time} ]]; then
+ if [[ -z ${last_time} ]] || [[ ${pgo_phase} -eq 1 ]]; then
now_time=$(date '+%Y%m%d-%H%M%S')
else
now_time=${last_time}
@@ -61,29 +59,27 @@ function init() {
log_file=${time_dir}/log
if [[ -d ${time_dir} ]]; then
- echo "[INFO] Use last time directory: ${time_dir}"
+ echo "[INFO] Use last time directory: ${time_dir}" | tee -a ${log_file}
else
mkdir -p ${time_dir}
- echo "[INFO] Create time directory: ${time_dir}"
+ echo "[INFO] Create time directory: ${time_dir}" | tee -a ${log_file}
fi
if [[ -f ${log_file} ]]; then
- echo "[INFO] Use last log: ${log_file}"
+ echo "[INFO] Use last log: ${log_file}" | tee -a ${log_file}
else
touch ${log_file}
- echo "[INFO] Init log file: ${log_file}"
+ echo "[INFO] Init log file: ${log_file}" | tee -a ${log_file}
fi
}
# 检测环境
function check_dependency() {
- sed -i '/a-fot/d' /etc/rc.d/rc.local
-
get_arch=$(arch)
if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]]; then
- echo "[INFO] Current arch: ${get_arch}"
+ echo "[INFO] Current arch: ${get_arch}" | tee -a ${log_file}
else
- echo "[ERROR] Unsupport arch: ${get_arch}"
+ echo "[ERROR] Unsupport arch: ${get_arch}" | tee -a ${log_file}
exit 1
fi
is_file_exist ${run_script} "run_script"
@@ -95,12 +91,12 @@ function check_dependency() {
if [[ ${pgo_phase} -eq 1 ]] && [[ ${pgo_mode} == "all" ]] && [[ -z ${disable_compilation} ]]; then
if echo 'int main() { return 0; }' | gcc -x c - -o /dev/null -Werror -fkernel-pgo >/dev/null 2>&1; then
- echo "[INFO] Current GCC supports kernel PGO in runtime, use -fkernel-pgo option."
+ echo "[INFO] Current GCC supports kernel PGO in runtime, use -fkernel-pgo option." | tee -a ${log_file}
option="-fkernel-pgo"
elif gcc -v 2>&1 | grep -- "--disable-tls" >/dev/null; then
- echo "[INFO] Current GCC is recompiled with --disable-tls, does support kernel PGO in runtime."
+ echo "[INFO] Current GCC is recompiled with --disable-tls, does support kernel PGO in runtime." | tee -a ${log_file}
else
- echo "[ERROR] Current GCC does not support kernel PGO, you may use openeuler GCC or recompile GCC with --disable-tls --disable-libsanitizer."
+ echo "[ERROR] Current GCC does not support kernel PGO, you may use openeuler GCC or recompile GCC with --disable-tls --disable-libsanitizer." | tee -a ${log_file}
fi
fi
}
@@ -109,7 +105,7 @@ function check_dependency() {
function modify_kernel_config() {
if [[ ${pgo_mode} == "all" ]]; then
if ! grep "PGO" $1 >/dev/null; then
- echo "[ERROR] Current version of kernel does not support PGO, please use newer ones or choose arc mode."
+ echo "[ERROR] Current version of kernel does not support PGO, please use newer ones or choose arc mode." | tee -a ${log_file}
exit 1
fi
sed -i 's/.*CONFIG_PGO_KERNEL.*/CONFIG_PGO_KERNEL=y/' $1
@@ -128,27 +124,27 @@ function modify_kernel_config() {
function first_compilation() {
if [[ -n ${disable_compilation} ]]; then
modify_kernel_config ${kernel_config}
- echo "[INFO] Kernel configuration has been generated, the path is: ${kernel_config}"
+ echo "[INFO] Kernel configuration has been generated, the path is: ${kernel_config}" | tee -a ${log_file}
if [[ -z ${option} ]]; then
sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += fkernel-pgo" ${makefile}
fi
- echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}" | tee -a ${log_file}
exit 0
fi
if [[ -z ${kernel_src} ]]; then
- echo "[WARNING] ${kernel_src} is not specified, A-FOT will download and build kernel source code automatically in 10 seconds."
+ echo "[WARNING] ${kernel_src} is not specified, A-FOT will download and build kernel source code automatically in 10 seconds." | tee -a ${log_file}
sleep 10
oe_version=$(grep 'openeulerversion' /etc/openEuler-latest | cut -d '=' -f 2)
- echo "[INFO] Current openEuler version: ${oe_version}"
+ echo "[INFO] Current openEuler version: ${oe_version}" | tee -a ${log_file}
url="https://repo.huaweicloud.com/openeuler/${oe_version}/source/Packages/"
- echo "[INFO] Download the kernel source rpm package from ${url}"
+ echo "[INFO] Download the kernel source rpm package from ${url}" | tee -a ${log_file}
cd ${work_path}
wget --no-parent --no-directories -r -A 'kernel-[0-9]*.rpm' ${url} --no-check-certificate >>${log_file} 2>&1
is_success $?
srpm=$(ls -t kernel-[0-9]*.rpm 2>/dev/null | head -n 1)
- echo "[INFO] Successfully downloaded kernel source rpm package: ${srpm}"
- echo "[INFO] Build kernel source code."
+ echo "[INFO] Successfully downloaded kernel source rpm package: ${srpm}" | tee -a ${log_file}
+ echo "[INFO] Build kernel source code." | tee -a ${log_file}
rpm -ivh ${srpm} >>${log_file} 2>&1
cd - >/dev/null
rpmbuild -bp ~/rpmbuild/SPECS/kernel.spec >>${log_file} 2>&1
@@ -156,44 +152,48 @@ function first_compilation() {
src_dir=$(ls -td ~/rpmbuild/BUILD/kernel-*/*-source/ | head -n 1)
cp -r ${src_dir} ${work_path}/kernel
kernel_src=${work_path}/kernel
- echo "[INFO] Successfully builded kernel source code: ${kernel_src}"
+ echo "[INFO] Successfully builded kernel source code: ${kernel_src}" | tee -a ${log_file}
else
- echo "[INFO] Build kernel in an existing source directory."
+ echo "[INFO] Build kernel in an existing source directory." | tee -a ${log_file}
fi
if [[ ! -d ${kernel_src} ]]; then
- echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist." | tee -a ${log_file}
exit 1
fi
cd ${kernel_src}
make openeuler_defconfig
modify_kernel_config .config
- echo "[INFO] Start PGO kernel compilation."
- Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
- (make clean -j ${parallel} && make KCFLAGS="${option}" ARCH=${Arch} -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ echo "[INFO] Start PGO kernel compilation." | tee -a ${log_file}
+ arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="${option}" ARCH=${arch} -j ${parallel} && make modules_install ARCH=${arch} -j ${parallel} && make install ARCH=${arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
is_success ${PIPESTATUS[0]}
cd - >/dev/null
}
# 第一次重启操作
function first_reboot() {
- grub2-set-default 0
next_cmd="${afot_path}/a-fot --opt_mode Auto_kernel_PGO --pgo_phase 2 --kernel_src ${kernel_src} --kernel_name ${kernel_name} --work_path ${work_path} --run_script ${run_script} --gcc_path ${gcc_path} --last_time ${now_time} -s"
+ for config in ${kernel_configs[@]}; do
+ next_cmd+=" --${config}"
+ done
+
+ grub2-set-default 0
if [[ -z ${silent} ]]; then
read -p $'PGO kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
if [[ ${reboot_p} == "y" ]]; then
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
- echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}" | tee -a ${log_file}
sleep 10
reboot
else
- echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}" | tee -a ${log_file}
fi
else
echo ${next_cmd} >>/etc/rc.d/rc.local
- chmod u+x /etc/rc.d/rc.local
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ chmod +x /etc/rc.d/rc.local
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
sleep 10
reboot
fi
@@ -202,7 +202,7 @@ function first_reboot() {
# 执行应用程序执行脚本
function execute_run_script() {
- echo "[INFO] Start to execute the run_script: ${run_script}"
+ echo "[INFO] Start to execute the run_script: ${run_script}" | tee -a ${log_file}
/bin/bash ${run_script} >>${log_file} 2>&1
is_success $?
}
@@ -213,18 +213,18 @@ function process_profiles() {
data_dir=/sys/kernel/debug/gcov
fi
if [[ ! -d ${data_dir} ]]; then
- echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist." | tee -a ${log_file}
exit 1
fi
temp_dir=$(mktemp -d)
cd ${work_path}
- echo "[INFO] Start collecting the profiles."
+ echo "[INFO] Start collecting the profiles." | tee -a ${log_file}
find ${data_dir} -type d -exec mkdir -p ${temp_dir}/\{\} \;
find ${data_dir} -name '*.gcda' -exec sh -c 'cat < $0 > '${temp_dir}'/$0' {} \;
find ${data_dir} -name '*.gcno' -exec sh -c 'cp -d $0 '${temp_dir}'/$0' {} \;
- echo "[INFO] Start post-processing the profiles."
+ echo "[INFO] Start post-processing the profiles." | tee -a ${log_file}
find ${temp_dir} -name '*.gcda' >list.txt
/usr/bin/g++ ${afot_path}/GcovSummaryAddTool.cpp -o calcsum
./calcsum list.txt
@@ -240,14 +240,14 @@ function process_profiles() {
done
rm -rf ${temp_dir}
cd - >/dev/null
- echo "[INFO] Profiles have been successfully processed, the path is: ${profile_dir}"
+ echo "[INFO] Profiles have been successfully processed, the path is: ${profile_dir}" | tee -a ${log_file}
}
# 使用profile编译优化的内核
function second_compilation() {
if [[ -n ${disable_compilation} ]]; then
- sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += -fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" ${makefile}
- echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += -fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -Wno-error=missing-profile -fprofile-dir=${profile_dir}" ${makefile}
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}" | tee -a ${log_file}
exit 0
fi
@@ -255,19 +255,37 @@ function second_compilation() {
kernel_src=${work_path}/kernel
fi
if [[ ! -d ${kernel_src} ]]; then
- echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist." | tee -a ${log_file}
exit 1
fi
+ if [[ -f ${time_dir}/.flag ]]; then
+ rm -f ${time_dir}/.flag
+ else
+ next_cmd="${afot_path}/a-fot --opt_mode Auto_kernel_PGO --pgo_phase 2 --kernel_src ${kernel_src} --kernel_name ${kernel_name} --work_path ${work_path} --run_script ${run_script} --gcc_path ${gcc_path} --last_time ${now_time} -s"
+ for config in ${kernel_configs[@]}; do
+ next_cmd+=" --${config}"
+ done
+ touch ${time_dir}/.flag
+ echo ${next_cmd} >>/etc/rc.d/rc.local
+ chmod +x /etc/rc.d/rc.local
+ echo "[INFO] Switch to normal kernel for faster compilation." | tee -a ${log_file}
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
+ grub2-set-default 1
+ sleep 10
+ reboot
+ exit 0
+ fi
+
cd ${kernel_src}
make openeuler_defconfig
for config in ${kernel_configs[@]}; do
sed -i 's/.*${config%%=*}.*/${config}/' .config
done
sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-${kernel_name}-pgoed\"/" .config
- echo "[INFO] Start optimized kernel compilation."
- Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
- (make clean -j ${parallel} && make KCFLAGS="-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ echo "[INFO] Start optimized kernel compilation." | tee -a ${log_file}
+ arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -Wno-error=missing-profile -fprofile-dir=${time_dir}/gcovdata" -j ${parallel} && make modules_install ARCH=${arch} -j ${parallel} && make install ARCH=${arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
is_success ${PIPESTATUS[0]}
cd - >/dev/null
}
@@ -279,7 +297,7 @@ function second_reboot() {
read -p $'Optimized kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
fi
if [[ -n ${silent} ]] || [[ ${reboot_p} == "y" ]]; then
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
sleep 10
reboot
exit 0
@@ -288,18 +306,19 @@ function second_reboot() {
# 执行入口
function main() {
+ cancel_afot_rc
check_config_items
+ check_dependency
+ init
if [[ ${pgo_phase} -eq 1 ]]; then
- check_dependency
- init
first_compilation
first_reboot
fi
if [[ ${pgo_phase} -eq 2 ]]; then
- check_dependency
- init
- execute_run_script
- process_profiles
+ if [[ ! -f ${time_dir}/.flag ]]; then
+ execute_run_script
+ process_profiles
+ fi
second_compilation
second_reboot
fi
diff --git a/auto_prefetch.sh b/auto_prefetch.sh
index b7f5932..3950b17 100644
--- a/auto_prefetch.sh
+++ b/auto_prefetch.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type create_gcov &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'"
+ echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -28,7 +28,7 @@ function prepare_env() {
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -36,29 +36,29 @@ function create_wrapper() {
# 执行perf采样，生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" ]]; then
perf_event="inst_retired.prec_dist:u,cache-misses:u"
gcov_file_name="${profile_data_path}/${gcov_name}.inst_retired.prec_dist:u"
- elif [[ ${get_arch} =~ "aarch64" ]];then
+ elif [[ ${get_arch} =~ "aarch64" ]]; then
perf_event="inst_retired:u,cache-misses:u"
gcov_file_name="${profile_data_path}/${gcov_name}.inst_retired:u"
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
- perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=0 >> ${log_file} 2>&1
+ create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=0 >>${log_file} 2>&1
is_file_exist "${gcov_file_name}"
is_file_exist "${profile_data_path}/${gcov_name}.cache-misses:u"
pkill ${application_name}
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -75,7 +75,7 @@ function prepare_new_env() {
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++
}
--
2.33.0

88
0005-refactor-Reuse-calcsum-delay-pre-reboot-operations.patch
View File

@ -1,88 +0,0 @@
From 3497d5910c57744d04f8d74e619bec347a2c141e Mon Sep 17 00:00:00 2001
From: xiongzhou4 <xiongzhou4@huawei.com>
Date: Tue, 11 Jul 2023 14:43:47 +0800
Subject: [PATCH] [refactor] Reuse calcsum, delay pre-reboot operations and
fixs typos.
---
GcovSummaryAddTool.cpp | 2 +-
a-fot | 2 +-
auto_kernel_pgo.sh | 17 ++++++++++-------
3 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/GcovSummaryAddTool.cpp b/GcovSummaryAddTool.cpp
index 656f3d5..bbcc94b 100644
--- a/GcovSummaryAddTool.cpp
+++ b/GcovSummaryAddTool.cpp
@@ -171,6 +171,6 @@ int main(int argc, char** argv)
return 1;
}
}
- fprintf(stderr, "[.] File procesed: %d \n", (int) fileNames.size());
+ fprintf(stderr, "[.] File processed: %d \n", (int) fileNames.size());
return 0;
}
diff --git a/a-fot b/a-fot
index bc9e38f..151efd7 100755
--- a/a-fot
+++ b/a-fot
@@ -197,7 +197,7 @@ For kernel PGO mode:
--work_path Script working directory (used to store the profile and the log)
--run_script Script path for running application
--gcc_path Compiler gcc path
---CONFIG_... Kernel building
+--CONFIG_... Kernel building configuration
--last_time Last time directory before rebooting (used to put log infos together)
-s Silent mode (reboot automatically after kernel installation)
-n Do not compile kernel automatically
diff --git a/auto_kernel_pgo.sh b/auto_kernel_pgo.sh
index d7814e8..9e90769 100644
--- a/auto_kernel_pgo.sh
+++ b/auto_kernel_pgo.sh
@@ -191,10 +191,10 @@ function first_reboot() {
echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}" | tee -a ${log_file}
fi
else
- echo ${next_cmd} >>/etc/rc.d/rc.local
- chmod +x /etc/rc.d/rc.local
echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
sleep 10
+ echo ${next_cmd} >>/etc/rc.d/rc.local
+ chmod +x /etc/rc.d/rc.local
reboot
fi
exit 0
@@ -226,8 +226,11 @@ function process_profiles() {
echo "[INFO] Start post-processing the profiles." | tee -a ${log_file}
find ${temp_dir} -name '*.gcda' >list.txt
- /usr/bin/g++ ${afot_path}/GcovSummaryAddTool.cpp -o calcsum
- ./calcsum list.txt
+ calcsum=${afot_path}/calcsum
+ if [[ ! -f ${calcsum} ]]; then
+ /usr/bin/g++ ${afot_path}/GcovSummaryAddTool.cpp -o ${calcsum}
+ fi
+ ${calcsum} list.txt
rm -f list.txt
profile_dir=${time_dir}/gcovdata
@@ -266,13 +269,13 @@ function second_compilation() {
for config in ${kernel_configs[@]}; do
next_cmd+=" --${config}"
done
+ echo "[INFO] Switch to normal kernel for faster compilation." | tee -a ${log_file}
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
+ sleep 10
touch ${time_dir}/.flag
echo ${next_cmd} >>/etc/rc.d/rc.local
chmod +x /etc/rc.d/rc.local
- echo "[INFO] Switch to normal kernel for faster compilation." | tee -a ${log_file}
- echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
grub2-set-default 1
- sleep 10
reboot
exit 0
fi
--
2.33.0

BIN
A-FOT-v1.0.tar.gz
View File

Binary file not shown.

87
A-FOT.spec
View File

@ -1,87 +0,0 @@
Name: A-FOT
Version: v1.0
Release: 4
Summary: automatic feedback-directed optimization tool for openEuler
License: MulanPSL-2.0
URL: https://gitee.com/openeuler/A-FOT
Source0: %{name}-%{version}.tar.gz
Buildrequires: gcc-c++
Requires: gcc gcc-c++ autofdo llvm-bolt Bear python3
Patch1: 0001-Add-Bear-to-A-FOT.patch
Patch2: 0002-Bugfix-Remove-Backslash-in-Options.patch
Patch3: 0003-PGO-kernel-Add-PGO-kernel-mode.patch
Patch4: 0004-Add-reboot-action-to-fasten-the-second-compilation.patch
Patch5: 0005-refactor-Reuse-calcsum-delay-pre-reboot-operations.patch
%description
A-FOT is an automatic feedback-directed optimization tool for openEuler
# skip debuginfo packages
%global debug_package %{nil}
%prep
%autosetup -n %{name}-%{version}
%build
g++ GcovSummaryAddTool.cpp -o calcsum
%install
mkdir -p %{buildroot}/%{_bindir}
cp a-fot %{buildroot}/%{_bindir}
cp a-fot.ini %{buildroot}/%{_bindir}
cp auto_fdo.sh %{buildroot}/%{_bindir}
cp auto_prefetch.sh %{buildroot}/%{_bindir}
cp auto_bolt.sh %{buildroot}/%{_bindir}
cp auto_kernel_pgo.sh %{buildroot}/%{_bindir}
cp split_json.py %{buildroot}/%{_bindir}
cp calcsum %{buildroot}/%{_bindir}
%files
%attr(755, root, root) %{_bindir}/a-fot
%attr(664, root, root) %{_bindir}/a-fot.ini
%attr(644, root, root) %{_bindir}/auto_fdo.sh
%attr(644, root, root) %{_bindir}/auto_prefetch.sh
%attr(644, root, root) %{_bindir}/auto_bolt.sh
%attr(644, root, root) %{_bindir}/auto_kernel_pgo.sh
%attr(644, root, root) %{_bindir}/split_json.py
%attr(755, root, root) %{_bindir}/calcsum
%license LICENSE
%doc README.md
%changelog
* Tue Jul 11 2023 Xiong Zhou <xiongzhou4@huawei.com> - v1.0-4
- Type:Sync
- ID:NA
- SUG:NA
- DESC:Sync patches from openeuler/A-FOT
Add reboot action to fasten the second compilation,
fix bugs and format codes.
Reuse calcsum, delay pre-reboot operations and fixs typos.
* Fri Jun 2 2023 Xiong Zhou <xiongzhou4@huawei.com> - v1.0-3
- Type:Sync
- ID:NA
- SUG:NA
- DESC:Sync patch from openeuler/A-FOT
Add PGO kernel mode.
* Mon Dec 19 2022 huitailangzju <804544223@qq.com> - v1.0-2
- Type:Sync
- ID:NA
- SUG:NA
- DESC:Remove backslash in options
* Thu Dec 01 2022 huitailangzju <804544223@qq.com> - v1.0-1
- Type:Sync
- ID:NA
- SUG:NA
- DESC:Add Bear to A-FOT
* Wed Mar 23 2022 liyancheng <412998149@qq.com> - 0
- Type:Init
- ID:NA
- SUG:NA
- DESC:Add automatic feedback optimization tool