A-FOT/0004-Add-reboot-action-to-fasten-the-second-compilation.patch
xiongzhou4 95be8a7d6b [PGO kernel] Sync patches from openeuler/A-FOT.
(cherry picked from commit 024f36760aefec39f1300a592e10a6f80b6cc57c)
2023-07-24 14:24:34 +08:00

890 lines
36 KiB
Diff
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

From 71ea86a3c08caaa65472f66f70dd550ead05aa9f Mon Sep 17 00:00:00 2001
From: xiongzhou4 <xiongzhou4@huawei.com>
Date: Tue, 13 Jun 2023 14:36:32 +0800
Subject: [PATCH] [perf] [fix] [style] Add reboot action to fasten the second
compilation, fix bugs and format codes.
---
a-fot | 128 +++++++++++++++++------------------------
a-fot.ini | 6 +-
auto_bolt.sh | 19 +++----
auto_fdo.sh | 24 ++++----
auto_kernel_pgo.sh | 139 ++++++++++++++++++++++++++-------------------
auto_prefetch.sh | 22 +++----
6 files changed, 164 insertions(+), 174 deletions(-)
diff --git a/a-fot b/a-fot
index 1e7d08e..bc9e38f 100755
--- a/a-fot
+++ b/a-fot
@@ -1,8 +1,8 @@
#!/bin/bash
-afot_path=$(cd "$(dirname "$0")";pwd)
+afot_path=$(cd "$(dirname "$0")" && pwd)
config_file=${afot_path}/a-fot.ini
-# Profile名字
+# profile名字
profile_name="profile.data"
# gcov名字
gcov_name="profile.gcov"
@@ -20,7 +20,7 @@ function parse_config() {
key=$(echo ${line} | awk -F "=" '{print $1}')
value=$(echo ${line} | awk -F "=" '{print $2}')
if [[ ${key} =~ "CONFIG_" ]]; then
- kernel_configs+="${key}=${value}"
+ kernel_configs+=("${key}=${value}")
else
eval "${key}=${value}"
fi
@@ -101,7 +101,7 @@ function parse_input_params() {
shift 2
;;
--CONFIG_*)
- kernel_configs+="${1:2}=$2"
+ kernel_configs+=("${1:2}=$2")
shift 2
;;
--last_time)
@@ -147,52 +147,26 @@ function parse_input_params() {
fi
}
-function check_config_item() {
- if [[ -z ${application_name} ]]; then
- echo "[ERROR] The configuration item 'application_name' is missing, please check!"
- exit 1
- fi
- if [[ -z ${bin_file} ]]; then
- echo "[ERROR] The configuration item 'bin_file' is missing, please check!"
+# 检查某个配置项是否存在
+function check_item() {
+ if [[ -z $1 ]]; then
+ echo "[ERROR] The configuration item '$2' is missing, please check!"
exit 1
fi
+}
- if [[ -z ${work_path} ]]; then
- echo "[ERROR] The configuration item 'work_path' is missing, please check!"
- exit 1
- fi
- if [[ -z ${build_script} ]]; then
- echo "[ERROR] The configuration item 'build_script' is missing, please check!"
- exit 1
- fi
- if [[ -z ${run_script} ]]; then
- echo "[ERROR] The configuration item 'run_script' is missing, please check!"
- exit 1
- fi
- if [[ -z ${max_waiting_time} ]]; then
- echo "[ERROR] The configuration item 'max_waiting_time' is missing, please check!"
- exit 1
- fi
- if [[ -z ${opt_mode} ]]; then
- echo "[ERROR] The configuration item 'opt_mode' is missing, please check!"
- exit 1
- fi
- if [[ -z ${perf_time} ]]; then
- echo "[ERROR] The configuration item 'perf_time' is missing, please check!"
- exit 1
- fi
- if [[ -z ${gcc_path} ]]; then
- echo "[ERROR] The configuration item 'gcc_path' is missing, please check!"
- exit 1
- fi
- if [[ -z ${check_success} ]]; then
- echo "[ERROR] The configuration item 'check_success' is missing, please check!"
- exit 1
- fi
- if [[ -z ${build_mode} ]]; then
- echo "[ERROR] The configuration item 'build_mode' is missing, please check!"
- exit 1
- fi
+function check_config_items() {
+ check_item ${application_name} application_name
+ check_item ${bin_file} bin_file
+ check_item ${work_path} work_path
+ check_item ${build_script} build_script
+ check_item ${run_script} run_script
+ check_item ${max_waiting_time} max_waiting_time
+ check_item ${opt_mode} opt_mode
+ check_item ${perf_time} perf_time
+ check_item ${gcc_path} gcc_path
+ check_item ${check_success} check_success
+ check_item ${build_mode} build_mode
}
function suggest_info() {
@@ -246,7 +220,7 @@ function load_script() {
source ${afot_path}/auto_bolt.sh
;;
*)
- echo "[ERROR] Optimization mode ${opt_mode} is not supported, Check the configuration item: opt_mode"
+ echo "[ERROR] Optimization mode ${opt_mode} is not supported, check the configuration item: opt_mode"
exit 1
;;
esac
@@ -254,15 +228,15 @@ function load_script() {
# 公共依赖检查项
function check_common_dependency() {
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]]; then
echo "[INFO] Current arch: ${get_arch}"
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
if ! type perf &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but perf is missing, try 'yum install perf'"
+ echo "[ERROR] Optimization mode ${opt_mode} but perf is missing, try 'yum install perf'."
exit 1
fi
is_file_exist ${build_script}
@@ -272,24 +246,24 @@ function check_common_dependency() {
# 拆分编译数据库
function split_option() {
- if [ "$bear_prefix" ];then
- python3 $afot_path/split_json.py -i $PWD/compile_commands.json
- mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
- mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
+ if [ "$bear_prefix" ]; then
+ python3 $afot_path/split_json.py -i $PWD/compile_commands.json
+ mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
+ mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
fi
}
# 使用原始编译选项进行编译
function first_compilation() {
- echo "[INFO] Start raw compilation"
+ echo "[INFO] Start raw compilation."
is_file_exist ${build_script} "build_script"
if [[ $build_mode =~ "Bear" ]]; then
- bear_prefix="bear -- "
- echo "[INFO] Build in Bear mode"
+ bear_prefix="bear -- "
+ echo "[INFO] Build in Bear mode."
else
- echo "[INFO] Build in Wrapper mode"
+ echo "[INFO] Build in Wrapper mode."
fi
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1
split_option first
is_file_exist ${bin_file}
is_success $?
@@ -312,24 +286,24 @@ function execute_run_script() {
echo "[INFO] Start to execute the run_script: ${run_script}"
process_id=$(pidof ${application_name})
if [[ -n ${process_id} ]]; then
- echo "[ERROR] Application: ${application_name} process already exists. The run_script will not be executed. Please check"
+ echo "[ERROR] Application: ${application_name} process already exists. The run_script will not be executed. Please check!"
exit 1
fi
is_file_exist ${run_script} "run_script"
- /bin/bash ${run_script} >> ${log_file} 2>&1 &
+ /bin/bash ${run_script} >>${log_file} 2>&1 &
is_success $?
}
# 探测应用进程是否存在
function detect_process() {
- echo "[INFO] Start to detect whether process ${application_name} is started"
+ echo "[INFO] Start to detect whether process ${application_name} is started."
detect_time=0
while [ -z $(pidof ${application_name}) ]; do
sleep 1
((detect_time = ${detect_time} + 1))
- echo "[INFO] Finding ${application_name}"
+ echo "[INFO] Finding ${application_name}."
if [[ ${detect_time} -gt ${max_waiting_time} ]]; then
- echo "[ERROR] Process ${application_name} is not found after ${max_waiting_time}. Please check"
+ echo "[ERROR] Process ${application_name} is not found after ${max_waiting_time}. Please check!"
exit 1
fi
done
@@ -340,12 +314,12 @@ function detect_process() {
# 需注意此检查依赖wrapper中编译器后添加第一个编译选项
# 因此需保证编译器后添加第一个编译选项为优化选项而非通用选项
function second_compilation() {
- echo "[INFO] Try compiling with the new compilation options"
+ echo "[INFO] Try compiling with the new compilation options."
if [[ ${check_success} -eq 1 ]]; then
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1 & build_id=$!
echo "[INFO] Found build id: ${build_id}"
add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}')
- build_status=`ps -p ${build_id} | grep -c ${build_id}`
+ build_status=$(ps -p ${build_id} | grep -c ${build_id})
opt_success=0
while [[ ${build_status} -ne 0 ]]; do
if [[ ${opt_success} -eq 0 ]]; then
@@ -355,13 +329,13 @@ function second_compilation() {
break
fi
fi
- build_status=`ps -p ${build_id} | grep -c ${build_id}`
+ build_status=$(ps -p ${build_id} | grep -c ${build_id})
done
wait
else
- $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+ $bear_prefix /bin/bash ${build_script} >>${log_file} 2>&1
fi
- echo "[INFO] Finish compiling with new compilation options"
+ echo "[INFO] Finish compiling with new compilation options."
split_option second
is_success $?
}
@@ -381,9 +355,9 @@ function is_file_exist() {
config_item=$2
if [[ ! -f ${file} ]]; then
if [[ -n ${config_item} ]]; then
- echo "[ERROR] The file ${file} does not exist. Check the configuration item: ${config_item}"
+ echo "[ERROR] The file ${file} does not exist! Check the configuration item: ${config_item}"
else
- echo "[ERROR] The file ${file} does not exist"
+ echo "[ERROR] The file ${file} does not exist!"
fi
exit 1
fi
@@ -391,7 +365,7 @@ function is_file_exist() {
#初始化profile文件夹和log文件
function init_profile_and_log() {
- # Profile和log所在路径
+ # profile和log所在路径
now_time=$(date '+%Y%m%d-%H%M%S')
profile_data_path=${work_path}/${now_time}
log_file=${work_path}/${now_time}/opt.log
@@ -413,10 +387,10 @@ function is_opt_success() {
if [[ ${check_success} -eq 1 ]]; then
if [[ ${opt_success} -eq 0 ]]; then
echo "[WARNING] Optimization may fail or the build process is too short, please check!"
- echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler"
+ echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler."
exit 1
else
- echo "[INFO] Optimization may success!"
+ echo "[INFO] Optimization success!"
fi
fi
exit 0
@@ -426,7 +400,7 @@ function do_optimization() {
if [[ ${opt_mode} == Auto_kernel_PGO ]]; then
source ${afot_path}/auto_kernel_pgo.sh
fi
- check_config_item
+ check_config_items
init_profile_and_log
load_script
diff --git a/a-fot.ini b/a-fot.ini
index c5da1b0..859ab92 100644
--- a/a-fot.ini
+++ b/a-fot.ini
@@ -1,7 +1,7 @@
# 文件和目录请使用绝对路径
# 优化模式AutoFDO、AutoPrefetch、AutoBOLT、Auto_kernel_PGO
-opt_mode=Auto_kernel_PGO
+opt_mode=AutoPrefetch
# 脚本工作目录(用来编译应用程序/存放profile、日志
work_path=/opt
# 应用运行脚本路径
@@ -25,7 +25,7 @@ perf_time=100
# 检测是否优化成功(1=启用0=禁用)
check_success=0
# 构建模式 Bear、Wrapper
-build_mode=Bear
+build_mode=Wrapper
# auto_kernel_PGO
# 针对内核的优化模式,请填写此部分配置
@@ -47,4 +47,4 @@ makefile=
# 内核配置文件路径(用于不自动编译内核的场景)
kernel_config=
# 内核生成的原始profile目录用于不自动编译内核的场景
-data_dir=
\ No newline at end of file
+data_dir=
diff --git a/auto_bolt.sh b/auto_bolt.sh
index ebcb2e5..2102541 100644
--- a/auto_bolt.sh
+++ b/auto_bolt.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type llvm-bolt &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but llvm-bolt is missing, try 'yum install llvm-bolt'"
+ echo "[ERROR] Optimization mode ${opt_mode} but llvm-bolt is missing, try 'yum install llvm-bolt'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -26,10 +26,9 @@ function prepare_env() {
esac
}
-
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -37,18 +36,17 @@ function create_wrapper() {
# 执行perf采样生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- perf record -e cycles:u -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e cycles:u -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- perf2bolt -p=${profile_data_path}/${profile_name} ${bin_file} -o ${profile_data_path}/${gcov_name} -nl >> ${log_file} 2>&1
+ perf2bolt -p=${profile_data_path}/${profile_name} ${bin_file} -o ${profile_data_path}/${gcov_name} -nl >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${gcov_name}"
pkill ${application_name}
}
-
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -64,10 +62,9 @@ function prepare_new_env() {
esac
}
-
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_fdo.sh b/auto_fdo.sh
index 8426b30..b8cc8cd 100644
--- a/auto_fdo.sh
+++ b/auto_fdo.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type create_gcov &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'"
+ echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -28,7 +28,7 @@ function prepare_env() {
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -36,28 +36,28 @@ function create_wrapper() {
# 执行perf采样生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" ]]; then
perf_event="br_inst_retired.near_taken:u"
use_lbr=1
- elif [[ ${get_arch} =~ "aarch64" ]];then
+ elif [[ ${get_arch} =~ "aarch64" ]]; then
perf_event="inst_retired:u"
use_lbr=0
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
- perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=${use_lbr} >> ${log_file} 2>&1
+ create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=${use_lbr} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${gcov_name}"
pkill ${application_name}
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -66,7 +66,7 @@ function prepare_new_env() {
export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}"
;;
*)
- echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
+ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear."
exit 1
;;
esac
@@ -74,7 +74,7 @@ function prepare_new_env() {
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++
}
diff --git a/auto_kernel_pgo.sh b/auto_kernel_pgo.sh
index 0b62220..d7814e8 100644
--- a/auto_kernel_pgo.sh
+++ b/auto_kernel_pgo.sh
@@ -2,25 +2,22 @@
parallel=$(cat /proc/cpuinfo | grep "processor" | wc -l)
-# 检查某个配置项是否存在
-function check_item() {
- if [[ -z $1 ]]; then
- echo "[ERROR] The configuration item '$2' is missing, please check!"
- exit 1
- fi
+# 取消A-FOT脚本的自动执行
+cancel_afot_rc() {
+ sed -i '/a-fot/d' /etc/rc.d/rc.local
}
# 检查所有配置项
function check_config_items() {
check_item ${pgo_phase} pgo_phase
if [[ ${pgo_phase} -ne 1 ]] && [[ ${pgo_phase} -ne 2 ]]; then
- echo "[ERROR] The value of configuration item 'pgo_phase' is invalid, please check!"
+ echo "[ERROR] The value of configuration item 'pgo_phase' is invalid, please check!" | tee -a ${log_file}
exit 1
fi
if [[ ${pgo_phase} -eq 1 ]]; then
check_item ${pgo_mode} pgo_mode
if [[ ${pgo_mode} != "arc" ]] && [[ ${pgo_mode} != "all" ]]; then
- echo "[ERROR] The value of configuration item 'pgo_mode' is invalid, please check!"
+ echo "[ERROR] The value of configuration item 'pgo_mode' is invalid, please check!" | tee -a ${log_file}
exit 1
fi
fi
@@ -28,7 +25,8 @@ function check_config_items() {
check_item ${kernel_name} kernel_name
check_item ${work_path} work_path
if [[ ${work_path} =~ "/tmp" ]]; then
- echo "[ERROR] Do not put work path under /tmp, or it will be cleaned after rebooting."
+ echo "[ERROR] Do not put work path under /tmp, or it will be cleaned after rebooting." | tee -a ${log_file}
+ exit 1
fi
check_item ${run_script} run_script
check_item ${gcc_path} gcc_path
@@ -43,7 +41,7 @@ function check_config_items() {
if [[ ${pgo_phase} -eq 2 ]]; then
check_item ${data_dir} data_dir
if [[ ! -d ${data_dir} ]]; then
- echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist." | tee -a ${log_file}
exit 1
fi
fi
@@ -52,7 +50,7 @@ function check_config_items() {
# 初始化文件和目录
function init() {
- if [[ -z ${last_time} ]]; then
+ if [[ -z ${last_time} ]] || [[ ${pgo_phase} -eq 1 ]]; then
now_time=$(date '+%Y%m%d-%H%M%S')
else
now_time=${last_time}
@@ -61,29 +59,27 @@ function init() {
log_file=${time_dir}/log
if [[ -d ${time_dir} ]]; then
- echo "[INFO] Use last time directory: ${time_dir}"
+ echo "[INFO] Use last time directory: ${time_dir}" | tee -a ${log_file}
else
mkdir -p ${time_dir}
- echo "[INFO] Create time directory: ${time_dir}"
+ echo "[INFO] Create time directory: ${time_dir}" | tee -a ${log_file}
fi
if [[ -f ${log_file} ]]; then
- echo "[INFO] Use last log: ${log_file}"
+ echo "[INFO] Use last log: ${log_file}" | tee -a ${log_file}
else
touch ${log_file}
- echo "[INFO] Init log file: ${log_file}"
+ echo "[INFO] Init log file: ${log_file}" | tee -a ${log_file}
fi
}
# 检测环境
function check_dependency() {
- sed -i '/a-fot/d' /etc/rc.d/rc.local
-
get_arch=$(arch)
if [[ ${get_arch} =~ "x86_64" || ${get_arch} =~ "aarch64" ]]; then
- echo "[INFO] Current arch: ${get_arch}"
+ echo "[INFO] Current arch: ${get_arch}" | tee -a ${log_file}
else
- echo "[ERROR] Unsupport arch: ${get_arch}"
+ echo "[ERROR] Unsupport arch: ${get_arch}" | tee -a ${log_file}
exit 1
fi
is_file_exist ${run_script} "run_script"
@@ -95,12 +91,12 @@ function check_dependency() {
if [[ ${pgo_phase} -eq 1 ]] && [[ ${pgo_mode} == "all" ]] && [[ -z ${disable_compilation} ]]; then
if echo 'int main() { return 0; }' | gcc -x c - -o /dev/null -Werror -fkernel-pgo >/dev/null 2>&1; then
- echo "[INFO] Current GCC supports kernel PGO in runtime, use -fkernel-pgo option."
+ echo "[INFO] Current GCC supports kernel PGO in runtime, use -fkernel-pgo option." | tee -a ${log_file}
option="-fkernel-pgo"
elif gcc -v 2>&1 | grep -- "--disable-tls" >/dev/null; then
- echo "[INFO] Current GCC is recompiled with --disable-tls, does support kernel PGO in runtime."
+ echo "[INFO] Current GCC is recompiled with --disable-tls, does support kernel PGO in runtime." | tee -a ${log_file}
else
- echo "[ERROR] Current GCC does not support kernel PGO, you may use openeuler GCC or recompile GCC with --disable-tls --disable-libsanitizer."
+ echo "[ERROR] Current GCC does not support kernel PGO, you may use openeuler GCC or recompile GCC with --disable-tls --disable-libsanitizer." | tee -a ${log_file}
fi
fi
}
@@ -109,7 +105,7 @@ function check_dependency() {
function modify_kernel_config() {
if [[ ${pgo_mode} == "all" ]]; then
if ! grep "PGO" $1 >/dev/null; then
- echo "[ERROR] Current version of kernel does not support PGO, please use newer ones or choose arc mode."
+ echo "[ERROR] Current version of kernel does not support PGO, please use newer ones or choose arc mode." | tee -a ${log_file}
exit 1
fi
sed -i 's/.*CONFIG_PGO_KERNEL.*/CONFIG_PGO_KERNEL=y/' $1
@@ -128,27 +124,27 @@ function modify_kernel_config() {
function first_compilation() {
if [[ -n ${disable_compilation} ]]; then
modify_kernel_config ${kernel_config}
- echo "[INFO] Kernel configuration has been generated, the path is: ${kernel_config}"
+ echo "[INFO] Kernel configuration has been generated, the path is: ${kernel_config}" | tee -a ${log_file}
if [[ -z ${option} ]]; then
sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += fkernel-pgo" ${makefile}
fi
- echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}" | tee -a ${log_file}
exit 0
fi
if [[ -z ${kernel_src} ]]; then
- echo "[WARNING] ${kernel_src} is not specified, A-FOT will download and build kernel source code automatically in 10 seconds."
+ echo "[WARNING] ${kernel_src} is not specified, A-FOT will download and build kernel source code automatically in 10 seconds." | tee -a ${log_file}
sleep 10
oe_version=$(grep 'openeulerversion' /etc/openEuler-latest | cut -d '=' -f 2)
- echo "[INFO] Current openEuler version: ${oe_version}"
+ echo "[INFO] Current openEuler version: ${oe_version}" | tee -a ${log_file}
url="https://repo.huaweicloud.com/openeuler/${oe_version}/source/Packages/"
- echo "[INFO] Download the kernel source rpm package from ${url}"
+ echo "[INFO] Download the kernel source rpm package from ${url}" | tee -a ${log_file}
cd ${work_path}
wget --no-parent --no-directories -r -A 'kernel-[0-9]*.rpm' ${url} --no-check-certificate >>${log_file} 2>&1
is_success $?
srpm=$(ls -t kernel-[0-9]*.rpm 2>/dev/null | head -n 1)
- echo "[INFO] Successfully downloaded kernel source rpm package: ${srpm}"
- echo "[INFO] Build kernel source code."
+ echo "[INFO] Successfully downloaded kernel source rpm package: ${srpm}" | tee -a ${log_file}
+ echo "[INFO] Build kernel source code." | tee -a ${log_file}
rpm -ivh ${srpm} >>${log_file} 2>&1
cd - >/dev/null
rpmbuild -bp ~/rpmbuild/SPECS/kernel.spec >>${log_file} 2>&1
@@ -156,44 +152,48 @@ function first_compilation() {
src_dir=$(ls -td ~/rpmbuild/BUILD/kernel-*/*-source/ | head -n 1)
cp -r ${src_dir} ${work_path}/kernel
kernel_src=${work_path}/kernel
- echo "[INFO] Successfully builded kernel source code: ${kernel_src}"
+ echo "[INFO] Successfully builded kernel source code: ${kernel_src}" | tee -a ${log_file}
else
- echo "[INFO] Build kernel in an existing source directory."
+ echo "[INFO] Build kernel in an existing source directory." | tee -a ${log_file}
fi
if [[ ! -d ${kernel_src} ]]; then
- echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist." | tee -a ${log_file}
exit 1
fi
cd ${kernel_src}
make openeuler_defconfig
modify_kernel_config .config
- echo "[INFO] Start PGO kernel compilation."
- Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
- (make clean -j ${parallel} && make KCFLAGS="${option}" ARCH=${Arch} -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ echo "[INFO] Start PGO kernel compilation." | tee -a ${log_file}
+ arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="${option}" ARCH=${arch} -j ${parallel} && make modules_install ARCH=${arch} -j ${parallel} && make install ARCH=${arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
is_success ${PIPESTATUS[0]}
cd - >/dev/null
}
# 第一次重启操作
function first_reboot() {
- grub2-set-default 0
next_cmd="${afot_path}/a-fot --opt_mode Auto_kernel_PGO --pgo_phase 2 --kernel_src ${kernel_src} --kernel_name ${kernel_name} --work_path ${work_path} --run_script ${run_script} --gcc_path ${gcc_path} --last_time ${now_time} -s"
+ for config in ${kernel_configs[@]}; do
+ next_cmd+=" --${config}"
+ done
+
+ grub2-set-default 0
if [[ -z ${silent} ]]; then
read -p $'PGO kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
if [[ ${reboot_p} == "y" ]]; then
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
- echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}" | tee -a ${log_file}
sleep 10
reboot
else
- echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}"
+ echo -e "[INFO] Please run this command to continue after rebooting:\n${next_cmd}" | tee -a ${log_file}
fi
else
echo ${next_cmd} >>/etc/rc.d/rc.local
- chmod u+x /etc/rc.d/rc.local
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ chmod +x /etc/rc.d/rc.local
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
sleep 10
reboot
fi
@@ -202,7 +202,7 @@ function first_reboot() {
# 执行应用程序执行脚本
function execute_run_script() {
- echo "[INFO] Start to execute the run_script: ${run_script}"
+ echo "[INFO] Start to execute the run_script: ${run_script}" | tee -a ${log_file}
/bin/bash ${run_script} >>${log_file} 2>&1
is_success $?
}
@@ -213,18 +213,18 @@ function process_profiles() {
data_dir=/sys/kernel/debug/gcov
fi
if [[ ! -d ${data_dir} ]]; then
- echo "[ERROR] GCOV data directory ${data_dir} does not exist."
+ echo "[ERROR] GCOV data directory ${data_dir} does not exist." | tee -a ${log_file}
exit 1
fi
temp_dir=$(mktemp -d)
cd ${work_path}
- echo "[INFO] Start collecting the profiles."
+ echo "[INFO] Start collecting the profiles." | tee -a ${log_file}
find ${data_dir} -type d -exec mkdir -p ${temp_dir}/\{\} \;
find ${data_dir} -name '*.gcda' -exec sh -c 'cat < $0 > '${temp_dir}'/$0' {} \;
find ${data_dir} -name '*.gcno' -exec sh -c 'cp -d $0 '${temp_dir}'/$0' {} \;
- echo "[INFO] Start post-processing the profiles."
+ echo "[INFO] Start post-processing the profiles." | tee -a ${log_file}
find ${temp_dir} -name '*.gcda' >list.txt
/usr/bin/g++ ${afot_path}/GcovSummaryAddTool.cpp -o calcsum
./calcsum list.txt
@@ -240,14 +240,14 @@ function process_profiles() {
done
rm -rf ${temp_dir}
cd - >/dev/null
- echo "[INFO] Profiles have been successfully processed, the path is: ${profile_dir}"
+ echo "[INFO] Profiles have been successfully processed, the path is: ${profile_dir}" | tee -a ${log_file}
}
# 使用profile编译优化的内核
function second_compilation() {
if [[ -n ${disable_compilation} ]]; then
- sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += -fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" ${makefile}
- echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}"
+ sed -i "/KBUILD_CFLAGS += \$(KCFLAGS)/ a\KBUILD_CFLAGS += -fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -Wno-error=missing-profile -fprofile-dir=${profile_dir}" ${makefile}
+ echo "[INFO] Kernel makefile has been generated, the path is: ${makefile}" | tee -a ${log_file}
exit 0
fi
@@ -255,19 +255,37 @@ function second_compilation() {
kernel_src=${work_path}/kernel
fi
if [[ ! -d ${kernel_src} ]]; then
- echo "[ERROR] Kernel source directory ${kernel_src} does not exist."
+ echo "[ERROR] Kernel source directory ${kernel_src} does not exist." | tee -a ${log_file}
exit 1
fi
+ if [[ -f ${time_dir}/.flag ]]; then
+ rm -f ${time_dir}/.flag
+ else
+ next_cmd="${afot_path}/a-fot --opt_mode Auto_kernel_PGO --pgo_phase 2 --kernel_src ${kernel_src} --kernel_name ${kernel_name} --work_path ${work_path} --run_script ${run_script} --gcc_path ${gcc_path} --last_time ${now_time} -s"
+ for config in ${kernel_configs[@]}; do
+ next_cmd+=" --${config}"
+ done
+ touch ${time_dir}/.flag
+ echo ${next_cmd} >>/etc/rc.d/rc.local
+ chmod +x /etc/rc.d/rc.local
+ echo "[INFO] Switch to normal kernel for faster compilation." | tee -a ${log_file}
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
+ grub2-set-default 1
+ sleep 10
+ reboot
+ exit 0
+ fi
+
cd ${kernel_src}
make openeuler_defconfig
for config in ${kernel_configs[@]}; do
sed -i 's/.*${config%%=*}.*/${config}/' .config
done
sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-${kernel_name}-pgoed\"/" .config
- echo "[INFO] Start optimized kernel compilation."
- Arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
- (make clean -j ${parallel} && make KCFLAGS="-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -fprofile-dir=${profile_dir}" -j ${parallel} && make modules_install ARCH=${Arch} -j ${parallel} && make install ARCH=${Arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
+ echo "[INFO] Start optimized kernel compilation." | tee -a ${log_file}
+ arch=$(arch | sed -e s/x86_64/x86/ -e s/aarch64.*/arm64/)
+ (make clean -j ${parallel} && make KCFLAGS="-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch -Wno-error=missing-profile -fprofile-dir=${time_dir}/gcovdata" -j ${parallel} && make modules_install ARCH=${arch} -j ${parallel} && make install ARCH=${arch} -j ${parallel}) 2>&1 | tee -a ${log_file}
is_success ${PIPESTATUS[0]}
cd - >/dev/null
}
@@ -279,7 +297,7 @@ function second_reboot() {
read -p $'Optimized kernel has been successfully installed. Reboot now to use?\nPress [y] to reboot now, [n] to reboot yourself later: ' reboot_p
fi
if [[ -n ${silent} ]] || [[ ${reboot_p} == "y" ]]; then
- echo "[WARNING] System will be rebooted in 10 seconds!!!"
+ echo "[WARNING] System will be rebooted in 10 seconds!!!" | tee -a ${log_file}
sleep 10
reboot
exit 0
@@ -288,18 +306,19 @@ function second_reboot() {
# 执行入口
function main() {
+ cancel_afot_rc
check_config_items
+ check_dependency
+ init
if [[ ${pgo_phase} -eq 1 ]]; then
- check_dependency
- init
first_compilation
first_reboot
fi
if [[ ${pgo_phase} -eq 2 ]]; then
- check_dependency
- init
- execute_run_script
- process_profiles
+ if [[ ! -f ${time_dir}/.flag ]]; then
+ execute_run_script
+ process_profiles
+ fi
second_compilation
second_reboot
fi
diff --git a/auto_prefetch.sh b/auto_prefetch.sh
index b7f5932..3950b17 100644
--- a/auto_prefetch.sh
+++ b/auto_prefetch.sh
@@ -4,13 +4,13 @@
function check_dependency() {
check_common_dependency
if ! type create_gcov &>/dev/null; then
- echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'"
+ echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'."
exit 1
fi
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_env() {
+function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
@@ -28,7 +28,7 @@ function prepare_env() {
# 创建原始wrapper
function create_wrapper() {
- echo "[INFO] Start generating the original wrapper"
+ echo "[INFO] Start generating the original wrapper."
echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
post_create_wrapper
@@ -36,29 +36,29 @@ function create_wrapper() {
# 执行perf采样生成profile文件
function perf_record() {
- echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
+ echo "[INFO] Start perf record by ${opt_mode} and generate a profile file."
process_id=$(pidof ${application_name})
- get_arch=`arch`
- if [[ ${get_arch} =~ "x86_64" ]];then
+ get_arch=$(arch)
+ if [[ ${get_arch} =~ "x86_64" ]]; then
perf_event="inst_retired.prec_dist:u,cache-misses:u"
gcov_file_name="${profile_data_path}/${gcov_name}.inst_retired.prec_dist:u"
- elif [[ ${get_arch} =~ "aarch64" ]];then
+ elif [[ ${get_arch} =~ "aarch64" ]]; then
perf_event="inst_retired:u,cache-misses:u"
gcov_file_name="${profile_data_path}/${gcov_name}.inst_retired:u"
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
- perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
+ perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >>${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
- create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=0 >> ${log_file} 2>&1
+ create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=0 >>${log_file} 2>&1
is_file_exist "${gcov_file_name}"
is_file_exist "${profile_data_path}/${gcov_name}.cache-misses:u"
pkill ${application_name}
}
# 根据模式选择Wrapper或者Bear模式构建
-function prepare_new_env() {
+function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
@@ -75,7 +75,7 @@ function prepare_new_env() {
#生成新的wrapper
function create_new_wrapper() {
- echo "[INFO] Start to generate a new wrapper"
+ echo "[INFO] Start to generate a new wrapper."
echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++
}
--
2.33.0