diff --git a/a-fot b/a-fot index 1520a9e..c633098 100644 --- a/a-fot +++ b/a-fot @@ -74,6 +74,10 @@ function parse_input_params() { check_success=$2 shift 2 ;; + --build_mode) + build_mode=$2 + shift 2 + ;; *) suggest_info exit 1 @@ -124,6 +128,10 @@ function check_config_item() { echo "[ERROR] The configuration item 'check_success' is missing, please check!" exit 1 fi + if [[ -z ${build_mode} ]]; then + echo "[ERROR] The configuration item 'build_mode' is missing, please check!" + exit 1 + fi } function suggest_info() { @@ -141,6 +149,7 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...] --run_script Script path for run application --max_waiting_time Maximum binary startup time (unit: seconds) --check_success Check optimization result +--build_mode Execute build scrip mode (Wrapper/Bear) """ } @@ -181,11 +190,27 @@ function check_common_dependency() { is_file_exist "${gcc_path}/bin/gcc" } +# 拆分编译数据库 +function split_option() { + if [ "$bear_prefix" ];then + python3 $current_path/split_json.py -i $PWD/compile_commands.json + mv $PWD/compile_commands.json $PWD/compile_commands_$1.json + mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json + fi +} + # 使用原始编译选项进行编译 function first_compilation() { echo "[INFO] Start raw compilation" is_file_exist ${build_script} "build_script" - /bin/bash ${build_script} >> ${log_file} 2>&1 + if [[ $build_mode =~ "Bear" ]]; then + bear_prefix="bear -- " + echo "[INFO] Build in Bear mode" + else + echo "[INFO] Build in Wrapper mode" + fi + $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 + split_option first is_file_exist ${bin_file} is_success $? } @@ -237,7 +262,7 @@ function detect_process() { function second_compilation() { echo "[INFO] Try compiling with the new compilation options" if [[ ${check_success} -eq 1 ]]; then - /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$! + $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$! echo "[INFO] Found build id: ${build_id}" add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}') build_status=`ps -p ${build_id} | grep -c ${build_id}` @@ -254,9 +279,10 @@ function second_compilation() { done wait else - /bin/bash ${build_script} >> ${log_file} 2>&1 + $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 fi echo "[INFO] Finish compiling with new compilation options" + split_option second is_success $? } @@ -308,10 +334,12 @@ function is_opt_success() { if [[ ${opt_success} -eq 0 ]]; then echo "[WARNING] Optimization may fail or the build process is too short, please check!" echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler" + exit 1 else echo "[INFO] Optimization may success!" fi fi + exit 0 } #执行入口，部分函数为加载不同优化脚本中得到 @@ -324,15 +352,17 @@ function main() { load_script check_dependency - create_wrapper + prepare_env first_compilation execute_run_script detect_process perf_record - create_new_wrapper + prepare_new_env second_compilation is_opt_success + exit "$?" } main "$@" +exit "$?" diff --git a/a-fot.ini b/a-fot.ini index 03a4702..b395504 100644 --- a/a-fot.ini +++ b/a-fot.ini @@ -18,4 +18,6 @@ perf_time=100 gcc_path=/usr/ # 检测是否优化成功(1=启用，0=禁用) check_success=0 +# 构建模式 （Bear、Wrapper） +build_mode=Bear # 结束行勿删 \ No newline at end of file diff --git a/auto_bolt.sh b/auto_bolt.sh index 625d1d5..ebcb2e5 100644 --- a/auto_bolt.sh +++ b/auto_bolt.sh @@ -9,11 +9,29 @@ function check_dependency() { fi } +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_env() { + case ${build_mode} in + "Wrapper") + create_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-Wl,-q" + export LINK_OPTIONS="-q" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + + # 创建原始wrapper function create_wrapper() { echo "[INFO] Start generating the original wrapper" - echo "${gcc_path}/bin/gcc -Wl,-q \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -Wl,-q \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++ post_create_wrapper } @@ -28,9 +46,28 @@ function perf_record() { pkill ${application_name} } + +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_new_env() { + case ${build_mode} in + "Wrapper") + create_new_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q" + export LINK_OPTIONS="-q" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + + #生成新的wrapper function create_new_wrapper() { echo "[INFO] Start to generate a new wrapper" - echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++ } diff --git a/auto_fdo.sh b/auto_fdo.sh index 3d7bbb8..8426b30 100644 --- a/auto_fdo.sh +++ b/auto_fdo.sh @@ -9,11 +9,28 @@ function check_dependency() { fi } +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_env() { + case ${build_mode} in + "Wrapper") + create_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-g" + export LINK_OPTIONS="-g" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + # 创建原始wrapper function create_wrapper() { echo "[INFO] Start generating the original wrapper" - echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++ post_create_wrapper } @@ -39,9 +56,25 @@ function perf_record() { pkill ${application_name} } +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_new_env() { + case ${build_mode} in + "Wrapper") + create_new_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + #生成新的wrapper function create_new_wrapper() { echo "[INFO] Start to generate a new wrapper" - echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++ } diff --git a/auto_prefetch.sh b/auto_prefetch.sh index 2562dd6..265828a 100644 --- a/auto_prefetch.sh +++ b/auto_prefetch.sh @@ -9,11 +9,28 @@ function check_dependency() { fi } +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_env() { + case ${build_mode} in + "Wrapper") + create_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-g" + export LINK_OPTIONS="-g" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + # 创建原始wrapper function create_wrapper() { echo "[INFO] Start generating the original wrapper" - echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++ post_create_wrapper } @@ -40,9 +57,25 @@ function perf_record() { pkill ${application_name} } +# 根据模式选择Wrapper或者Bear模式构建 +function prepare_new_env() { + case ${build_mode} in + "Wrapper") + create_new_wrapper + ;; + "Bear") + export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2" + ;; + *) + echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" + exit 1 + ;; + esac +} + #生成新的wrapper function create_new_wrapper() { echo "[INFO] Start to generate a new wrapper" - echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/gcc - echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/g++ + echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc + echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++ } diff --git a/split_json.py b/split_json.py new file mode 100644 index 0000000..bb6547f --- /dev/null +++ b/split_json.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# _*_ coding:utf-8 _*_ + + +""" + Copyright@2022-2022, All rights reserved, Powered by: Huawei Tech.Co.,Ltd. + +split_json is a interface to split compile commands according to its execution result. + +It will generate a new JSON file for the failed compile commands and remove these commands from source file. + +Input: + compile commands json +Output: + compile commands json including commands which has been successfully executed. + compile commands json including commands which executed unsuccessfully. + +""" +__author__ = 'z00500762' + +import argparse +import os +import json +import sys +import logging + + +class SplitJson: + compile_commands_success = list() + compile_commands_fail = list() + + def __init__(self, input_json): + self.input = input_json + + @staticmethod + def validate(execution): + if "arguments" not in execution: + return False + if "directory" not in execution: + return False + if "exec_result" not in execution: + return False + return True + + def get_compile_commands(self): + compile_commands = list() + try: + with open(self.input, "r", encoding='utf-8', errors='ignore') as json_file: + compile_commands = json.load(json_file) + if len(compile_commands) == 0: + logging.info("compile commands json file is empty: %s", self.input) + except IOError as exception: + logging.error("open compile commands json file failed: %s", exception) + except json.decoder.JSONDecodeError as exception: + logging.error("json decode file failed: %s", exception) + + return compile_commands + + def split_commands(self): + compile_commands = self.get_compile_commands() + for item in compile_commands: + if not self.validate(item): + logging.info("discard invalid commands: %s", str(item)) + if not item.get("rebuild"): + self.compile_commands_success.append(item) + else: + self.compile_commands_fail.append(item) + self.write_json() + + def write_json(self): + compile_commands_success_file = os.path.splitext(self.input)[0] + ".json" + compile_commands_fail_file = os.path.splitext(self.input)[0] + ".fail.json" + with open(compile_commands_success_file, 'w+') as fw: + json.dump(self.compile_commands_success, fw, sort_keys=False, indent=4) + + with open(compile_commands_fail_file, 'w+') as fw: + json.dump(self.compile_commands_fail, fw, sort_keys=False, indent=4) + + +def main(input_json): + if not os.path.isabs(input_json): + input_json = os.path.join(os.getcwd(), input_json) + if not os.path.exists(input_json): + logging.error("compile_command_file not exists : %s", input_json) + return -1 + + sj = SplitJson(input_json) + sj.split_commands() + + +if __name__ == "__main__": + cmd_parser = argparse.ArgumentParser(description="split compile commands json") + cmd_parser.add_argument( + '-i', '--input', dest='input_json', metavar='store', action='store', + help='json to split' + ) + args = cmd_parser.parse_args() + sys.exit(main(args.input_json))