!8 [sync] PR-4: Add Bear to A-FOT

From: @openeuler-sync-bot 
Reviewed-by: @li-yancheng 
Signed-off-by: @li-yancheng

0001-Add-Bear-to-A-FOT.patch

@@ -0,0 +1,426 @@
+diff --git a/a-fot b/a-fot
+index 1520a9e..c633098 100644
+--- a/a-fot
++++ b/a-fot
+@@ -74,6 +74,10 @@ function parse_input_params() {
+       check_success=$2
+       shift 2
+       ;;
++    --build_mode)
++      build_mode=$2
++      shift 2
++      ;;
+     *)
+       suggest_info
+       exit 1
+@@ -124,6 +128,10 @@ function check_config_item() {
+     echo "[ERROR] The configuration item 'check_success' is missing, please check!"
+     exit 1
+   fi
++  if [[ -z ${build_mode} ]]; then
++    echo "[ERROR] The configuration item 'build_mode' is missing, please check!"
++    exit 1
++  fi
+ }
+ 
+ function suggest_info() {
+@@ -141,6 +149,7 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...]
+ --run_script        Script path for run application
+ --max_waiting_time  Maximum binary startup time (unit: seconds)
+ --check_success     Check optimization result
++--build_mode        Execute build scrip mode (Wrapper/Bear)
+ """
+ }
+ 
+@@ -181,11 +190,27 @@ function check_common_dependency() {
+   is_file_exist "${gcc_path}/bin/gcc"
+ }
+ 
++# 拆分编译数据库
++function split_option() {
++  if [ "$bear_prefix" ];then
++        python3 $current_path/split_json.py -i $PWD/compile_commands.json
++        mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
++        mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
++  fi
++}
++
+ # 使用原始编译选项进行编译
+ function first_compilation() {
+   echo "[INFO] Start raw compilation"
+   is_file_exist ${build_script} "build_script"
+-  /bin/bash ${build_script} >> ${log_file} 2>&1
++  if [[ $build_mode =~ "Bear" ]]; then
++      bear_prefix="bear -- "
++      echo "[INFO] Build in Bear mode"
++  else
++      echo "[INFO] Build in Wrapper mode"
++  fi
++  $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
++  split_option first
+   is_file_exist ${bin_file}
+   is_success $?
+ }
+@@ -237,7 +262,7 @@ function detect_process() {
+ function second_compilation() {
+   echo "[INFO] Try compiling with the new compilation options"
+   if [[ ${check_success} -eq 1 ]]; then
+-    /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
++    $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
+     echo "[INFO] Found build id: ${build_id}"
+     add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}')
+     build_status=`ps -p ${build_id} | grep -c ${build_id}`
+@@ -254,9 +279,10 @@ function second_compilation() {
+     done
+     wait
+   else
+-    /bin/bash ${build_script} >> ${log_file} 2>&1
++    $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
+   fi
+   echo "[INFO] Finish compiling with new compilation options"
++  split_option second
+   is_success $?
+ }
+ 
+@@ -308,10 +334,12 @@ function is_opt_success() {
+     if [[ ${opt_success} -eq 0 ]]; then
+       echo "[WARNING] Optimization may fail or the build process is too short, please check!"
+       echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler"
++      exit 1
+     else
+       echo "[INFO] Optimization may success!"
+     fi
+   fi
++  exit 0
+ }
+ 
+ #执行入口，部分函数为加载不同优化脚本中得到
+@@ -324,15 +352,17 @@ function main() {
+   load_script
+ 
+   check_dependency
+-  create_wrapper
++  prepare_env
+   first_compilation
+   execute_run_script
+   detect_process
+   perf_record
+-  create_new_wrapper
+ 
++  prepare_new_env
+   second_compilation
+   is_opt_success
++  exit "$?"
+ }
+ 
+ main "$@"
++exit "$?"
+diff --git a/a-fot.ini b/a-fot.ini
+index 03a4702..b395504 100644
+--- a/a-fot.ini
++++ b/a-fot.ini
+@@ -18,4 +18,6 @@ perf_time=100
+ gcc_path=/usr/
+ # 检测是否优化成功(1=启用，0=禁用)
+ check_success=0
++# 构建模式 （Bear、Wrapper）
++build_mode=Bear
+ # 结束行勿删
+\ No newline at end of file
+diff --git a/auto_bolt.sh b/auto_bolt.sh
+index 625d1d5..ebcb2e5 100644
+--- a/auto_bolt.sh
++++ b/auto_bolt.sh
+@@ -9,11 +9,29 @@ function check_dependency() {
+   fi
+ }
+ 
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-Wl,-q"
++    export LINK_OPTIONS="-q"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
++
+ # 创建原始wrapper
+ function create_wrapper() {
+   echo "[INFO] Start generating the original wrapper"
+-  echo "${gcc_path}/bin/gcc -Wl,-q \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -Wl,-q \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
+   post_create_wrapper
+ }
+ 
+@@ -28,9 +46,28 @@ function perf_record() {
+   pkill ${application_name}
+ }
+ 
++
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_new_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_new_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q"
++    export LINK_OPTIONS="-q"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
++
+ #生成新的wrapper
+ function create_new_wrapper() {
+   echo "[INFO] Start to generate a new wrapper"
+-  echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++
+ }
+diff --git a/auto_fdo.sh b/auto_fdo.sh
+index 3d7bbb8..8426b30 100644
+--- a/auto_fdo.sh
++++ b/auto_fdo.sh
+@@ -9,11 +9,28 @@ function check_dependency() {
+   fi
+ }
+ 
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-g"
++    export LINK_OPTIONS="-g"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
+ # 创建原始wrapper
+ function create_wrapper() {
+   echo "[INFO] Start generating the original wrapper"
+-  echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
+   post_create_wrapper
+ }
+ 
+@@ -39,9 +56,25 @@ function perf_record() {
+   pkill ${application_name}
+ }
+ 
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_new_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_new_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
+ #生成新的wrapper
+ function create_new_wrapper() {
+   echo "[INFO] Start to generate a new wrapper"
+-  echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++
+ }
+diff --git a/auto_prefetch.sh b/auto_prefetch.sh
+index 2562dd6..265828a 100644
+--- a/auto_prefetch.sh
++++ b/auto_prefetch.sh
+@@ -9,11 +9,28 @@ function check_dependency() {
+   fi
+ }
+ 
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-g"
++    export LINK_OPTIONS="-g"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
+ # 创建原始wrapper
+ function create_wrapper() {
+   echo "[INFO] Start generating the original wrapper"
+-  echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++
+   post_create_wrapper
+ }
+ 
+@@ -40,9 +57,25 @@ function perf_record() {
+   pkill ${application_name}
+ }
+ 
++# 根据模式选择Wrapper或者Bear模式构建
++function  prepare_new_env() {
++  case ${build_mode} in
++  "Wrapper")
++    create_new_wrapper
++    ;;
++  "Bear")
++    export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2"
++    ;;
++  *)
++    echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
++    exit 1
++    ;;
++  esac
++}
++
+ #生成新的wrapper
+ function create_new_wrapper() {
+   echo "[INFO] Start to generate a new wrapper"
+-  echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/gcc
+-  echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/g++
++  echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc
++  echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++
+ }
+diff --git a/split_json.py b/split_json.py
+new file mode 100644
+index 0000000..bb6547f
+--- /dev/null
++++ b/split_json.py
+@@ -0,0 +1,98 @@
++#!/usr/bin/env python3
++# _*_ coding:utf-8 _*_
++
++
++"""
++    Copyright@2022-2022, All rights reserved, Powered by: Huawei Tech.Co.,Ltd.
++
++split_json is a interface to split compile commands according to its execution result.
++
++It will generate a new JSON file for the failed compile commands and remove these commands from source file.
++
++Input:
++    compile commands json
++Output:
++    compile commands json including commands which has been successfully executed.
++    compile commands json including commands which executed unsuccessfully.
++
++"""
++__author__ = 'z00500762'
++
++import argparse
++import os
++import json
++import sys
++import logging
++
++
++class SplitJson:
++    compile_commands_success = list()
++    compile_commands_fail = list()
++
++    def __init__(self, input_json):
++        self.input = input_json
++
++    @staticmethod
++    def validate(execution):
++        if "arguments" not in execution:
++            return False
++        if "directory" not in execution:
++            return False
++        if "exec_result" not in execution:
++            return False
++        return True
++
++    def get_compile_commands(self):
++        compile_commands = list()
++        try:
++            with open(self.input, "r", encoding='utf-8', errors='ignore') as json_file:
++                compile_commands = json.load(json_file)
++                if len(compile_commands) == 0:
++                    logging.info("compile commands json file is empty: %s", self.input)
++        except IOError as exception:
++            logging.error("open compile commands json file failed: %s", exception)
++        except json.decoder.JSONDecodeError as exception:
++            logging.error("json decode file failed: %s", exception)
++
++        return compile_commands
++
++    def split_commands(self):
++        compile_commands = self.get_compile_commands()
++        for item in compile_commands:
++            if not self.validate(item):
++                logging.info("discard invalid commands: %s", str(item))
++            if not item.get("rebuild"):
++                self.compile_commands_success.append(item)
++            else:
++                self.compile_commands_fail.append(item)
++        self.write_json()
++
++    def write_json(self):
++        compile_commands_success_file = os.path.splitext(self.input)[0] + ".json"
++        compile_commands_fail_file = os.path.splitext(self.input)[0] + ".fail.json"
++        with open(compile_commands_success_file, 'w+') as fw:
++            json.dump(self.compile_commands_success, fw, sort_keys=False, indent=4)
++
++        with open(compile_commands_fail_file, 'w+') as fw:
++            json.dump(self.compile_commands_fail, fw, sort_keys=False, indent=4)
++
++
++def main(input_json):
++    if not os.path.isabs(input_json):
++        input_json = os.path.join(os.getcwd(), input_json)
++    if not os.path.exists(input_json):
++        logging.error("compile_command_file not exists : %s", input_json)
++        return -1
++
++    sj = SplitJson(input_json)
++    sj.split_commands()
++
++
++if __name__ == "__main__":
++    cmd_parser = argparse.ArgumentParser(description="split compile commands json")
++    cmd_parser.add_argument(
++        '-i', '--input', dest='input_json', metavar='store', action='store',
++        help='json to split'
++    )
++    args = cmd_parser.parse_args()
++    sys.exit(main(args.input_json))

A-FOT.spec

@@ -1,13 +1,15 @@
 Name:           A-FOT
 Version:        v1.0
-Release:        0
+Release:        1
 Summary:        automatic feedback-directed optimization tool for openEuler
 License:        MulanPSL-2.0
 URL:            https://gitee.com/openeuler/A-FOT
 
 Source0:        %{name}-%{version}.tar.gz
 
-Requires:       gcc gcc-c++ autofdo llvm-bolt
+Requires:       gcc gcc-c++ autofdo llvm-bolt Bear python3
+
+Patch1: 0001-Add-Bear-to-A-FOT.patch
 
 %description
 A-FOT is an automatic feedback-directed optimization tool for openEuler
@@ -25,6 +27,7 @@ cp a-fot.ini %{buildroot}/%{_bindir}
 cp auto_fdo.sh %{buildroot}/%{_bindir}
 cp auto_prefetch.sh %{buildroot}/%{_bindir}
 cp auto_bolt.sh %{buildroot}/%{_bindir}
+cp split_json.py %{buildroot}/%{_bindir}
 
 %files
 %attr(755, root, root) %{_bindir}/a-fot
@@ -32,10 +35,17 @@ cp auto_bolt.sh %{buildroot}/%{_bindir}
 %attr(644, root, root) %{_bindir}/auto_fdo.sh
 %attr(644, root, root) %{_bindir}/auto_prefetch.sh
 %attr(644, root, root) %{_bindir}/auto_bolt.sh
+%attr(644, root, root) %{_bindir}/split_json.py
 %license LICENSE
 %doc README.md
 
 %changelog
+* Thu Dec 01 2022 huitailangzju <804544223@qq.com> - v1.0-1
+- Type:Sync
+- ID:NA
+- SUG:NA
+- DESC:Add Bear to A-FOT
+
 * Wed Mar 23 2022 liyancheng <412998149@qq.com> - 0
 - Type:Init
 - ID:NA