From f2a5570a0821739a9e674e989ad1f734cc159570 Mon Sep 17 00:00:00 2001 From: h00502206 Date: Tue, 4 Jun 2024 21:52:50 +0800 Subject: [PATCH] Add block correction optimization --- bolt/CMakeLists.txt | 3 + .../bolt/include/bolt/Core/BinaryBasicBlock.h | 34 +- .../bolt/Core/BinaryBasicBlockFeature.h | 287 +++++ .../bolt/include/bolt/Passes/FeatureMiner.h | 36 +- .../include/bolt/Passes/StaticBranchInfo.h | 9 +- .../bolt/include/bolt/Profile/DataReader.h | 74 +- .../bolt/lib/Core/BinaryBasicBlockFeature.cpp | 25 + bolt/lib/Core/CMakeLists.txt | 1 + bolt/lib/Passes/CMakeLists.txt | 2 +- bolt/lib/Passes/FeatureMiner.cpp | 1137 +++++------------ .../bolt/lib/Passes/StaticBranchInfo.cpp | 6 +- bolt/lib/Profile/CMakeLists.txt | 1 + bolt/lib/Profile/DataReader.cpp | 126 +- .../bolt/lib/Rewrite/RewriteInstance.cpp | 7 + 14 files changed, 885 insertions(+), 863 deletions(-) create mode 100644 bolt/include/bolt/Core/BinaryBasicBlockFeature.h create mode 100644 bolt/lib/Core/BinaryBasicBlockFeature.cpp diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt index 3de930496..09dabcdee 100644 --- a/bolt/CMakeLists.txt +++ b/bolt/CMakeLists.txt @@ -9,6 +9,9 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|aarch64") set(BOLT_ENABLE_RUNTIME ON) endif() + +add_compile_options(-fexceptions) + set(BOLT_CLANG_EXE "" CACHE FILEPATH "Path to clang executable for the target \ architecture for use in BOLT tests") set(BOLT_LLD_EXE "" CACHE FILEPATH "Path to lld executable for the target \ diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h index 0a82f467e..bfeb174de 100644 --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -15,6 +15,7 @@ #ifndef BOLT_CORE_BINARY_BASIC_BLOCK_H #define BOLT_CORE_BINARY_BASIC_BLOCK_H +#include "bolt/Core/BinaryBasicBlockFeature.h" #include "bolt/Core/FunctionLayout.h" #include "bolt/Core/MCPlus.h" #include "llvm/ADT/GraphTraits.h" @@ -25,6 +26,7 @@ #include "llvm/Support/raw_ostream.h" #include #include +#include namespace llvm { class MCCodeEmitter; @@ -62,6 +64,13 @@ public: using BranchInfoType = SmallVector; + std::set ChildrenSet; + + std::set ParentSet; + + BinaryBasicBlockFeature block_features; + + private: /// Vector of all instructions in the block. InstructionListType Instructions; @@ -384,9 +393,11 @@ public: /// corresponding to a jump condition which could be true or false. /// Return nullptr if the basic block does not have a conditional jump. BinaryBasicBlock *getConditionalSuccessor(bool Condition) { - if (succ_size() != 2) - return nullptr; - return Successors[Condition == true ? 0 : 1]; + if (succ_size() == 2) + return Successors[Condition == true ? 0 : 1]; + if (succ_size() == 1) + return Successors[0]; + return nullptr; } const BinaryBasicBlock *getConditionalSuccessor(bool Condition) const { @@ -407,6 +418,11 @@ public: return const_cast(this)->getFallthrough(); } + /// Return branch info corresponding to only branch. + const BinaryBranchInfo &getOnlyBranchInfo() const { + return BranchInfo[0]; + }; + /// Return branch info corresponding to a taken branch. const BinaryBranchInfo &getTakenBranchInfo() const { assert(BranchInfo.size() == 2 && @@ -807,6 +823,16 @@ public: OutputAddressRange.second = Address; } + /// Sets features of this BB. + void setFeatures(BinaryBasicBlockFeature BBF) { + block_features = BBF; + } + + /// Gets numberic features of this BB. + BinaryBasicBlockFeature getFeatures() { + return block_features; + } + /// Gets the memory address range of this BB in the input binary. std::pair getInputAddressRange() const { return InputRange; @@ -974,7 +1000,7 @@ private: #if defined(LLVM_ON_UNIX) /// Keep the size of the BinaryBasicBlock within a reasonable size class /// (jemalloc bucket) on Linux -static_assert(sizeof(BinaryBasicBlock) <= 256, ""); +static_assert(sizeof(BinaryBasicBlock) <= 2048, ""); #endif bool operator<(const BinaryBasicBlock &LHS, const BinaryBasicBlock &RHS); diff --git a/bolt/include/bolt/Core/BinaryBasicBlockFeature.h b/bolt/include/bolt/Core/BinaryBasicBlockFeature.h new file mode 100644 index 000000000..0f123bd49 --- /dev/null +++ b/bolt/include/bolt/Core/BinaryBasicBlockFeature.h @@ -0,0 +1,287 @@ +//===- bolt/Core/BinaryBasicBlockFeature.h - Low-level basic block -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Features of BinaryBasicBlock +// +//===----------------------------------------------------------------------===// + +#ifndef BOLT_CORE_BINARY_BASIC_BLOCK_FEATURE_H +#define BOLT_CORE_BINARY_BASIC_BLOCK_FEATURE_H + +#include "bolt/Core/FunctionLayout.h" +#include "bolt/Core/MCPlus.h" +#include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +namespace llvm { + +namespace bolt { + +class BinaryBasicBlockFeature { + +public: + + int32_t Opcode; + + int16_t Direction; + + int32_t CmpOpcode; + + int16_t LoopHeader; + + int16_t ProcedureType; + + int64_t Count; + + int64_t FallthroughCount; + + int64_t TotalLoops; + + int64_t LoopDepth; + + int64_t LoopNumBlocks; + + int64_t LocalExitingBlock; + + int64_t LocalLatchBlock; + + int64_t LocalLoopHeader; + + int64_t Call; + + int64_t DeltaTaken; + + int64_t NumLoads; + + int64_t NumCalls; + + int64_t OperandRAType; + + int64_t OperandRBType; + + int64_t BasicBlockSize; + + int64_t NumBasicBlocks; + + int64_t HasIndirectCalls; + + std::vector EndOpcode_vec; + + std::vector LoopHeader_vec; + + std::vector Backedge_vec; + + std::vector Exit_vec; + + std::vector Call_vec; + + std::vector BasicBlockSize_vec; + + std::vector InferenceFeatures; + + uint64_t FuncExec; + + int32_t ParentChildNum; + + int32_t ParentCount; + + int32_t ChildParentNum; + + int32_t ChildCount; + +public: + + void setOpcode(const int32_t &BlockOpcode) { + Opcode = BlockOpcode; + } + + void setDirection(const int16_t &BlockDirection) { + Direction = BlockDirection; + } + + void setCmpOpcode(const int32_t &BlockCmpOpcode) { + CmpOpcode = BlockCmpOpcode; + } + + void setLoopHeader(const int16_t &BlockLoopHeader) { + LoopHeader = BlockLoopHeader; + } + + void setProcedureType(const int16_t &BlockProcedureType) { + ProcedureType = BlockProcedureType; + } + + void setCount(const int64_t &BlockCount) { + Count = BlockCount; + } + + void setFallthroughCount(const int64_t &BlockFallthroughCount) { + FallthroughCount = BlockFallthroughCount; + } + + void setTotalLoops(const int64_t &BlockTotalLoops) { + TotalLoops = BlockTotalLoops; + } + + void setLoopDepth(const int64_t &BlockLoopDepth) { + LoopDepth = BlockLoopDepth; + } + + void setLoopNumBlocks(const int64_t &BlockLoopNumBlocks) { + LoopNumBlocks = BlockLoopNumBlocks; + } + + void setLocalExitingBlock(const int64_t &BlockLocalExitingBlock) { + LocalExitingBlock = BlockLocalExitingBlock; + } + + void setLocalLatchBlock(const int64_t &BlockLocalLatchBlock) { + LocalLatchBlock = BlockLocalLatchBlock; + } + + void setLocalLoopHeader(const int64_t &BlockLocalLoopHeader) { + LocalLoopHeader = BlockLocalLoopHeader; + } + + void setDeltaTaken(const int64_t &BlockDeltaTaken) { + DeltaTaken = BlockDeltaTaken; + } + + void setNumLoads(const int64_t &BlockNumLoads) { + NumLoads = BlockNumLoads; + } + + void setNumCalls(const int64_t &BlockNumCalls) { + NumCalls = BlockNumCalls; + } + + void setOperandRAType(const int64_t &BlockOperandRAType) { + OperandRAType = BlockOperandRAType; + } + + void setOperandRBType(const int64_t &BlockOperandRBType) { + OperandRBType = BlockOperandRBType; + } + + void setBasicBlockSize(const int64_t &BlockBasicBlockSize) { + BasicBlockSize = BlockBasicBlockSize; + } + + void setNumBasicBlocks(const int64_t &BlockNumBasicBlocks) { + NumBasicBlocks = BlockNumBasicBlocks; + } + + void setHasIndirectCalls(const int64_t &BlockHasIndirectCalls) { + HasIndirectCalls = BlockHasIndirectCalls; + } + + void setEndOpcodeVec(const int32_t &EndOpcode) { + EndOpcode_vec.push_back(EndOpcode); + } + + void setLoopHeaderVec(const int16_t &LoopHeader) { + LoopHeader_vec.push_back(LoopHeader); + } + + void setBackedgeVec(const int16_t &Backedge) { + Backedge_vec.push_back(Backedge); + } + + void setExitVec(const int16_t &Exit) { + Exit_vec.push_back(Exit); + } + + void setCallVec(const int16_t &Call) { + Call_vec.push_back(Call); + } + + void setBasicBlockSizeVec(const int64_t &BasicBlockSize) { + BasicBlockSize_vec.push_back(BasicBlockSize); + } + + void setFunExec(const uint64_t &BlockFuncExec) { + FuncExec = BlockFuncExec; + } + + void setParentChildNum(const int32_t &BlockParentChildNum) { + ParentChildNum = BlockParentChildNum; + } + + void setParentCount(const int32_t &BlockParentCount) { + ParentCount = BlockParentCount; + } + + void setChildParentNum(const int32_t &BlockChildParentNum) { + ChildParentNum = BlockChildParentNum; + } + + void setChildCount(const int32_t &BlockChildCount) { + ChildCount = BlockChildCount; + } + + + + void setInferenceFeatures() { + + if (Count == -1 || FallthroughCount == -1) { + return; + } + if (ParentChildNum == -1 && ParentCount == -1 && ChildParentNum == -1 && ChildCount == -1){ + return; + } + + InferenceFeatures.push_back(static_cast(Direction)); + InferenceFeatures.push_back(static_cast(LoopHeader)); + InferenceFeatures.push_back(static_cast(ProcedureType)); + InferenceFeatures.push_back(static_cast(OperandRAType)); + InferenceFeatures.push_back(static_cast(OperandRBType)); + InferenceFeatures.push_back(static_cast(LoopHeader_vec[0])); + InferenceFeatures.push_back(static_cast(Backedge_vec[0])); + InferenceFeatures.push_back(static_cast(Exit_vec[0])); + InferenceFeatures.push_back(static_cast(LoopHeader_vec[1])); + InferenceFeatures.push_back(static_cast(Call_vec[0])); + InferenceFeatures.push_back(static_cast(LocalExitingBlock)); + InferenceFeatures.push_back(static_cast(HasIndirectCalls)); + InferenceFeatures.push_back(static_cast(LocalLatchBlock)); + InferenceFeatures.push_back(static_cast(LocalLoopHeader)); + InferenceFeatures.push_back(static_cast(Opcode)); + InferenceFeatures.push_back(static_cast(CmpOpcode)); + InferenceFeatures.push_back(static_cast(EndOpcode_vec[0])); + InferenceFeatures.push_back(static_cast(EndOpcode_vec[1])); + InferenceFeatures.push_back(static_cast(FuncExec)); + InferenceFeatures.push_back(static_cast(NumBasicBlocks)); + InferenceFeatures.push_back(static_cast(BasicBlockSize)); + InferenceFeatures.push_back(static_cast(BasicBlockSize_vec[0])); + InferenceFeatures.push_back(static_cast(BasicBlockSize_vec[1])); + InferenceFeatures.push_back(static_cast(LoopNumBlocks)); + InferenceFeatures.push_back(static_cast(NumLoads)); + InferenceFeatures.push_back(static_cast(NumCalls)); + InferenceFeatures.push_back(static_cast(TotalLoops)); + InferenceFeatures.push_back(static_cast(DeltaTaken)); + InferenceFeatures.push_back(static_cast(LoopDepth)); + InferenceFeatures.push_back(static_cast(ParentChildNum)); + InferenceFeatures.push_back(static_cast(ParentCount)); + InferenceFeatures.push_back(static_cast(ChildParentNum)); + InferenceFeatures.push_back(static_cast(ChildCount)); + } + + std::vector getInferenceFeatures() { + return InferenceFeatures; + } + +}; +} +} + +#endif \ No newline at end of file diff --git a/bolt/include/bolt/Passes/FeatureMiner.h b/bolt/include/bolt/Passes/FeatureMiner.h index 916e5515d..b03666ebf 100644 --- a/bolt/include/bolt/Passes/FeatureMiner.h +++ b/bolt/include/bolt/Passes/FeatureMiner.h @@ -14,19 +14,12 @@ #ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ #define LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ -// #include "BinaryContext.h" -// #include "BinaryFunction.h" -// #include "BinaryLoop.h" -// #include "DominatorAnalysis.h" -// #include "Passes/BinaryPasses.h" -// #include "Passes/StaticBranchInfo.h" #include "bolt/Core/BinaryData.h" #include "bolt/Core/BinaryFunction.h" #include "bolt/Core/BinaryLoop.h" #include "bolt/Passes/DominatorAnalysis.h" #include "bolt/Passes/BinaryPasses.h" #include "bolt/Passes/StaticBranchInfo.h" - #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" @@ -36,6 +29,7 @@ #include #include #include +#include namespace llvm { namespace bolt { @@ -43,7 +37,6 @@ namespace bolt { class FeatureMiner : public BinaryFunctionPass { private: std::unique_ptr SBI; - /// BasicBlockInfo - This structure holds feature information about the target /// BasicBlock of either the taken or the fallthrough paths of a given branch. struct BasicBlockInfo { @@ -57,7 +50,7 @@ private: Optional NumLoads; Optional NumStores; Optional EndOpcode; // 0 = NOTHING - StringRef EndOpcodeStr = "UNDEF"; + std::string EndOpcodeStr = "UNDEF"; Optional BasicBlockSize; std::string FromFunName = "UNDEF"; uint32_t FromBb; @@ -76,8 +69,8 @@ private: /// BranchFeaturesInfo - This structure holds feature information about each /// two-way branch from the program. struct BranchFeaturesInfo { - StringRef OpcodeStr = "UNDEF"; - StringRef CmpOpcodeStr = "UNDEF"; + std::string OpcodeStr = "UNDEF"; + std::string CmpOpcodeStr = "UNDEF"; bool Simple = 0; Optional Opcode; @@ -130,28 +123,28 @@ private: }; typedef std::unique_ptr BFIPtr; + std::vector BranchesInfoSet; + /// getProcedureType - Determines which category the function falls into: /// Leaf, Non-leaf or Calls-self. int8_t getProcedureType(BinaryFunction &Function, BinaryContext &BC); /// addSuccessorInfo - Discovers feature information for the target successor /// basic block, and inserts it into the static branch info container. - void addSuccessorInfo(DominatorAnalysis &DA, - DominatorAnalysis &PDA, BFIPtr const &BFI, - BinaryFunction &Function, BinaryContext &BC, - MCInst &Inst, BinaryBasicBlock &BB, bool Succ); + void addSuccessorInfo(BFIPtr const &BFI, BinaryFunction &Function, + BinaryContext &BC, BinaryBasicBlock &BB, bool SuccType); /// extractFeatures - Extracts the feature information for each two-way branch /// from the program. void extractFeatures(BinaryFunction &Function, - BinaryContext &BC, - raw_ostream &Printer); + BinaryContext &BC); + void generateInstFeatures(BinaryContext &BC, BinaryBasicBlock &BB, BFIPtr const &BFI, int Index); /// dumpSuccessorFeatures - Dumps the feature information about the target /// BasicBlock of either the taken or the fallthrough paths of a given branch. - void dumpSuccessorFeatures(raw_ostream &Printer, BBIPtr &Successor); + void generateSuccessorFeatures(BBIPtr &Successor, BinaryBasicBlockFeature *BBF); /// dumpFeatures - Dumps the feature information about each two-way branch /// from the program. @@ -167,12 +160,17 @@ public: explicit FeatureMiner(const cl::opt &PrintPass) : BinaryFunctionPass(PrintPass) {} + std::ofstream trainPrinter; + + const char *getName() const override { return "feature-miner"; } void runOnFunctions(BinaryContext &BC) override; + void inferenceFeatures(BinaryFunction &Function); + void generateProfileFeatures(BinaryBasicBlock *BB, BinaryBasicBlockFeature *BBF); }; } // namespace bolt } // namespace llvm -#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ */ \ No newline at end of file +#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ */ diff --git a/bolt/include/bolt/Passes/StaticBranchInfo.h b/bolt/include/bolt/Passes/StaticBranchInfo.h index 1713d3367..54a1f7cff 100644 --- a/bolt/include/bolt/Passes/StaticBranchInfo.h +++ b/bolt/include/bolt/Passes/StaticBranchInfo.h @@ -18,16 +18,11 @@ #ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ #define LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ -// #include "BinaryContext.h" -// #include "BinaryFunction.h" -// #include "BinaryLoop.h" #include "bolt/Core/BinaryContext.h" #include "bolt/Core/BinaryFunction.h" #include "bolt/Core/BinaryLoop.h" - #include "llvm/MC/MCSymbol.h" -// add new include -#include +#include namespace llvm { namespace bolt { @@ -113,4 +108,4 @@ public: } // namespace bolt } // namespace llvm -#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ */ \ No newline at end of file +#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ */ diff --git a/bolt/include/bolt/Profile/DataReader.h b/bolt/include/bolt/Profile/DataReader.h index 60cedfeb3..7e2c318fc 100644 --- a/bolt/include/bolt/Profile/DataReader.h +++ b/bolt/include/bolt/Profile/DataReader.h @@ -21,6 +21,7 @@ #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" +#include #include #include @@ -43,6 +44,13 @@ inline raw_ostream &operator<<(raw_ostream &OS, const LBREntry &LBR) { return OS; } +extern "C" { +typedef void *(*CreateONNXRunnerFunc)(const char *); +typedef void (*DeleteONNXRunnerFunc)(void *); +typedef float (*RunONNXModelFunc)(void *, std::vector &, + std::vector &, std::vector &); +} + struct Location { bool IsSymbol; StringRef Name; @@ -262,7 +270,8 @@ struct FuncSampleData { class DataReader : public ProfileReaderBase { public: explicit DataReader(StringRef Filename) - : ProfileReaderBase(Filename), Diag(errs()) {} + : ProfileReaderBase(Filename), Diag(errs()), onnxRunner(nullptr), + lib_handle(nullptr), handleOnnxRuntime(nullptr) {} StringRef getReaderName() const override { return "branch profile reader"; } @@ -281,7 +290,70 @@ public: /// Return all event names used to collect this profile virtual StringSet<> getEventNames() const override { return EventNames; } + ~DataReader() { + // delete onnxrunner; + if (onnxRunner && lib_handle && handleOnnxRuntime) { + DeleteONNXRunnerFunc deleteONNXRunner = + (DeleteONNXRunnerFunc)dlsym(lib_handle, "deleteONNXRunner"); + deleteONNXRunner(onnxRunner); + dlclose(lib_handle); + dlclose(handleOnnxRuntime); + } + } + + /// Initialize the onnxruntime model. + void initializeONNXRunner(const std::string &modelPath) { + if (!onnxRunner && !lib_handle && !handleOnnxRuntime) { + handleOnnxRuntime = + dlopen("libonnxruntime.so.1.16.3", RTLD_LAZY | RTLD_GLOBAL); + lib_handle = + dlopen("libONNXRunner.so", RTLD_LAZY); + CreateONNXRunnerFunc createONNXRunner = + (CreateONNXRunnerFunc)dlsym(lib_handle, "createONNXRunner"); + onnxRunner = createONNXRunner(modelPath.c_str()); + } + } + + /// Inference step for predicting the BB counts based on the BB features. + float ONNXInference(std::vector &input_string, + std::vector &input_int64, + std::vector &input_float) { + if (onnxRunner && lib_handle) { + RunONNXModelFunc runONNXModel = + (RunONNXModelFunc)dlsym(lib_handle, "runONNXModel"); + float model_pred = + runONNXModel(onnxRunner, input_string, input_int64, input_float); + return model_pred; + } + return -1.0; + } + protected: + /// The onnxruntime model pointer read from the input model path. + void *onnxRunner; + + /// The library handle of the ai4compiler framwork. + void *lib_handle; + + /// The library handle of the onnxruntime. + void *handleOnnxRuntime; + + /// The annotating threshold for the model prediction. + float threshold = 0.95; + + /// Return the annotating threshold for the model prediction. + float getThreshold() const { return threshold; } + + /// The counting value of the total modified BB-count number. + uint64_t modified_BB_total = 0; + + /// Add the total modified BB-count number by the BB modifiied number within + /// the funciton. + void addModifiedBBTotal(uint64_t &value) { modified_BB_total += value; } + + /// Return the counting value of the total modified BB-count number. + uint64_t getModifiedBBTotal() const { return modified_BB_total; } + /// Read profile information available for the function. void readProfile(BinaryFunction &BF); diff --git a/bolt/lib/Core/BinaryBasicBlockFeature.cpp b/bolt/lib/Core/BinaryBasicBlockFeature.cpp new file mode 100644 index 000000000..81d8c7546 --- /dev/null +++ b/bolt/lib/Core/BinaryBasicBlockFeature.cpp @@ -0,0 +1,25 @@ +//===- bolt/Core/BinaryBasicBlockFeature.cpp - Low-level basic block -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the BinaryBasicBlock class. +// +//===----------------------------------------------------------------------===// + +#include "bolt/Core/BinaryBasicBlock.h" +#include "bolt/Core/BinaryBasicBlockFeature.h" + + +#define DEBUG_TYPE "bolt" + +namespace llvm { +namespace bolt { + + + +} // namespace bolt +} // namespace llvm \ No newline at end of file diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt index 501b5eb6e..25e9fb020 100644 --- a/bolt/lib/Core/CMakeLists.txt +++ b/bolt/lib/Core/CMakeLists.txt @@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_library(LLVMBOLTCore BinaryBasicBlock.cpp + BinaryBasicBlockFeature.cpp BinaryContext.cpp BinaryData.cpp BinaryEmitter.cpp diff --git a/bolt/lib/Passes/CMakeLists.txt b/bolt/lib/Passes/CMakeLists.txt index 901ff614c..9f5abd101 100644 --- a/bolt/lib/Passes/CMakeLists.txt +++ b/bolt/lib/Passes/CMakeLists.txt @@ -12,9 +12,9 @@ add_llvm_library(LLVMBOLTPasses DataflowAnalysis.cpp DataflowInfoManager.cpp ExtTSPReorderAlgorithm.cpp - FeatureMiner.cpp FrameAnalysis.cpp FrameOptimizer.cpp + FeatureMiner.cpp HFSort.cpp HFSortPlus.cpp IdenticalCodeFolding.cpp diff --git a/bolt/lib/Passes/FeatureMiner.cpp b/bolt/lib/Passes/FeatureMiner.cpp index 680222906..abee69745 100644 --- a/bolt/lib/Passes/FeatureMiner.cpp +++ b/bolt/lib/Passes/FeatureMiner.cpp @@ -11,15 +11,10 @@ // https://dl.acm.org/doi/10.1145/239912.239923 //===----------------------------------------------------------------------===// -// #include "Passes/FeatureMiner.h" -// #include "Passes/DataflowInfoManager.h" -// #include "llvm/Support/CommandLine.h" -// #include "llvm/Support/Options.h" #include "bolt/Passes/FeatureMiner.h" #include "bolt/Passes/DataflowInfoManager.h" +#include "bolt/Passes/StaticBranchInfo.h" #include "llvm/Support/CommandLine.h" - -// add new include #include "llvm/Support/FileSystem.h" #undef DEBUG_TYPE @@ -29,24 +24,7 @@ using namespace llvm; using namespace bolt; namespace opts { - -extern cl::OptionCategory InferenceCategory; - -cl::opt VespaUseDFS( - "vespa-dfs", - cl::desc("use DFS ordering when using -gen-features option"), - cl::init(false), - cl::ReallyHidden, - cl::ZeroOrMore, - cl::cat(InferenceCategory)); - -cl::opt IncludeValidProfile( - "beetle-valid-profile-info", - cl::desc("include valid profile information."), - cl::init(false), - cl::ReallyHidden, - cl::ZeroOrMore, - cl::cat(InferenceCategory)); +extern cl::opt BlockCorrection; } // namespace opts @@ -75,50 +53,19 @@ int8_t FeatureMiner::getProcedureType(BinaryFunction &Function, return ProcedureType; // leaf type } -void FeatureMiner::addSuccessorInfo(DominatorAnalysis &DA, - DominatorAnalysis &PDA, - BFIPtr const &BFI, BinaryFunction &Function, - BinaryContext &BC, MCInst &Inst, - BinaryBasicBlock &BB, bool SuccType) { +void FeatureMiner::addSuccessorInfo(BFIPtr const &BFI, BinaryFunction &Function, + BinaryContext &BC, BinaryBasicBlock &BB, bool SuccType) { BinaryBasicBlock *Successor = BB.getConditionalSuccessor(SuccType); if (!Successor) return; - unsigned NumLoads{0}; - unsigned NumStores{0}; - unsigned NumCallsExit{0}; unsigned NumCalls{0}; - unsigned NumCallsInvoke{0}; - unsigned NumTailCalls{0}; - unsigned NumIndirectCalls{0}; for (auto &Inst : BB) { - if (BC.MIB->isLoad(Inst)) { - ++NumLoads; - } else if (BC.MIB->isStore(Inst)) { - ++NumStores; - } else if (BC.MIB->isCall(Inst)) { + if (BC.MIB->isCall(Inst)) { ++NumCalls; - - if (BC.MIB->isIndirectCall(Inst)) - ++NumIndirectCalls; - - if (BC.MIB->isInvoke(Inst)) - ++NumCallsInvoke; - - if (BC.MIB->isTailCall(Inst)) - ++NumTailCalls; - - if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { - StringRef CalleeName = CalleeSymbol->getName(); - if (CalleeName == "__cxa_throw@PLT" || - CalleeName == "_Unwind_Resume@PLT" || - CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || - CalleeName == "abort@PLT") - ++NumCallsExit; - } } } @@ -138,117 +85,78 @@ void FeatureMiner::addSuccessorInfo(DominatorAnalysis &DA, SuccBBInfo->Backedge = SBI->isBackEdge(&BB, Successor); MCInst *SuccInst = Successor->getTerminatorBefore(nullptr); + // Store information about the branch type ending sucessor basic block SuccBBInfo->EndOpcode = (SuccInst && BC.MIA->isBranch(*SuccInst)) ? SuccInst->getOpcode() : 0; // 0 = NOTHING - if (SuccBBInfo->EndOpcode != 0) - SuccBBInfo->EndOpcodeStr = BC.MII->getName(SuccInst->getOpcode()); - else - SuccBBInfo->EndOpcodeStr = "NOTHING"; // Check if the successor basic block contains // a procedure call and store it. SuccBBInfo->Call = (NumCalls > 0) ? 1 // Contains a call instruction : 0; // Does not contain a call instruction - SuccBBInfo->NumStores = NumStores; - SuccBBInfo->NumLoads = NumLoads; - SuccBBInfo->NumCallsExit = NumCallsExit; - SuccBBInfo->NumCalls = NumCalls; - - SuccBBInfo->NumCallsInvoke = NumCallsInvoke; - SuccBBInfo->NumIndirectCalls = NumIndirectCalls; - SuccBBInfo->NumTailCalls = NumTailCalls; - - auto InstSucc = Successor->getLastNonPseudoInstr(); - if (InstSucc) { - // Check if the source basic block dominates its - // target basic block and store it. - SuccBBInfo->BranchDominates = (DA.doesADominateB(Inst, *InstSucc) == true) - ? 1 // Dominates - : 0; // Does not dominate - - // Check if the target basic block postdominates - // the source basic block and store it. - SuccBBInfo->BranchPostdominates = - (PDA.doesADominateB(*InstSucc, Inst) == true) - ? 1 // Postdominates - : 0; // Does not postdominate - } - /// The follwoing information is used as an identifier only for - /// the purpose of matching the inferred probabilities with the branches - /// in the binary. - SuccBBInfo->FromFunName = Function.getPrintName(); - SuccBBInfo->FromBb = BB.getInputOffset(); - BinaryFunction *ToFun = Successor->getFunction(); - SuccBBInfo->ToFunName = ToFun->getPrintName(); - SuccBBInfo->ToBb = Successor->getInputOffset(); - - auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); - if (Offset) { - uint32_t TargetOffset = Successor->getInputOffset(); - uint32_t BranchOffset = Offset.get(); - BFI->BranchOffset = BranchOffset; - if (BranchOffset != UINT32_MAX && TargetOffset != UINT32_MAX) { - int64_t Delta = TargetOffset - BranchOffset; - BFI->DeltaTaken = std::abs(Delta); - } - } + uint32_t Offset = BB.getEndOffset(); if (SuccType) { BFI->TrueSuccessor = std::move(SuccBBInfo); - // Check if the taken branch is a forward - // or a backwards branch and store it. + // or a backwards branch and store it BFI->Direction = (Function.isForwardBranch(&BB, Successor) == true) - ? 1 // Forward branch - : 0; // Backwards branch - - auto TakenBranchInfo = BB.getTakenBranchInfo(); - BFI->Count = TakenBranchInfo.Count; - BFI->MissPredicted = TakenBranchInfo.MispredictedCount; + ? 1 // Forward branch + : 0; // Backwards branch + + auto OnlyBranchInfo = BB.getOnlyBranchInfo(); + BFI->Count = OnlyBranchInfo.Count; + + if (Offset) { + uint32_t TargetOffset = Successor->getInputOffset(); + uint32_t BranchOffset = Offset; + if (BranchOffset != UINT32_MAX && TargetOffset != UINT32_MAX) { + int64_t Delta = static_cast(TargetOffset) - static_cast(BranchOffset); + BFI->DeltaTaken = std::abs(Delta); + } + } } else { + if (BB.succ_size() == 2) { + auto FallthroughBranchInfo = BB.getFallthroughBranchInfo(); + BFI->FallthroughCount = FallthroughBranchInfo.Count; + } else { + auto OnlyBranchInfo = BB.getOnlyBranchInfo(); + BFI->FallthroughCount = OnlyBranchInfo.Count; + } BFI->FalseSuccessor = std::move(SuccBBInfo); - - auto FallthroughBranchInfo = BB.getFallthroughBranchInfo(); - BFI->FallthroughCount = FallthroughBranchInfo.Count; - BFI->FallthroughMissPredicted = FallthroughBranchInfo.MispredictedCount; } } -void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC, - raw_ostream &Printer) { +void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC) { int8_t ProcedureType = getProcedureType(Function, BC); -// auto Info = DataflowInfoManager(BC, Function, nullptr, nullptr); auto Info = DataflowInfoManager(Function, nullptr, nullptr); - auto &DA = Info.getDominatorAnalysis(); - auto &PDA = Info.getPostDominatorAnalysis(); const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); - bool Simple = Function.isSimple(); -// const auto &Order = opts::VespaUseDFS ? Function.dfs() : Function.getLayout(); + bool Simple = Function.isSimple(); const auto &Order = Function.dfs(); + std::string Function_name = Function.getPrintName(); + for (auto *BBA : Order) { - + auto &BB = *BBA; - unsigned NumOuterLoops{0}; + + BinaryBasicBlockFeature BBF = BB.getFeatures(); + unsigned TotalLoops{0}; - unsigned MaximumLoopDepth{0}; unsigned LoopDepth{0}; - unsigned LoopNumExitEdges{0}; - unsigned LoopNumExitBlocks{0}; - unsigned LoopNumExitingBlocks{0}; - unsigned LoopNumLatches{0}; unsigned LoopNumBlocks{0}; - unsigned LoopNumBackEdges{0}; bool LocalExitingBlock{false}; bool LocalLatchBlock{false}; bool LocalLoopHeader{false}; + + generateProfileFeatures(&BB, &BBF); + BinaryLoop *Loop = LoopsInfo.getLoopFor(&BB); if (Loop) { SmallVector ExitingBlocks; @@ -263,414 +171,301 @@ void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC, SmallVector Latches; Loop->getLoopLatches(Latches); - NumOuterLoops = LoopsInfo.OuterLoops; TotalLoops = LoopsInfo.TotalLoops; - MaximumLoopDepth = LoopsInfo.MaximumDepth; LoopDepth = Loop->getLoopDepth(); - LoopNumExitEdges = ExitEdges.size(); - LoopNumExitBlocks = ExitBlocks.size(); - LoopNumExitingBlocks = ExitingBlocks.size(); - LoopNumLatches = Latches.size(); LoopNumBlocks = Loop->getNumBlocks(); - LoopNumBackEdges = Loop->getNumBackEdges(); - LocalExitingBlock = Loop->isLoopExiting(&BB); LocalLatchBlock = Loop->isLoopLatch(&BB); LocalLoopHeader = ((Loop->getHeader() == (&BB)) ? 1 : 0); } unsigned NumLoads{0}; - unsigned NumStores{0}; - unsigned NumCallsExit{0}; unsigned NumCalls{0}; - unsigned NumCallsInvoke{0}; - unsigned NumTailCalls{0}; unsigned NumIndirectCalls{0}; - unsigned NumSelfCalls{0}; for (auto &Inst : BB) { if (BC.MIB->isLoad(Inst)) { ++NumLoads; - } else if (BC.MIB->isStore(Inst)) { - ++NumStores; } else if (BC.MIB->isCall(Inst)) { ++NumCalls; - if (BC.MIB->isIndirectCall(Inst)) ++NumIndirectCalls; - - if (BC.MIB->isInvoke(Inst)) - ++NumCallsInvoke; - - if (BC.MIB->isTailCall(Inst)) - ++NumTailCalls; - - if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { - StringRef CalleeName = CalleeSymbol->getName(); - if (CalleeName == "__cxa_throw@PLT" || - CalleeName == "_Unwind_Resume@PLT" || - CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || - CalleeName == "abort@PLT") - ++NumCallsExit; - else if (CalleeName == Function.getPrintName()) { - ++NumSelfCalls; - } - } } } int Index = -2; bool LoopHeader = SBI->isLoopHeader(&BB); + + BFIPtr BFI = std::make_unique(); + + + BFI->TotalLoops = TotalLoops; + BFI->LoopDepth = LoopDepth; + BFI->LoopNumBlocks = LoopNumBlocks; + BFI->LocalExitingBlock = LocalExitingBlock; + BFI->LocalLatchBlock = LocalLatchBlock; + BFI->LocalLoopHeader = LocalLoopHeader; + BFI->NumCalls = NumCalls; + BFI->BasicBlockSize = BB.size(); + BFI->NumBasicBlocks = Function.size(); + + BFI->NumLoads = NumLoads; + BFI->NumIndirectCalls = NumIndirectCalls; + BFI->LoopHeader = LoopHeader; + BFI->ProcedureType = ProcedureType; + + // Adding taken successor info. + addSuccessorInfo(BFI, Function, BC, BB, true); + // Adding fall through successor info. + addSuccessorInfo(BFI, Function, BC, BB, false); + + MCInst ConditionalInst; + bool hasConditionalBranch = false; + MCInst UnconditionalInst; + bool hasUnconditionalBranch = false; + for (auto &Inst : BB) { ++Index; - - if (!BC.MIA->isConditionalBranch(Inst)) + if (!BC.MIA->isConditionalBranch(Inst) && !BC.MIA->isUnconditionalBranch(Inst)) continue; - BFIPtr BFI = std::make_unique(); - - BFI->Simple = Simple; - BFI->NumOuterLoops = NumOuterLoops; - BFI->TotalLoops = TotalLoops; - BFI->MaximumLoopDepth = MaximumLoopDepth; - BFI->LoopDepth = LoopDepth; - BFI->LoopNumExitEdges = LoopNumExitEdges; - BFI->LoopNumExitBlocks = LoopNumExitBlocks; - BFI->LoopNumExitingBlocks = LoopNumExitingBlocks; - BFI->LoopNumLatches = LoopNumLatches; - BFI->LoopNumBlocks = LoopNumBlocks; - BFI->LoopNumBackEdges = LoopNumBackEdges; - - BFI->LocalExitingBlock = LocalExitingBlock; - BFI->LocalLatchBlock = LocalLatchBlock; - BFI->LocalLoopHeader = LocalLoopHeader; - - BFI->Call = ((NumCalls > 0) ? 1 : 0); - BFI->NumCalls = NumCalls; - - BFI->BasicBlockSize = BB.size(); - BFI->NumBasicBlocks = Function.size(); - BFI->NumSelfCalls = NumSelfCalls; - - BFI->NumLoads = NumLoads; - BFI->NumStores = NumStores; - BFI->NumCallsExit = NumCallsExit; - - BFI->NumCallsInvoke = NumCallsInvoke; - BFI->NumIndirectCalls = NumIndirectCalls; - BFI->NumTailCalls = NumTailCalls; - - // Check if branch's basic block is a loop header and store it. - BFI->LoopHeader = LoopHeader; - - // Adding taken successor info. - addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, true); - // Adding fall through successor info. - addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, false); - - // Holds the branch opcode info. - BFI->Opcode = Inst.getOpcode(); - BFI->OpcodeStr = BC.MII->getName(Inst.getOpcode()); - - // Holds the branch's procedure type. - BFI->ProcedureType = ProcedureType; - - BFI->CmpOpcode = 0; - if (Index > -1) { - auto Cmp = BB.begin() + Index; - - if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { - // Holding the branch comparison opcode info. - BFI->CmpOpcode = (*Cmp).getOpcode(); + generateInstFeatures(BC, BB, BFI, Index); - BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); - - auto getOperandType = [&](const MCOperand &Operand) -> int32_t { - if (Operand.isReg()) - return 0; - else if (Operand.isImm()) - return 1; - // else if (Operand.isFPImm()) - else if (Operand.isSFPImm()) - return 2; - else if (Operand.isExpr()) - return 3; - else - return -1; - }; - - const auto InstInfo = BC.MII->get((*Cmp).getOpcode()); - unsigned NumDefs = InstInfo.getNumDefs(); - int32_t NumPrimeOperands = - MCPlus::getNumPrimeOperands(*Cmp) - NumDefs; - switch (NumPrimeOperands) { - case 6: { - int32_t RBType = getOperandType((*Cmp).getOperand(NumDefs)); - int32_t RAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); - - if (RBType == 0 && RAType == 0) { - BFI->OperandRBType = RBType; - BFI->OperandRAType = RAType; - } else if (RBType == 0 && (RAType == 1 || RAType == 2)) { - RAType = getOperandType((*Cmp).getOperand(NumPrimeOperands - 1)); - - if (RAType != 1 && RAType != 2) { - RAType = -1; - } - - BFI->OperandRBType = RBType; - BFI->OperandRAType = RAType; - } else { - BFI->OperandRAType = -1; - BFI->OperandRBType = -1; - } - break; - } - case 2: - BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); - BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); - break; - case 3: - BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); - BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 2)); - break; - case 1: - BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs)); - break; - default: - BFI->OperandRAType = -1; - BFI->OperandRBType = -1; - break; - } - - } else { - Index -= 1; - for (int Idx = Index; Idx > -1; Idx--) { - auto Cmp = BB.begin() + Idx; - if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { - // Holding the branch comparison opcode info. - BFI->CmpOpcode = (*Cmp).getOpcode(); - BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); - break; - } - } - } + if (BC.MIA->isConditionalBranch(Inst)) { + ConditionalInst = Inst; + hasConditionalBranch = true; } + + + if (BC.MIA->isUnconditionalBranch(Inst)){ + UnconditionalInst = Inst; + hasUnconditionalBranch = true; + } + + } - //======================================================================== - - auto &FalseSuccessor = BFI->FalseSuccessor; - auto &TrueSuccessor = BFI->TrueSuccessor; + if (hasConditionalBranch) { + BFI->Opcode = ConditionalInst.getOpcode(); - if (!FalseSuccessor && !TrueSuccessor) - continue; + } else { + if (hasUnconditionalBranch) { + BFI->Opcode = UnconditionalInst.getOpcode(); - int64_t BranchOffset = - (BFI->BranchOffset.hasValue()) - ? static_cast(*(BFI->BranchOffset)) - : -1; - if(BranchOffset == -1) - continue; + } else { + auto Inst = BB.getLastNonPseudoInstr(); + BFI->Opcode = Inst->getOpcode(); + generateInstFeatures(BC, BB, BFI, Index); + } + } + + auto &FalseSuccessor = BFI->FalseSuccessor; + auto &TrueSuccessor = BFI->TrueSuccessor; - int16_t ProcedureType = (BFI->ProcedureType.hasValue()) + int16_t ProcedureType = (BFI->ProcedureType.hasValue()) ? static_cast(*(BFI->ProcedureType)) : -1; - int16_t Direction = (BFI->Direction.hasValue()) - ? static_cast(*(BFI->Direction)) - : -1; + int64_t Count = (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; - int16_t LoopHeader = (BFI->LoopHeader.hasValue()) - ? static_cast(*(BFI->LoopHeader)) - : -1; - - int32_t Opcode = - (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; - - int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) - ? static_cast(*(BFI->CmpOpcode)) - : -1; - - int64_t Count = - (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; - - int64_t MissPredicted = (BFI->MissPredicted.hasValue()) - ? static_cast(*(BFI->MissPredicted)) - : -1; - - int64_t FallthroughCount = - (BFI->FallthroughCount.hasValue()) + int64_t FallthroughCount = (BFI->FallthroughCount.hasValue()) ? static_cast(*(BFI->FallthroughCount)) : -1; - int64_t FallthroughMissPredicted = - (BFI->FallthroughMissPredicted.hasValue()) - ? static_cast(*(BFI->FallthroughMissPredicted)) - : -1; - - int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) - ? static_cast(*(BFI->NumOuterLoops)) - : -1; - int64_t TotalLoops = (BFI->TotalLoops.hasValue()) - ? static_cast(*(BFI->TotalLoops)) + int16_t LoopHeaderValid = (BFI->LoopHeader.hasValue()) + ? static_cast(*(BFI->LoopHeader)) : -1; - int64_t MaximumLoopDepth = - (BFI->MaximumLoopDepth.hasValue()) - ? static_cast(*(BFI->MaximumLoopDepth)) - : -1; - int64_t LoopDepth = (BFI->LoopDepth.hasValue()) + + int64_t TotalLoopsValid = (BFI->TotalLoops.hasValue()) + ? static_cast(*(BFI->TotalLoops)) + : -1; + int64_t LoopDepthValid = (BFI->LoopDepth.hasValue()) ? static_cast(*(BFI->LoopDepth)) : -1; - int64_t LoopNumExitEdges = - (BFI->LoopNumExitEdges.hasValue()) - ? static_cast(*(BFI->LoopNumExitEdges)) - : -1; - int64_t LoopNumExitBlocks = - (BFI->LoopNumExitBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumExitBlocks)) - : -1; - int64_t LoopNumExitingBlocks = - (BFI->LoopNumExitingBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumExitingBlocks)) - : -1; - int64_t LoopNumLatches = - (BFI->LoopNumLatches.hasValue()) - ? static_cast(*(BFI->LoopNumLatches)) - : -1; - int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumBlocks)) - : -1; - int64_t LoopNumBackEdges = - (BFI->LoopNumBackEdges.hasValue()) - ? static_cast(*(BFI->LoopNumBackEdges)) - : -1; + int64_t LoopNumBlocksValid = (BFI->LoopNumBlocks.hasValue()) + ? static_cast(*(BFI->LoopNumBlocks)) + : -1; + int64_t LocalExitingBlockValid = + (BFI->LocalExitingBlock.hasValue()) + ? static_cast(*(BFI->LocalExitingBlock)) + : -1; - int64_t LocalExitingBlock = - (BFI->LocalExitingBlock.hasValue()) - ? static_cast(*(BFI->LocalExitingBlock)) - : -1; + int64_t LocalLatchBlockValid = (BFI->LocalLatchBlock.hasValue()) + ? static_cast(*(BFI->LocalLatchBlock)) + : -1; - int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) - ? static_cast(*(BFI->LocalLatchBlock)) - : -1; + int64_t LocalLoopHeaderValid = (BFI->LocalLoopHeader.hasValue()) + ? static_cast(*(BFI->LocalLoopHeader)) + : -1; - int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) - ? static_cast(*(BFI->LocalLoopHeader)) - : -1; + int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) + ? static_cast(*(BFI->CmpOpcode)) + : -1; - int64_t Call = - (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; + int64_t OperandRAType = (BFI->OperandRAType.hasValue()) + ? static_cast(*(BFI->OperandRAType)) + : 10; - int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) - ? static_cast(*(BFI->DeltaTaken)) - : -1; + int64_t OperandRBType = (BFI->OperandRBType.hasValue()) + ? static_cast(*(BFI->OperandRBType)) + : 10; + int16_t Direction = (BFI->Direction.hasValue()) + ? static_cast(*(BFI->Direction)) + : -1; - int64_t NumLoads = (BFI->NumLoads.hasValue()) - ? static_cast(*(BFI->NumLoads)) - : -1; + int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) + ? static_cast(*(BFI->DeltaTaken)) + : -1; - int64_t NumStores = (BFI->NumStores.hasValue()) - ? static_cast(*(BFI->NumStores)) - : -1; + int64_t NumLoadsValid = (BFI->NumLoads.hasValue()) + ? static_cast(*(BFI->NumLoads)) + : -1; - int64_t BasicBlockSize = - (BFI->BasicBlockSize.hasValue()) - ? static_cast(*(BFI->BasicBlockSize)) - : -1; + int64_t BasicBlockSize = + (BFI->BasicBlockSize.hasValue()) + ? static_cast(*(BFI->BasicBlockSize)) + : -1; - int64_t NumBasicBlocks = - (BFI->NumBasicBlocks.hasValue()) - ? static_cast(*(BFI->NumBasicBlocks)) - : -1; + int64_t NumBasicBlocks = + (BFI->NumBasicBlocks.hasValue()) + ? static_cast(*(BFI->NumBasicBlocks)) + : -1; - int64_t NumCalls = (BFI->NumCalls.hasValue()) + int64_t NumCallsValid = (BFI->NumCalls.hasValue()) ? static_cast(*(BFI->NumCalls)) : -1; - int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) - ? static_cast(*(BFI->NumSelfCalls)) - : -1; - - int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) - ? static_cast(*(BFI->NumCallsExit)) - : -1; - - int64_t OperandRAType = (BFI->OperandRAType.hasValue()) - ? static_cast(*(BFI->OperandRAType)) - : -1; - - int64_t OperandRBType = (BFI->OperandRBType.hasValue()) - ? static_cast(*(BFI->OperandRBType)) - : -1; - - int64_t NumCallsInvoke = - (BFI->NumCallsInvoke.hasValue()) - ? static_cast(*(BFI->NumCallsInvoke)) - : -1; - - int64_t NumIndirectCalls = + int64_t NumIndirectCallsValid = (BFI->NumIndirectCalls.hasValue()) ? static_cast(*(BFI->NumIndirectCalls)) : -1; - int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) - ? static_cast(*(BFI->NumTailCalls)) - : -1; - - Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," - << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr - << "," << LoopHeader << "," << ProcedureType << "," << Count - << "," << MissPredicted << "," << FallthroughCount << "," - << FallthroughMissPredicted << "," << NumOuterLoops << "," - << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth - << "," << LoopDepth << "," << LoopNumExitEdges << "," - << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," - << LoopNumLatches << "," << LoopNumBlocks << "," - << LoopNumBackEdges << "," << LocalExitingBlock << "," - << LocalLatchBlock << "," << LocalLoopHeader << "," << Call << "," - << DeltaTaken << "," << NumLoads << "," << NumStores << "," - << NumCalls << "," << OperandRAType << "," << OperandRBType << "," - << BasicBlockSize << "," << NumBasicBlocks << "," - << NumCallsInvoke << "," << NumIndirectCalls << "," - << NumTailCalls << "," << NumSelfCalls; - - if (FalseSuccessor && TrueSuccessor) { - dumpSuccessorFeatures(Printer, TrueSuccessor); - dumpSuccessorFeatures(Printer, FalseSuccessor); - - FalseSuccessor.reset(); - TrueSuccessor.reset(); - } - BFI.reset(); - - std::string BranchOffsetStr = (BranchOffset == -1) ? "None" : Twine::utohexstr(BranchOffset).str(); + int64_t HasIndirectCalls = (NumIndirectCallsValid > 0) ? 1 : 0; + + int32_t Opcode = (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; + + uint64_t fun_exec = Function.getExecutionCount(); + fun_exec = (fun_exec != UINT64_MAX) ? fun_exec : 0; + + BBF.setDirection(Direction); + BBF.setDeltaTaken(DeltaTaken); + BBF.setOpcode(Opcode); + BBF.setCmpOpcode(CmpOpcode); + BBF.setOperandRAType(OperandRAType); + BBF.setOperandRBType(OperandRBType); + BBF.setFunExec(fun_exec); + BBF.setTotalLoops(TotalLoopsValid); + BBF.setLoopDepth(LoopDepthValid); + BBF.setLoopNumBlocks(LoopNumBlocksValid); + BBF.setLocalExitingBlock(LocalExitingBlockValid); + BBF.setLocalLatchBlock(LocalLatchBlockValid); + BBF.setLocalLoopHeader(LocalLoopHeaderValid); + BBF.setNumCalls(NumCallsValid); + BBF.setBasicBlockSize(BasicBlockSize); + BBF.setNumBasicBlocks(NumBasicBlocks); + BBF.setNumLoads(NumLoadsValid); + BBF.setHasIndirectCalls(HasIndirectCalls); + BBF.setLoopHeader(LoopHeaderValid); + BBF.setProcedureType(ProcedureType); + BBF.setCount(Count); + BBF.setFallthroughCount(FallthroughCount); + + + generateSuccessorFeatures(TrueSuccessor, &BBF); + generateSuccessorFeatures(FalseSuccessor, &BBF); + + FalseSuccessor.reset(); + TrueSuccessor.reset(); + + BBF.setInferenceFeatures(); + BB.setFeatures(BBF); + + BFI.reset(); + } +} - uint64_t fun_exec = Function.getExecutionCount(); - fun_exec = (fun_exec != UINT64_MAX) ? fun_exec : 0; - Printer << "," << Twine::utohexstr(Function.getAddress()) << "," - << fun_exec << "," << Function.getFunctionNumber() << "," - << Function.getOneName() << "," << Function.getPrintName() - << "," << BranchOffsetStr - << "\n"; +void FeatureMiner::generateInstFeatures(BinaryContext &BC, BinaryBasicBlock &BB, BFIPtr const &BFI, int Index) { + + // Holds the branch opcode info. + + BFI->CmpOpcode = 0; + if (Index > -1) { + auto Cmp = BB.begin() + Index; + if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { + // Holding the branch comparison opcode info. + BFI->CmpOpcode = (*Cmp).getOpcode(); + auto getOperandType = [&](const MCOperand &Operand) -> int32_t { + if (Operand.isReg()) + return 0; + else if (Operand.isImm()) + return 1; + else if (Operand.isSFPImm()) + return 2; + else if (Operand.isExpr()) + return 3; + else + return -1; + }; + + const auto InstInfo = BC.MII->get((*Cmp).getOpcode()); + unsigned NumDefs = InstInfo.getNumDefs(); + int32_t NumPrimeOperands = + MCPlus::getNumPrimeOperands(*Cmp) - NumDefs; + switch (NumPrimeOperands) { + case 6: { + int32_t RBType = getOperandType((*Cmp).getOperand(NumDefs)); + int32_t RAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); + + if (RBType == 0 && RAType == 0) { + BFI->OperandRBType = RBType; + BFI->OperandRAType = RAType; + } else if (RBType == 0 && (RAType == 1 || RAType == 2)) { + RAType = getOperandType((*Cmp).getOperand(NumPrimeOperands - 1)); + + if (RAType != 1 && RAType != 2) { + RAType = -1; + } - //======================================================================== + BFI->OperandRBType = RBType; + BFI->OperandRAType = RAType; + } else { + BFI->OperandRAType = -1; + BFI->OperandRBType = -1; + } + break; + } + case 2: + BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); + BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); + break; + case 3: + BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); + BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 2)); + break; + case 1: + BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs)); + break; + default: + BFI->OperandRAType = -1; + BFI->OperandRBType = -1; + break; + } - // this->BranchesInfoSet.push_back(std::move(BFI)); + } else { + Index -= 1; + for (int Idx = Index; Idx > -1; Idx--) { + auto Cmp = BB.begin() + Idx; + if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { + // Holding the branch comparison opcode info. + BFI->CmpOpcode = (*Cmp).getOpcode(); + break; + } + } } } } -void FeatureMiner::dumpSuccessorFeatures(raw_ostream &Printer, - BBIPtr &Successor) { - int16_t BranchDominates = - (Successor->BranchDominates.hasValue()) - ? static_cast(*(Successor->BranchDominates)) - : -1; +void FeatureMiner::generateSuccessorFeatures(BBIPtr &Successor, BinaryBasicBlockFeature *BBF) { - int16_t BranchPostdominates = - (Successor->BranchPostdominates.hasValue()) - ? static_cast(*(Successor->BranchPostdominates)) - : -1; int16_t LoopHeader = (Successor->LoopHeader.hasValue()) ? static_cast(*(Successor->LoopHeader)) @@ -690,378 +485,86 @@ void FeatureMiner::dumpSuccessorFeatures(raw_ostream &Printer, ? static_cast(*(Successor->EndOpcode)) : -1; - int64_t NumLoads = (Successor->NumLoads.hasValue()) - ? static_cast(*(Successor->NumLoads)) - : -1; - - int64_t NumStores = (Successor->NumStores.hasValue()) - ? static_cast(*(Successor->NumStores)) - : -1; - int64_t BasicBlockSize = (Successor->BasicBlockSize.hasValue()) ? static_cast(*(Successor->BasicBlockSize)) : -1; - int64_t NumCalls = (Successor->NumCalls.hasValue()) - ? static_cast(*(Successor->NumCalls)) - : -1; - - int64_t NumCallsExit = (Successor->NumCallsExit.hasValue()) - ? static_cast(*(Successor->NumCallsExit)) - : -1; - - int64_t NumCallsInvoke = - (Successor->NumCallsInvoke.hasValue()) - ? static_cast(*(Successor->NumCallsInvoke)) - : -1; - - int64_t NumIndirectCalls = - (Successor->NumIndirectCalls.hasValue()) - ? static_cast(*(Successor->NumIndirectCalls)) - : -1; + BBF->setEndOpcodeVec(EndOpcode); + BBF->setLoopHeaderVec(LoopHeader); + BBF->setBackedgeVec(Backedge); + BBF->setExitVec(Exit); + BBF->setCallVec(Call); + BBF->setBasicBlockSizeVec(BasicBlockSize); - int64_t NumTailCalls = (Successor->NumTailCalls.hasValue()) - ? static_cast(*(Successor->NumTailCalls)) - : -1; - - Printer << "," << BranchDominates << "," << BranchPostdominates << "," - << EndOpcode << "," << Successor->EndOpcodeStr << "," << LoopHeader - << "," << Backedge << "," << Exit << "," << Call << "," - << Successor->FromFunName << "," - << Twine::utohexstr(Successor->FromBb) << "," << Successor->ToFunName - << "," << Twine::utohexstr(Successor->ToBb) << "," << NumLoads << "," - << NumStores << "," << BasicBlockSize << "," << NumCalls << "," - << NumCallsExit << "," << NumIndirectCalls << "," << NumCallsInvoke - << "," << NumTailCalls; } -void FeatureMiner::dumpFeatures(raw_ostream &Printer, uint64_t FunctionAddress, - uint64_t FunctionFrequency) { - - for (auto const &BFI : BranchesInfoSet) { - auto &FalseSuccessor = BFI->FalseSuccessor; - auto &TrueSuccessor = BFI->TrueSuccessor; - - if (!FalseSuccessor && !TrueSuccessor) - continue; - - int16_t ProcedureType = (BFI->ProcedureType.hasValue()) - ? static_cast(*(BFI->ProcedureType)) - : -1; - - int16_t Direction = - (BFI->Direction.hasValue()) ? static_cast(*(BFI->Direction)) : -1; - - int16_t LoopHeader = (BFI->LoopHeader.hasValue()) - ? static_cast(*(BFI->LoopHeader)) - : -1; - - int32_t Opcode = - (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; - - int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) - ? static_cast(*(BFI->CmpOpcode)) - : -1; - - int64_t Count = - (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; - - int64_t MissPredicted = (BFI->MissPredicted.hasValue()) - ? static_cast(*(BFI->MissPredicted)) - : -1; - - int64_t FallthroughCount = - (BFI->FallthroughCount.hasValue()) - ? static_cast(*(BFI->FallthroughCount)) - : -1; - - int64_t FallthroughMissPredicted = - (BFI->FallthroughMissPredicted.hasValue()) - ? static_cast(*(BFI->FallthroughMissPredicted)) - : -1; - - int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) - ? static_cast(*(BFI->NumOuterLoops)) - : -1; - int64_t TotalLoops = (BFI->TotalLoops.hasValue()) - ? static_cast(*(BFI->TotalLoops)) - : -1; - int64_t MaximumLoopDepth = - (BFI->MaximumLoopDepth.hasValue()) - ? static_cast(*(BFI->MaximumLoopDepth)) - : -1; - int64_t LoopDepth = (BFI->LoopDepth.hasValue()) - ? static_cast(*(BFI->LoopDepth)) - : -1; - int64_t LoopNumExitEdges = - (BFI->LoopNumExitEdges.hasValue()) - ? static_cast(*(BFI->LoopNumExitEdges)) - : -1; - int64_t LoopNumExitBlocks = - (BFI->LoopNumExitBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumExitBlocks)) - : -1; - int64_t LoopNumExitingBlocks = - (BFI->LoopNumExitingBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumExitingBlocks)) - : -1; - int64_t LoopNumLatches = (BFI->LoopNumLatches.hasValue()) - ? static_cast(*(BFI->LoopNumLatches)) - : -1; - int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) - ? static_cast(*(BFI->LoopNumBlocks)) - : -1; - int64_t LoopNumBackEdges = - (BFI->LoopNumBackEdges.hasValue()) - ? static_cast(*(BFI->LoopNumBackEdges)) - : -1; - - int64_t LocalExitingBlock = - (BFI->LocalExitingBlock.hasValue()) - ? static_cast(*(BFI->LocalExitingBlock)) - : -1; - - int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) - ? static_cast(*(BFI->LocalLatchBlock)) - : -1; - - int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) - ? static_cast(*(BFI->LocalLoopHeader)) - : -1; - - int64_t Call = - (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; - - int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) - ? static_cast(*(BFI->DeltaTaken)) - : -1; - - int64_t NumLoads = (BFI->NumLoads.hasValue()) - ? static_cast(*(BFI->NumLoads)) - : -1; - - int64_t NumStores = (BFI->NumStores.hasValue()) - ? static_cast(*(BFI->NumStores)) - : -1; - - int64_t BasicBlockSize = (BFI->BasicBlockSize.hasValue()) - ? static_cast(*(BFI->BasicBlockSize)) - : -1; - - int64_t BranchOffset = (BFI->BranchOffset.hasValue()) - ? static_cast(*(BFI->BranchOffset)): -1; - - int64_t NumBasicBlocks = (BFI->NumBasicBlocks.hasValue()) - ? static_cast(*(BFI->NumBasicBlocks)) - : -1; - - int64_t NumCalls = (BFI->NumCalls.hasValue()) - ? static_cast(*(BFI->NumCalls)) - : -1; - - int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) - ? static_cast(*(BFI->NumSelfCalls)) - : -1; - - int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) - ? static_cast(*(BFI->NumCallsExit)) - : -1; - - int64_t OperandRAType = (BFI->OperandRAType.hasValue()) - ? static_cast(*(BFI->OperandRAType)) - : -1; - - int64_t OperandRBType = (BFI->OperandRBType.hasValue()) - ? static_cast(*(BFI->OperandRBType)) - : -1; - - int64_t NumCallsInvoke = (BFI->NumCallsInvoke.hasValue()) - ? static_cast(*(BFI->NumCallsInvoke)) - : -1; +void FeatureMiner::runOnFunctions(BinaryContext &BC){ - int64_t NumIndirectCalls = - (BFI->NumIndirectCalls.hasValue()) - ? static_cast(*(BFI->NumIndirectCalls)) - : -1; - - int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) - ? static_cast(*(BFI->NumTailCalls)) - : -1; - - Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," - << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr << "," - << LoopHeader << "," << ProcedureType << "," << Count << "," - << MissPredicted << "," << FallthroughCount << "," - << FallthroughMissPredicted << "," << NumOuterLoops << "," - << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth - << "," << LoopDepth << "," << LoopNumExitEdges << "," - << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," - << LoopNumLatches << "," << LoopNumBlocks << "," << LoopNumBackEdges - << "," << LocalExitingBlock << "," << LocalLatchBlock << "," - << LocalLoopHeader << "," << Call << "," << DeltaTaken << "," - << NumLoads << "," << NumStores << "," << NumCalls << "," - << OperandRAType << "," << OperandRBType << "," << BasicBlockSize - << "," << NumBasicBlocks << "," << NumCallsInvoke << "," - << NumIndirectCalls << "," << NumTailCalls << "," << NumSelfCalls; - - if (FalseSuccessor && TrueSuccessor) { - dumpSuccessorFeatures(Printer, TrueSuccessor); - dumpSuccessorFeatures(Printer, FalseSuccessor); - } - - Printer << "," << Twine::utohexstr(FunctionAddress) << "," - << FunctionFrequency << "\n"; - } - BranchesInfoSet.clear(); } -void FeatureMiner::runOnFunctions(BinaryContext &BC) { - auto FileName = "features_new.csv"; - outs() << "BOLT-INFO: Starting feature miner pass\n"; - - std::error_code EC; -// raw_fd_ostream Printer(FileName, EC, sys::fs::F_None); - raw_fd_ostream Printer(FileName, EC, sys::fs::OF_None); - - if (EC) { - errs() << "BOLT-WARNING: " << EC.message() << ", unable to open " - << FileName << " for output.\n"; - return; - } - - auto FILENAME = "profile_data_regular.fdata"; -// raw_fd_ostream Printer2(FILENAME, EC, sys::fs::F_None); - raw_fd_ostream Printer2(FILENAME, EC, sys::fs::OF_None); - - if (EC) { - dbgs() << "BOLT-WARNING: " << EC.message() << ", unable to open" - << " " << FILENAME << " for output.\n"; - return; - } +void FeatureMiner::inferenceFeatures(BinaryFunction &Function){ - // CSV file header - Printer << "FUN_TYPE,OPCODE,OPCODE_STR,DIRECTION,CMP_OPCODE,CMP_OPCODE_STR," - "LOOP_HEADER,PROCEDURE_TYPE," - "COUNT_TAKEN,MISS_TAKEN,COUNT_NOT_TAKEN,MISS_NOT_TAKEN," - "NUM_OUTER_LOOPS,NUM_CALLS_EXIT,TOTAL_LOOPS,MAXIMUM_LOOP_DEPTH," - "LOOP_DEPTH,LOOP_NUM_EXIT_EDGES,LOOP_NUM_EXIT_BLOCKS," - "LOOP_NUM_EXITING_BLOCKS,LOOP_NUM_LATCHES,LOOP_NUM_BLOCKS," - "LOOP_NUM_BAKCEDGES,LOCAL_EXITING_BLOCK,LOCAL_LATCH_BLOCK," - "LOCAL_LOOP_HEADER,CALL,DELTA_TAKEN,NUM_LOADS,NUM_STORES," - "NUM_CALLS,OPERAND_RA_TYPE,OPERAND_RB_TYPE,BASIC_BLOCK_SIZE," - "NUM_BASIC_BLOCKS,NUM_CALLS_INVOKE,NUM_INDIRECT_CALLS," - "NUM_TAIL_CALLS,NUM_SELF_CALLS,TS_DOMINATES,TS_POSTDOMINATES," - "TS_END_OPCODE,TS_END_OPCODE_STR,TS_LOOP_HEADER,TS_BACKEDGE,TS_" - "EXIT,TS_CALL," - "TS_FROM_FUN_NAME,TS_FROM_BB,TS_TO_FUN_NAME,TS_TO_BB,TS_NUM_LOADS," - "TS_NUM_STORES,TS_BASIC_BLOCK_SIZE,TS_NUM_CALLS,TS_NUM_CALLS_EXIT," - "TS_NUM_INDIRECT_CALL,TS_NUM_CALLS_INVOKE,TS_NUM_TAIL_CALLS," - "FS_DOMINATES,FS_POSTDOMINATES,FS_END_OPCODE,FS_END_OPCODE_STR,FS_" - "LOOP_HEADER," - "FS_BACKEDGE,FS_EXIT,FS_CALL,FS_FROM_FUN_NAME,FS_FROM_BB," - "FS_TO_FUN_NAME,FS_TO_BB,FS_NUM_LOADS,FS_NUM_STORES," - "FS_BASIC_BLOCK_SIZE,FS_NUM_CALLS,FS_NUM_CALLS_EXIT," - "FS_NUM_INDIRECT_CALL,FS_NUM_CALLS_INVOKE,FS_NUM_TAIL_CALLS," - "FUN_ENTRY_ADDRESS,FUN_ENTRY_FREQUENCY" - ",FUN_UNIQUE_NUMBER,FUN_ONE_NAME,FUN_PRINT_NAME," - "BRANCH_ADDRESS\n"; - - auto &BFs = BC.getBinaryFunctions(); SBI = std::make_unique(); - for (auto &BFI : BFs) { - BinaryFunction &Function = BFI.second; - if (Function.empty() || (Function.hasValidProfile() && opts::IncludeValidProfile)) - continue; - - if (!Function.isLoopFree()) { - const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); - SBI->findLoopEdgesInfo(LoopsInfo); - } - extractFeatures(Function, BC, Printer); - - SBI->clear(); + if (Function.empty()) + return; - // dumpProfileData(Function, Printer2); + if (!Function.isLoopFree()) { + const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); + SBI->findLoopEdgesInfo(LoopsInfo); } - outs() << "BOLT-INFO: Dumping two-way conditional branches' features" - << " at " << FileName << "\n"; -} - -/*void FeatureMiner::dumpProfileData(BinaryFunction &Function, - raw_ostream &Printer) { - BinaryContext &BC = Function.getBinaryContext(); + extractFeatures(Function, BC); - std::string FromFunName = Function.getPrintName(); - for (auto &BB : Function) { - auto LastInst = BB.getLastNonPseudoInstr(); - - for (auto &Inst : BB) { - if (!BC.MIB->isCall(Inst) && !BC.MIB->isBranch(Inst) && - LastInst != (&Inst)) - continue; - - auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); - - if (!Offset) - continue; - - uint64_t TakenFreqEdge = 0; - auto FromBb = Offset.get(); - std::string ToFunName; - uint32_t ToBb; - - if (BC.MIB->isCall(Inst)) { - auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst); - if (!CalleeSymbol) - continue; - - ToFunName = CalleeSymbol->getName(); - ToBb = 0; + SBI->clear(); +} - if (BC.MIB->getConditionalTailCall(Inst)) { +void FeatureMiner::generateProfileFeatures(BinaryBasicBlock *BB, BinaryBasicBlockFeature *BBF) { + int32_t parentChildNum, parentCount, childParentNum, childCount; - if (BC.MIB->hasAnnotation(Inst, "CTCTakenCount")) { - auto CountAnnt = - BC.MIB->tryGetAnnotationAs(Inst, "CTCTakenCount"); - if (CountAnnt) { - TakenFreqEdge = (*CountAnnt); - } - } - } else { - if (BC.MIB->hasAnnotation(Inst, "Count")) { - auto CountAnnt = - BC.MIB->tryGetAnnotationAs(Inst, "Count"); - if (CountAnnt) { - TakenFreqEdge = (*CountAnnt); - } - } - } + if (BB->ParentSet.size() == 0) { + parentChildNum = -1; + parentCount = -1; + } else { + parentChildNum = std::numeric_limits::max(); + parentCount = 0; + for (BinaryBasicBlock *parent: BB->ParentSet) { + if (parent->ChildrenSet.size() < parentChildNum) { + parentChildNum = parent->ChildrenSet.size(); + parentCount = parent->getExecutionCount(); + } else if (parent->ChildrenSet.size() == parentChildNum && parent->getExecutionCount() > parentCount) { + parentCount = parent->getExecutionCount(); + } + } + } - if (TakenFreqEdge > 0) - Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) - << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) << " " - << 0 << " " << TakenFreqEdge << "\n"; - } else { - for (BinaryBasicBlock *SuccBB : BB.successors()) { - TakenFreqEdge = BB.getBranchInfo(*SuccBB).Count; - BinaryFunction *ToFun = SuccBB->getFunction(); - ToFunName = ToFun->getPrintName(); - ToBb = SuccBB->getInputOffset(); - - if (TakenFreqEdge > 0) - Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) - << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) - << " " << 0 << " " << TakenFreqEdge << "\n"; - } + if (BB->ChildrenSet.size() == 0) { + childParentNum = -1; + childCount = -1; + } else { + childParentNum = std::numeric_limits::max(); + childCount = 0; + for (BinaryBasicBlock *child: BB->ChildrenSet) { + if (child->ParentSet.size() < childParentNum) { + childParentNum = child->ParentSet.size(); + childCount = child->getExecutionCount(); + } else if (child->ParentSet.size() == childParentNum && child->getExecutionCount() > childCount) { + childCount = child->getExecutionCount(); } } } + + int64_t parentCountCatch = parentCount > 0 ? 1 : 0; + int64_t childCountCatch = childCount > 0 ? 1 : 0; + + BBF->setParentChildNum(parentChildNum); + BBF->setParentCount(parentCountCatch); + BBF->setChildParentNum(childParentNum); + BBF->setChildCount(childCountCatch); + } -*/ } // namespace bolt } // namespace llvm \ No newline at end of file diff --git a/bolt/lib/Passes/StaticBranchInfo.cpp b/bolt/lib/Passes/StaticBranchInfo.cpp index 13426b397..5a3e0ec9c 100644 --- a/bolt/lib/Passes/StaticBranchInfo.cpp +++ b/bolt/lib/Passes/StaticBranchInfo.cpp @@ -15,10 +15,9 @@ // //===----------------------------------------------------------------------===// -// #include "Passes/StaticBranchInfo.h" -// #include "BinaryBasicBlock.h" #include "bolt/Passes/StaticBranchInfo.h" #include "bolt/Core/BinaryBasicBlock.h" +#include "bolt/Core/BinaryLoop.h" namespace llvm { namespace bolt { @@ -90,7 +89,6 @@ bool StaticBranchInfo::isExitEdge(const BinaryLoop::Edge &CFGEdge) const { bool StaticBranchInfo::isExitEdge(const BinaryBasicBlock *SrcBB, const BinaryBasicBlock *DstBB) const { -// const BinaryLoop::Edge CFGEdge = std::make_pair(SrcBB, DstBB); const BinaryLoop::Edge CFGEdge = std::make_pair(const_cast(SrcBB), const_cast(DstBB)); return isExitEdge(CFGEdge); } @@ -159,4 +157,4 @@ void StaticBranchInfo::clear() { } } // namespace bolt -} // namespace llvm \ No newline at end of file +} // namespace llvm diff --git a/bolt/lib/Profile/CMakeLists.txt b/bolt/lib/Profile/CMakeLists.txt index f4397e331..2273f4572 100644 --- a/bolt/lib/Profile/CMakeLists.txt +++ b/bolt/lib/Profile/CMakeLists.txt @@ -11,6 +11,7 @@ add_llvm_library(LLVMBOLTProfile Support ) + target_link_libraries(LLVMBOLTProfile PRIVATE LLVMBOLTCore diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp index a51201d3b..2ae715049 100644 --- a/bolt/lib/Profile/DataReader.cpp +++ b/bolt/lib/Profile/DataReader.cpp @@ -11,33 +11,48 @@ // //===----------------------------------------------------------------------===// -#include "bolt/Profile/DataReader.h" #include "bolt/Core/BinaryFunction.h" +#include "bolt/Passes/FeatureMiner.h" #include "bolt/Passes/MCF.h" +#include "bolt/Profile/DataReader.h" #include "bolt/Utils/Utils.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" +#include +#include +#include #include +#include -#undef DEBUG_TYPE +#undef DEBUG_TYPE #define DEBUG_TYPE "bolt-prof" using namespace llvm; namespace opts { - +extern cl::opt BlockCorrection; extern cl::OptionCategory BoltCategory; extern llvm::cl::opt Verbosity; -static cl::opt -DumpData("dump-data", - cl::desc("dump parsed bolt data for debugging"), - cl::Hidden, - cl::cat(BoltCategory)); +static cl::opt InputModelFilename("model-path", + cl::desc(""), + cl::Optional, + cl::cat(BoltCategory)); + +static cl::opt DumpData("dump-data", + cl::desc("dump parsed bolt data for debugging"), + cl::Hidden, cl::cat(BoltCategory)); } // namespace opts +extern "C" { +typedef void *(*CreateONNXRunnerFunc)(const char *); +typedef void (*DeleteONNXRunnerFunc)(void *); +typedef float (*RunONNXModelFunc)(void *, std::vector &, + std::vector &, std::vector &); +} + namespace llvm { namespace bolt { @@ -322,6 +337,16 @@ Error DataReader::readProfilePreCFG(BinaryContext &BC) { } Error DataReader::readProfile(BinaryContext &BC) { + + if (opts::BlockCorrection) { + if (opts::InputModelFilename.empty()) { + outs() << "llvm-bolt: expected -model-path= option.\n"; + exit(1); + } else { + DataReader::initializeONNXRunner(opts::InputModelFilename); + } + } + for (auto &BFI : BC.getBinaryFunctions()) { BinaryFunction &Function = BFI.second; readProfile(Function); @@ -333,6 +358,12 @@ Error DataReader::readProfile(BinaryContext &BC) { ++NumUnused; BC.setNumUnusedProfiledObjects(NumUnused); + if (opts::BlockCorrection) { + uint64_t modified_total = DataReader::getModifiedBBTotal(); + outs() << "BOLT-INFO: total modified CFG BB count number is " + << modified_total << ".\n"; + } + return Error::success(); } @@ -564,11 +595,75 @@ float DataReader::evaluateProfileData(BinaryFunction &BF, return MatchRatio; } +void generateChildrenParentCount(BinaryBasicBlock *BB) { + typedef GraphTraits GraphT; + + for (typename GraphT::ChildIteratorType CI = GraphT::child_begin(BB), + E = GraphT::child_end(BB); + CI != E; ++CI) { + typename GraphT::NodeRef Child = *CI; + BB->ChildrenSet.insert(Child); + Child->ParentSet.insert(BB); + } +} + +void generateChildrenParentCount(BinaryFunction &BF) { + for (BinaryBasicBlock &BB : BF) { + generateChildrenParentCount(&BB); + } +} + +uint64_t estimateBBCount(DataReader *dataReaderRef, BinaryBasicBlock *BB, + float threshold) { + uint64_t modified = 0; + std::vector input_string; + std::vector input_int64; + std::vector input_float; + + BinaryBasicBlockFeature BBF = BB->getFeatures(); + input_int64 = BBF.getInferenceFeatures(); + + if (input_int64.empty()) { + return 0; + } + + float model_pred = + dataReaderRef->ONNXInference(input_string, input_int64, input_float); + if (BB->getExecutionCount() == 0 && model_pred >= threshold) { + uint64_t min_neighbor_count = std::numeric_limits::max(); + for (BinaryBasicBlock *parent : BB->ParentSet) { + if (parent->getExecutionCount() > 0 && + parent->getExecutionCount() < min_neighbor_count) + min_neighbor_count = parent->getExecutionCount(); + } + for (BinaryBasicBlock *child : BB->ChildrenSet) { + if (child->getExecutionCount() > 0 && + child->getExecutionCount() < min_neighbor_count) + min_neighbor_count = child->getExecutionCount(); + } + if (min_neighbor_count != std::numeric_limits::max()) { + BB->setExecutionCount(min_neighbor_count); + modified = 1; + } + } + return modified; +} + +uint64_t estimateBBCount(DataReader *dataReaderRef, BinaryFunction &BF, + float threshold) { + uint64_t modified_total_func = 0; + const auto &Order = BF.dfs(); + for (auto *BBA : Order) { + auto &BB = *BBA; + modified_total_func += estimateBBCount(dataReaderRef, &BB, threshold); + } + return modified_total_func; +} + void DataReader::readSampleData(BinaryFunction &BF) { FuncSampleData *SampleDataOrErr = getFuncSampleData(BF.getNames()); if (!SampleDataOrErr) return; - // Basic samples mode territory (without LBR info) // First step is to assign BB execution count based on samples from perf BF.ProfileMatchRatio = 1.0f; @@ -609,6 +704,17 @@ void DataReader::readSampleData(BinaryFunction &BF) { BF.ExecutionCount = TotalEntryCount; + if (opts::BlockCorrection) { + generateChildrenParentCount(BF); + std::unique_ptr FM = + std::make_unique(opts::BlockCorrection); + FM->inferenceFeatures(BF); + + float threshold = DataReader::getThreshold(); + uint64_t modified_total_func = estimateBBCount(this, BF, threshold); + DataReader::addModifiedBBTotal(modified_total_func); + } + estimateEdgeCounts(BF); } @@ -1324,7 +1430,7 @@ std::vector fetchMapEntriesRegex( return AllData; } -} +} // namespace bool DataReader::mayHaveProfileData(const BinaryFunction &Function) { if (getBranchData(Function) || getMemData(Function)) diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index fd21724c5..3bed5c16e 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -16,6 +16,7 @@ #include "bolt/Core/ParallelUtilities.h" #include "bolt/Core/Relocation.h" #include "bolt/Passes/CacheMetrics.h" +#include "bolt/Passes/FeatureMiner.h" #include "bolt/Passes/ReorderFunctions.h" #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Profile/DataAggregator.h" @@ -104,6 +105,12 @@ cl::opt DumpDotAll( "enable '-print-loops' for color-coded blocks"), cl::Hidden, cl::cat(BoltCategory)); +cl::opt BlockCorrection( + "block-correction", + cl::desc("capture features useful for ML model to inference the count on the binary basic block" + " and correct them on CFG."), + cl::ZeroOrMore, cl::cat(BoltOptCategory)); + static cl::list ForceFunctionNames("funcs", cl::CommaSeparated, -- 2.33.0