From 96e6ced072eaea03169672fb38a4900a4835c421 Mon Sep 17 00:00:00 2001 From: zhouzeping Date: Wed, 5 Jun 2024 12:23:30 +0800 Subject: [PATCH] Add CFG block count correction optimization. --- ...e-code-related-to-feature-extracting.patch | 1593 +++++++++++ 0005-Add-block-correction-optimization.patch | 2325 +++++++++++++++++ llvm-bolt.spec | 10 +- 3 files changed, 3927 insertions(+), 1 deletion(-) create mode 100644 0004-Added-open-source-code-related-to-feature-extracting.patch create mode 100644 0005-Add-block-correction-optimization.patch diff --git a/0004-Added-open-source-code-related-to-feature-extracting.patch b/0004-Added-open-source-code-related-to-feature-extracting.patch new file mode 100644 index 0000000..3bd91e1 --- /dev/null +++ b/0004-Added-open-source-code-related-to-feature-extracting.patch @@ -0,0 +1,1593 @@ +From 96eff0ec88e75a49cc186476efd84370e6137b42 Mon Sep 17 00:00:00 2001 +From: h00502206 +Date: Tue, 4 Jun 2024 20:18:05 +0800 +Subject: [PATCH] Added open-source code related to feature extracting from + 'angelica-moreira: https://github.com/angelica-moreira/BOLT' on the basis of + llvm-bolt, and modified some code to pass the compilation. + +--- + .../bolt/include/bolt/Passes/FeatureMiner.h | 178 +++ + .../include/bolt/Passes/StaticBranchInfo.h | 116 ++ + bolt/lib/Passes/CMakeLists.txt | 2 + + bolt/lib/Passes/FeatureMiner.cpp | 1067 +++++++++++++++++ + .../bolt/lib/Passes/StaticBranchInfo.cpp | 162 +++ + 5 files changed, 1525 insertions(+) + create mode 100644 bolt/include/bolt/Passes/FeatureMiner.h + create mode 100644 bolt/include/bolt/Passes/StaticBranchInfo.h + create mode 100644 bolt/lib/Passes/FeatureMiner.cpp + create mode 100644 bolt/lib/Passes/StaticBranchInfo.cpp + +diff --git a/bolt/include/bolt/Passes/FeatureMiner.h b/bolt/include/bolt/Passes/FeatureMiner.h +new file mode 100644 +index 000000000..916e5515d +--- /dev/null ++++ b/bolt/include/bolt/Passes/FeatureMiner.h +@@ -0,0 +1,178 @@ ++//===--- Passes/FeatureMiner.h ---------------------------------------------===// ++// ++// The LLVM Compiler Infrastructure ++// ++// This file is distributed under the University of Illinois Open Source ++// License. See LICENSE.TXT for details. ++// ++//===----------------------------------------------------------------------===// ++// A very simple feature extractor based on Calder's paper ++// Evidence-based static branch prediction using machine learning ++// https://dl.acm.org/doi/10.1145/239912.239923 ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ ++#define LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ ++ ++// #include "BinaryContext.h" ++// #include "BinaryFunction.h" ++// #include "BinaryLoop.h" ++// #include "DominatorAnalysis.h" ++// #include "Passes/BinaryPasses.h" ++// #include "Passes/StaticBranchInfo.h" ++#include "bolt/Core/BinaryData.h" ++#include "bolt/Core/BinaryFunction.h" ++#include "bolt/Core/BinaryLoop.h" ++#include "bolt/Passes/DominatorAnalysis.h" ++#include "bolt/Passes/BinaryPasses.h" ++#include "bolt/Passes/StaticBranchInfo.h" ++ ++#include "llvm/ADT/DenseMap.h" ++#include "llvm/ADT/Optional.h" ++#include "llvm/ADT/StringRef.h" ++#include "llvm/MC/MCInst.h" ++#include "llvm/Support/raw_ostream.h" ++#include ++#include ++#include ++#include ++ ++namespace llvm { ++namespace bolt { ++ ++class FeatureMiner : public BinaryFunctionPass { ++private: ++ std::unique_ptr SBI; ++ ++ /// BasicBlockInfo - This structure holds feature information about the target ++ /// BasicBlock of either the taken or the fallthrough paths of a given branch. ++ struct BasicBlockInfo { ++ Optional BranchDominates; // 1 - dominates, 0 - does not dominate ++ Optional BranchPostdominates; // 1 - postdominates, 0 - does not PD ++ Optional LoopHeader; // 1 - loop header, 0 - not a loop header ++ Optional Backedge; // 1 - loop back, 0 - not a loop back ++ Optional Exit; // 1 - loop exit, 0 - not a loop exit ++ Optional Call; // 1 - program call, 0 - not a program call ++ Optional NumCalls; ++ Optional NumLoads; ++ Optional NumStores; ++ Optional EndOpcode; // 0 = NOTHING ++ StringRef EndOpcodeStr = "UNDEF"; ++ Optional BasicBlockSize; ++ std::string FromFunName = "UNDEF"; ++ uint32_t FromBb; ++ std::string ToFunName = "UNDEF"; ++ uint32_t ToBb; ++ ++ ++ Optional NumCallsExit; ++ Optional NumCallsInvoke; ++ Optional NumIndirectCalls; ++ Optional NumTailCalls; ++ }; ++ ++ typedef std::unique_ptr BBIPtr; ++ ++ /// BranchFeaturesInfo - This structure holds feature information about each ++ /// two-way branch from the program. ++ struct BranchFeaturesInfo { ++ StringRef OpcodeStr = "UNDEF"; ++ StringRef CmpOpcodeStr = "UNDEF"; ++ bool Simple = 0; ++ ++ Optional Opcode; ++ Optional CmpOpcode; ++ Optional Count; ++ Optional MissPredicted; ++ Optional FallthroughCount; ++ Optional FallthroughMissPredicted; ++ BBIPtr TrueSuccessor = std::make_unique(); ++ BBIPtr FalseSuccessor = std::make_unique(); ++ Optional ProcedureType; // 1 - Leaf, 0 - NonLeaf, 2 - CallSelf ++ Optional LoopHeader; // 1 — loop header, 0 - not a loop header ++ Optional Direction; // 1 - Forward Branch, 0 - Backward Branch ++ ++ Optional NumOuterLoops; ++ Optional TotalLoops; ++ Optional MaximumLoopDepth; ++ Optional LoopDepth; ++ Optional LoopNumExitEdges; ++ Optional LoopNumExitBlocks; ++ Optional LoopNumExitingBlocks; ++ Optional LoopNumLatches; ++ Optional LoopNumBlocks; ++ Optional LoopNumBackEdges; ++ Optional NumLoads; ++ Optional NumStores; ++ ++ Optional LocalExitingBlock; ++ Optional LocalLatchBlock; ++ Optional LocalLoopHeader; ++ Optional Call; ++ ++ Optional NumCalls; ++ Optional NumCallsExit; ++ Optional NumCallsInvoke; ++ Optional NumIndirectCalls; ++ Optional NumTailCalls; ++ Optional NumSelfCalls; ++ ++ Optional NumBasicBlocks; ++ ++ Optional DeltaTaken; ++ ++ Optional OperandRAType; ++ Optional OperandRBType; ++ ++ Optional BasicBlockSize; ++ ++ Optional BranchOffset; ++ }; ++ ++ typedef std::unique_ptr BFIPtr; ++ std::vector BranchesInfoSet; ++ ++ /// getProcedureType - Determines which category the function falls into: ++ /// Leaf, Non-leaf or Calls-self. ++ int8_t getProcedureType(BinaryFunction &Function, BinaryContext &BC); ++ ++ /// addSuccessorInfo - Discovers feature information for the target successor ++ /// basic block, and inserts it into the static branch info container. ++ void addSuccessorInfo(DominatorAnalysis &DA, ++ DominatorAnalysis &PDA, BFIPtr const &BFI, ++ BinaryFunction &Function, BinaryContext &BC, ++ MCInst &Inst, BinaryBasicBlock &BB, bool Succ); ++ ++ /// extractFeatures - Extracts the feature information for each two-way branch ++ /// from the program. ++ void extractFeatures(BinaryFunction &Function, ++ BinaryContext &BC, ++ raw_ostream &Printer); ++ ++ /// dumpSuccessorFeatures - Dumps the feature information about the target ++ /// BasicBlock of either the taken or the fallthrough paths of a given branch. ++ void dumpSuccessorFeatures(raw_ostream &Printer, BBIPtr &Successor); ++ ++ /// dumpFeatures - Dumps the feature information about each two-way branch ++ /// from the program. ++ void dumpFeatures(raw_ostream &Printer, uint64_t FunctionAddress, ++ uint64_t FunctionFrequency); ++ ++ /// dumpProfileData - Dumps a limited version of the inout profile data ++ /// that contains only profile for conditional branches, unconditional ++ /// branches and terminators that aren't branches. ++ void dumpProfileData(BinaryFunction &Function, raw_ostream &Printer); ++ ++public: ++ explicit FeatureMiner(const cl::opt &PrintPass) ++ : BinaryFunctionPass(PrintPass) {} ++ ++ const char *getName() const override { return "feature-miner"; } ++ ++ void runOnFunctions(BinaryContext &BC) override; ++}; ++ ++} // namespace bolt ++} // namespace llvm ++ ++#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ */ +\ No newline at end of file +diff --git a/bolt/include/bolt/Passes/StaticBranchInfo.h b/bolt/include/bolt/Passes/StaticBranchInfo.h +new file mode 100644 +index 000000000..1713d3367 +--- /dev/null ++++ b/bolt/include/bolt/Passes/StaticBranchInfo.h +@@ -0,0 +1,116 @@ ++//===------ Passes/StaticBranchInfo.h -------------------------------------===// ++// ++// The LLVM Compiler Infrastructure ++// ++// This file is distributed under the University of Illinois Open Source ++// License. See LICENSE.TXT for details. ++// ++//===----------------------------------------------------------------------===// ++// ++// This is an auxiliary class to the feature miner, static branch probability ++// and frequency passes. This class is responsible for finding loop info (loop ++// back edges, loop exit edges and loop headers) of a function. It also finds ++// basic block info (if a block contains store and call instructions) and if a ++// basic block contains a call to the exit. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ ++#define LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ ++ ++// #include "BinaryContext.h" ++// #include "BinaryFunction.h" ++// #include "BinaryLoop.h" ++#include "bolt/Core/BinaryContext.h" ++#include "bolt/Core/BinaryFunction.h" ++#include "bolt/Core/BinaryLoop.h" ++ ++#include "llvm/MC/MCSymbol.h" ++// add new include ++#include ++ ++namespace llvm { ++namespace bolt { ++ ++class StaticBranchInfo { ++ ++public: ++ /// An edge indicates that a control flow may go from a basic block (source) ++ /// to an other one (destination), and this pair of basic blocks will be used ++ /// to index maps and retrieve content of sets. ++ typedef std::pair Edge; ++ ++private: ++ /// Holds the loop headers of a given function. ++ DenseSet LoopHeaders; ++ ++ /// Holds the loop backedges of a given function. ++ DenseSet BackEdges; ++ ++ /// Holds the loop exit edges of a given function. ++ DenseSet ExitEdges; ++ ++ /// Holds the basic blocks of a given function ++ /// that contains at least one call instructions. ++ DenseSet CallSet; ++ ++ /// Holds the basic blocks of a given function ++ /// that contains at least one store instructions. ++ DenseSet StoreSet; ++ ++ unsigned NumLoads; ++ unsigned NumStores; ++ ++public: ++ unsigned getNumLoads() { return NumLoads; } ++ ++ unsigned getNumStores() { return NumStores; } ++ ++ /// findLoopEdgesInfo - Finds all loop back edges, loop exit eges ++ /// and loop headers within the function. ++ void findLoopEdgesInfo(const BinaryLoopInfo &LoopsInfo); ++ ++ /// findBasicBlockInfo - Finds all call and store instructions within ++ /// the basic blocks of a given function. ++ void findBasicBlockInfo(const BinaryFunction &Function, BinaryContext &BC); ++ ++ /// isBackEdge - Checks if the edge is a loop back edge. ++ bool isBackEdge(const Edge &CFGEdge) const; ++ ++ /// isBackEdge - Checks if the edge is a loop back edge. ++ bool isBackEdge(const BinaryBasicBlock *SrcBB, ++ const BinaryBasicBlock *DstBB) const; ++ ++ /// isExitEdge - Checks if the edge is a loop exit edge. ++ bool isExitEdge(const BinaryLoop::Edge &CFGEdge) const; ++ ++ /// isExitEdge - Checks if the edge is a loop exit edge. ++ bool isExitEdge(const BinaryBasicBlock *SrcBB, ++ const BinaryBasicBlock *DstBB) const; ++ ++ /// isLoopHeader - Checks if the basic block is a loop header. ++ bool isLoopHeader(const BinaryBasicBlock *BB) const; ++ ++ /// hasCallInst - Checks if the basic block has a call instruction. ++ bool hasCallInst(const BinaryBasicBlock *BB) const; ++ ++ /// hasStoreInst - Checks if the basic block has a store instruction. ++ bool hasStoreInst(const BinaryBasicBlock *BB) const; ++ ++ /// callToExit - Checks if a basic block invokes exit function. ++ bool callToExit(BinaryBasicBlock *BB, BinaryContext &BC) const; ++ ++ /// countBackEdges - Compute the number of BB's successor that are back edges. ++ unsigned countBackEdges(BinaryBasicBlock *BB) const; ++ ++ /// countExitEdges - Compute the number of BB's successor that are exit edges. ++ unsigned countExitEdges(BinaryBasicBlock *BB) const; ++ ++ /// clear - Cleans up all the content from the data structs used. ++ void clear(); ++}; ++ ++} // namespace bolt ++} // namespace llvm ++ ++#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ */ +\ No newline at end of file +diff --git a/bolt/lib/Passes/CMakeLists.txt b/bolt/lib/Passes/CMakeLists.txt +index bb296263b..901ff614c 100644 +--- a/bolt/lib/Passes/CMakeLists.txt ++++ b/bolt/lib/Passes/CMakeLists.txt +@@ -12,6 +12,7 @@ add_llvm_library(LLVMBOLTPasses + DataflowAnalysis.cpp + DataflowInfoManager.cpp + ExtTSPReorderAlgorithm.cpp ++ FeatureMiner.cpp + FrameAnalysis.cpp + FrameOptimizer.cpp + HFSort.cpp +@@ -39,6 +40,7 @@ add_llvm_library(LLVMBOLTPasses + StackAvailableExpressions.cpp + StackPointerTracking.cpp + StackReachingUses.cpp ++ StaticBranchInfo.cpp + StokeInfo.cpp + TailDuplication.cpp + ThreeWayBranch.cpp +diff --git a/bolt/lib/Passes/FeatureMiner.cpp b/bolt/lib/Passes/FeatureMiner.cpp +new file mode 100644 +index 000000000..680222906 +--- /dev/null ++++ b/bolt/lib/Passes/FeatureMiner.cpp +@@ -0,0 +1,1067 @@ ++//===--- Passes/FeatureMiner.cpp ------------------------------------------===// ++// ++// The LLVM Compiler Infrastructure ++// ++// This file is distributed under the University of Illinois Open Source ++// License. See LICENSE.TXT for details. ++// ++//===----------------------------------------------------------------------===// ++// A very simple feature extractor based on Calder's paper ++// Evidence-based static branch prediction using machine learning ++// https://dl.acm.org/doi/10.1145/239912.239923 ++//===----------------------------------------------------------------------===// ++ ++// #include "Passes/FeatureMiner.h" ++// #include "Passes/DataflowInfoManager.h" ++// #include "llvm/Support/CommandLine.h" ++// #include "llvm/Support/Options.h" ++#include "bolt/Passes/FeatureMiner.h" ++#include "bolt/Passes/DataflowInfoManager.h" ++#include "llvm/Support/CommandLine.h" ++ ++// add new include ++#include "llvm/Support/FileSystem.h" ++ ++#undef DEBUG_TYPE ++#define DEBUG_TYPE "bolt-feature-miner" ++ ++using namespace llvm; ++using namespace bolt; ++ ++namespace opts { ++ ++extern cl::OptionCategory InferenceCategory; ++ ++cl::opt VespaUseDFS( ++ "vespa-dfs", ++ cl::desc("use DFS ordering when using -gen-features option"), ++ cl::init(false), ++ cl::ReallyHidden, ++ cl::ZeroOrMore, ++ cl::cat(InferenceCategory)); ++ ++cl::opt IncludeValidProfile( ++ "beetle-valid-profile-info", ++ cl::desc("include valid profile information."), ++ cl::init(false), ++ cl::ReallyHidden, ++ cl::ZeroOrMore, ++ cl::cat(InferenceCategory)); ++ ++} // namespace opts ++ ++namespace llvm { ++namespace bolt { ++ ++class BinaryFunction; ++ ++int8_t FeatureMiner::getProcedureType(BinaryFunction &Function, ++ BinaryContext &BC) { ++ int8_t ProcedureType = 1; ++ for (auto &BB : Function) { ++ for (auto &Inst : BB) { ++ if (BC.MIB->isCall(Inst)) { ++ ProcedureType = 0; // non-leaf type ++ if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { ++ const auto *Callee = BC.getFunctionForSymbol(CalleeSymbol); ++ if (Callee && ++ Callee->getFunctionNumber() == Function.getFunctionNumber()) { ++ return 2; // call self type ++ } ++ } ++ } ++ } ++ } ++ return ProcedureType; // leaf type ++} ++ ++void FeatureMiner::addSuccessorInfo(DominatorAnalysis &DA, ++ DominatorAnalysis &PDA, ++ BFIPtr const &BFI, BinaryFunction &Function, ++ BinaryContext &BC, MCInst &Inst, ++ BinaryBasicBlock &BB, bool SuccType) { ++ ++ BinaryBasicBlock *Successor = BB.getConditionalSuccessor(SuccType); ++ ++ if (!Successor) ++ return; ++ ++ unsigned NumLoads{0}; ++ unsigned NumStores{0}; ++ unsigned NumCallsExit{0}; ++ unsigned NumCalls{0}; ++ unsigned NumCallsInvoke{0}; ++ unsigned NumTailCalls{0}; ++ unsigned NumIndirectCalls{0}; ++ ++ for (auto &Inst : BB) { ++ if (BC.MIB->isLoad(Inst)) { ++ ++NumLoads; ++ } else if (BC.MIB->isStore(Inst)) { ++ ++NumStores; ++ } else if (BC.MIB->isCall(Inst)) { ++ ++NumCalls; ++ ++ if (BC.MIB->isIndirectCall(Inst)) ++ ++NumIndirectCalls; ++ ++ if (BC.MIB->isInvoke(Inst)) ++ ++NumCallsInvoke; ++ ++ if (BC.MIB->isTailCall(Inst)) ++ ++NumTailCalls; ++ ++ if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { ++ StringRef CalleeName = CalleeSymbol->getName(); ++ if (CalleeName == "__cxa_throw@PLT" || ++ CalleeName == "_Unwind_Resume@PLT" || ++ CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || ++ CalleeName == "abort@PLT") ++ ++NumCallsExit; ++ } ++ } ++ } ++ ++ BBIPtr SuccBBInfo = std::make_unique(); ++ ++ // Check if the successor basic block is a loop header and store it. ++ SuccBBInfo->LoopHeader = SBI->isLoopHeader(Successor); ++ ++ SuccBBInfo->BasicBlockSize = Successor->size(); ++ ++ // Check if the edge getting to the successor basic block is a loop ++ // exit edge and store it. ++ SuccBBInfo->Exit = SBI->isExitEdge(&BB, Successor); ++ ++ // Check if the edge getting to the successor basic block is a loop ++ // back edge and store it. ++ SuccBBInfo->Backedge = SBI->isBackEdge(&BB, Successor); ++ ++ MCInst *SuccInst = Successor->getTerminatorBefore(nullptr); ++ // Store information about the branch type ending sucessor basic block ++ SuccBBInfo->EndOpcode = (SuccInst && BC.MIA->isBranch(*SuccInst)) ++ ? SuccInst->getOpcode() ++ : 0; // 0 = NOTHING ++ if (SuccBBInfo->EndOpcode != 0) ++ SuccBBInfo->EndOpcodeStr = BC.MII->getName(SuccInst->getOpcode()); ++ else ++ SuccBBInfo->EndOpcodeStr = "NOTHING"; ++ ++ // Check if the successor basic block contains ++ // a procedure call and store it. ++ SuccBBInfo->Call = (NumCalls > 0) ? 1 // Contains a call instruction ++ : 0; // Does not contain a call instruction ++ ++ SuccBBInfo->NumStores = NumStores; ++ SuccBBInfo->NumLoads = NumLoads; ++ SuccBBInfo->NumCallsExit = NumCallsExit; ++ SuccBBInfo->NumCalls = NumCalls; ++ ++ SuccBBInfo->NumCallsInvoke = NumCallsInvoke; ++ SuccBBInfo->NumIndirectCalls = NumIndirectCalls; ++ SuccBBInfo->NumTailCalls = NumTailCalls; ++ ++ auto InstSucc = Successor->getLastNonPseudoInstr(); ++ if (InstSucc) { ++ // Check if the source basic block dominates its ++ // target basic block and store it. ++ SuccBBInfo->BranchDominates = (DA.doesADominateB(Inst, *InstSucc) == true) ++ ? 1 // Dominates ++ : 0; // Does not dominate ++ ++ // Check if the target basic block postdominates ++ // the source basic block and store it. ++ SuccBBInfo->BranchPostdominates = ++ (PDA.doesADominateB(*InstSucc, Inst) == true) ++ ? 1 // Postdominates ++ : 0; // Does not postdominate ++ } ++ ++ /// The follwoing information is used as an identifier only for ++ /// the purpose of matching the inferred probabilities with the branches ++ /// in the binary. ++ SuccBBInfo->FromFunName = Function.getPrintName(); ++ SuccBBInfo->FromBb = BB.getInputOffset(); ++ BinaryFunction *ToFun = Successor->getFunction(); ++ SuccBBInfo->ToFunName = ToFun->getPrintName(); ++ SuccBBInfo->ToBb = Successor->getInputOffset(); ++ ++ auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); ++ if (Offset) { ++ uint32_t TargetOffset = Successor->getInputOffset(); ++ uint32_t BranchOffset = Offset.get(); ++ BFI->BranchOffset = BranchOffset; ++ if (BranchOffset != UINT32_MAX && TargetOffset != UINT32_MAX) { ++ int64_t Delta = TargetOffset - BranchOffset; ++ BFI->DeltaTaken = std::abs(Delta); ++ } ++ } ++ ++ if (SuccType) { ++ BFI->TrueSuccessor = std::move(SuccBBInfo); ++ ++ // Check if the taken branch is a forward ++ // or a backwards branch and store it. ++ BFI->Direction = (Function.isForwardBranch(&BB, Successor) == true) ++ ? 1 // Forward branch ++ : 0; // Backwards branch ++ ++ auto TakenBranchInfo = BB.getTakenBranchInfo(); ++ BFI->Count = TakenBranchInfo.Count; ++ BFI->MissPredicted = TakenBranchInfo.MispredictedCount; ++ } else { ++ BFI->FalseSuccessor = std::move(SuccBBInfo); ++ ++ auto FallthroughBranchInfo = BB.getFallthroughBranchInfo(); ++ BFI->FallthroughCount = FallthroughBranchInfo.Count; ++ BFI->FallthroughMissPredicted = FallthroughBranchInfo.MispredictedCount; ++ } ++} ++ ++void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC, ++ raw_ostream &Printer) { ++ int8_t ProcedureType = getProcedureType(Function, BC); ++// auto Info = DataflowInfoManager(BC, Function, nullptr, nullptr); ++ auto Info = DataflowInfoManager(Function, nullptr, nullptr); ++ auto &DA = Info.getDominatorAnalysis(); ++ auto &PDA = Info.getPostDominatorAnalysis(); ++ const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); ++ bool Simple = Function.isSimple(); ++ ++// const auto &Order = opts::VespaUseDFS ? Function.dfs() : Function.getLayout(); ++ const auto &Order = Function.dfs(); ++ ++ for (auto *BBA : Order) { ++ ++ auto &BB = *BBA; ++ unsigned NumOuterLoops{0}; ++ unsigned TotalLoops{0}; ++ unsigned MaximumLoopDepth{0}; ++ unsigned LoopDepth{0}; ++ unsigned LoopNumExitEdges{0}; ++ unsigned LoopNumExitBlocks{0}; ++ unsigned LoopNumExitingBlocks{0}; ++ unsigned LoopNumLatches{0}; ++ unsigned LoopNumBlocks{0}; ++ unsigned LoopNumBackEdges{0}; ++ ++ bool LocalExitingBlock{false}; ++ bool LocalLatchBlock{false}; ++ bool LocalLoopHeader{false}; ++ ++ BinaryLoop *Loop = LoopsInfo.getLoopFor(&BB); ++ if (Loop) { ++ SmallVector ExitingBlocks; ++ Loop->getExitingBlocks(ExitingBlocks); ++ ++ SmallVector ExitBlocks; ++ Loop->getExitBlocks(ExitBlocks); ++ ++ SmallVector ExitEdges; ++ Loop->getExitEdges(ExitEdges); ++ ++ SmallVector Latches; ++ Loop->getLoopLatches(Latches); ++ ++ NumOuterLoops = LoopsInfo.OuterLoops; ++ TotalLoops = LoopsInfo.TotalLoops; ++ MaximumLoopDepth = LoopsInfo.MaximumDepth; ++ LoopDepth = Loop->getLoopDepth(); ++ LoopNumExitEdges = ExitEdges.size(); ++ LoopNumExitBlocks = ExitBlocks.size(); ++ LoopNumExitingBlocks = ExitingBlocks.size(); ++ LoopNumLatches = Latches.size(); ++ LoopNumBlocks = Loop->getNumBlocks(); ++ LoopNumBackEdges = Loop->getNumBackEdges(); ++ ++ LocalExitingBlock = Loop->isLoopExiting(&BB); ++ LocalLatchBlock = Loop->isLoopLatch(&BB); ++ LocalLoopHeader = ((Loop->getHeader() == (&BB)) ? 1 : 0); ++ } ++ ++ unsigned NumLoads{0}; ++ unsigned NumStores{0}; ++ unsigned NumCallsExit{0}; ++ unsigned NumCalls{0}; ++ unsigned NumCallsInvoke{0}; ++ unsigned NumTailCalls{0}; ++ unsigned NumIndirectCalls{0}; ++ unsigned NumSelfCalls{0}; ++ ++ for (auto &Inst : BB) { ++ if (BC.MIB->isLoad(Inst)) { ++ ++NumLoads; ++ } else if (BC.MIB->isStore(Inst)) { ++ ++NumStores; ++ } else if (BC.MIB->isCall(Inst)) { ++ ++NumCalls; ++ ++ if (BC.MIB->isIndirectCall(Inst)) ++ ++NumIndirectCalls; ++ ++ if (BC.MIB->isInvoke(Inst)) ++ ++NumCallsInvoke; ++ ++ if (BC.MIB->isTailCall(Inst)) ++ ++NumTailCalls; ++ ++ if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { ++ StringRef CalleeName = CalleeSymbol->getName(); ++ if (CalleeName == "__cxa_throw@PLT" || ++ CalleeName == "_Unwind_Resume@PLT" || ++ CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || ++ CalleeName == "abort@PLT") ++ ++NumCallsExit; ++ else if (CalleeName == Function.getPrintName()) { ++ ++NumSelfCalls; ++ } ++ } ++ } ++ } ++ ++ int Index = -2; ++ bool LoopHeader = SBI->isLoopHeader(&BB); ++ for (auto &Inst : BB) { ++ ++Index; ++ ++ if (!BC.MIA->isConditionalBranch(Inst)) ++ continue; ++ ++ BFIPtr BFI = std::make_unique(); ++ ++ BFI->Simple = Simple; ++ BFI->NumOuterLoops = NumOuterLoops; ++ BFI->TotalLoops = TotalLoops; ++ BFI->MaximumLoopDepth = MaximumLoopDepth; ++ BFI->LoopDepth = LoopDepth; ++ BFI->LoopNumExitEdges = LoopNumExitEdges; ++ BFI->LoopNumExitBlocks = LoopNumExitBlocks; ++ BFI->LoopNumExitingBlocks = LoopNumExitingBlocks; ++ BFI->LoopNumLatches = LoopNumLatches; ++ BFI->LoopNumBlocks = LoopNumBlocks; ++ BFI->LoopNumBackEdges = LoopNumBackEdges; ++ ++ BFI->LocalExitingBlock = LocalExitingBlock; ++ BFI->LocalLatchBlock = LocalLatchBlock; ++ BFI->LocalLoopHeader = LocalLoopHeader; ++ ++ BFI->Call = ((NumCalls > 0) ? 1 : 0); ++ BFI->NumCalls = NumCalls; ++ ++ BFI->BasicBlockSize = BB.size(); ++ BFI->NumBasicBlocks = Function.size(); ++ BFI->NumSelfCalls = NumSelfCalls; ++ ++ BFI->NumLoads = NumLoads; ++ BFI->NumStores = NumStores; ++ BFI->NumCallsExit = NumCallsExit; ++ ++ BFI->NumCallsInvoke = NumCallsInvoke; ++ BFI->NumIndirectCalls = NumIndirectCalls; ++ BFI->NumTailCalls = NumTailCalls; ++ ++ // Check if branch's basic block is a loop header and store it. ++ BFI->LoopHeader = LoopHeader; ++ ++ // Adding taken successor info. ++ addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, true); ++ // Adding fall through successor info. ++ addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, false); ++ ++ // Holds the branch opcode info. ++ BFI->Opcode = Inst.getOpcode(); ++ BFI->OpcodeStr = BC.MII->getName(Inst.getOpcode()); ++ ++ // Holds the branch's procedure type. ++ BFI->ProcedureType = ProcedureType; ++ ++ BFI->CmpOpcode = 0; ++ if (Index > -1) { ++ auto Cmp = BB.begin() + Index; ++ ++ if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { ++ // Holding the branch comparison opcode info. ++ BFI->CmpOpcode = (*Cmp).getOpcode(); ++ ++ BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); ++ ++ auto getOperandType = [&](const MCOperand &Operand) -> int32_t { ++ if (Operand.isReg()) ++ return 0; ++ else if (Operand.isImm()) ++ return 1; ++ // else if (Operand.isFPImm()) ++ else if (Operand.isSFPImm()) ++ return 2; ++ else if (Operand.isExpr()) ++ return 3; ++ else ++ return -1; ++ }; ++ ++ const auto InstInfo = BC.MII->get((*Cmp).getOpcode()); ++ unsigned NumDefs = InstInfo.getNumDefs(); ++ int32_t NumPrimeOperands = ++ MCPlus::getNumPrimeOperands(*Cmp) - NumDefs; ++ switch (NumPrimeOperands) { ++ case 6: { ++ int32_t RBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ int32_t RAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); ++ ++ if (RBType == 0 && RAType == 0) { ++ BFI->OperandRBType = RBType; ++ BFI->OperandRAType = RAType; ++ } else if (RBType == 0 && (RAType == 1 || RAType == 2)) { ++ RAType = getOperandType((*Cmp).getOperand(NumPrimeOperands - 1)); ++ ++ if (RAType != 1 && RAType != 2) { ++ RAType = -1; ++ } ++ ++ BFI->OperandRBType = RBType; ++ BFI->OperandRAType = RAType; ++ } else { ++ BFI->OperandRAType = -1; ++ BFI->OperandRBType = -1; ++ } ++ break; ++ } ++ case 2: ++ BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); ++ break; ++ case 3: ++ BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 2)); ++ break; ++ case 1: ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs)); ++ break; ++ default: ++ BFI->OperandRAType = -1; ++ BFI->OperandRBType = -1; ++ break; ++ } ++ ++ } else { ++ Index -= 1; ++ for (int Idx = Index; Idx > -1; Idx--) { ++ auto Cmp = BB.begin() + Idx; ++ if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { ++ // Holding the branch comparison opcode info. ++ BFI->CmpOpcode = (*Cmp).getOpcode(); ++ BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); ++ break; ++ } ++ } ++ } ++ } ++ ++ //======================================================================== ++ ++ auto &FalseSuccessor = BFI->FalseSuccessor; ++ auto &TrueSuccessor = BFI->TrueSuccessor; ++ ++ if (!FalseSuccessor && !TrueSuccessor) ++ continue; ++ ++ int64_t BranchOffset = ++ (BFI->BranchOffset.hasValue()) ++ ? static_cast(*(BFI->BranchOffset)) ++ : -1; ++ if(BranchOffset == -1) ++ continue; ++ ++ int16_t ProcedureType = (BFI->ProcedureType.hasValue()) ++ ? static_cast(*(BFI->ProcedureType)) ++ : -1; ++ ++ int16_t Direction = (BFI->Direction.hasValue()) ++ ? static_cast(*(BFI->Direction)) ++ : -1; ++ ++ int16_t LoopHeader = (BFI->LoopHeader.hasValue()) ++ ? static_cast(*(BFI->LoopHeader)) ++ : -1; ++ ++ int32_t Opcode = ++ (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; ++ ++ int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) ++ ? static_cast(*(BFI->CmpOpcode)) ++ : -1; ++ ++ int64_t Count = ++ (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; ++ ++ int64_t MissPredicted = (BFI->MissPredicted.hasValue()) ++ ? static_cast(*(BFI->MissPredicted)) ++ : -1; ++ ++ int64_t FallthroughCount = ++ (BFI->FallthroughCount.hasValue()) ++ ? static_cast(*(BFI->FallthroughCount)) ++ : -1; ++ ++ int64_t FallthroughMissPredicted = ++ (BFI->FallthroughMissPredicted.hasValue()) ++ ? static_cast(*(BFI->FallthroughMissPredicted)) ++ : -1; ++ ++ int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) ++ ? static_cast(*(BFI->NumOuterLoops)) ++ : -1; ++ int64_t TotalLoops = (BFI->TotalLoops.hasValue()) ++ ? static_cast(*(BFI->TotalLoops)) ++ : -1; ++ int64_t MaximumLoopDepth = ++ (BFI->MaximumLoopDepth.hasValue()) ++ ? static_cast(*(BFI->MaximumLoopDepth)) ++ : -1; ++ int64_t LoopDepth = (BFI->LoopDepth.hasValue()) ++ ? static_cast(*(BFI->LoopDepth)) ++ : -1; ++ int64_t LoopNumExitEdges = ++ (BFI->LoopNumExitEdges.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitEdges)) ++ : -1; ++ int64_t LoopNumExitBlocks = ++ (BFI->LoopNumExitBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitBlocks)) ++ : -1; ++ int64_t LoopNumExitingBlocks = ++ (BFI->LoopNumExitingBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitingBlocks)) ++ : -1; ++ int64_t LoopNumLatches = ++ (BFI->LoopNumLatches.hasValue()) ++ ? static_cast(*(BFI->LoopNumLatches)) ++ : -1; ++ int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumBlocks)) ++ : -1; ++ int64_t LoopNumBackEdges = ++ (BFI->LoopNumBackEdges.hasValue()) ++ ? static_cast(*(BFI->LoopNumBackEdges)) ++ : -1; ++ ++ int64_t LocalExitingBlock = ++ (BFI->LocalExitingBlock.hasValue()) ++ ? static_cast(*(BFI->LocalExitingBlock)) ++ : -1; ++ ++ int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) ++ ? static_cast(*(BFI->LocalLatchBlock)) ++ : -1; ++ ++ int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) ++ ? static_cast(*(BFI->LocalLoopHeader)) ++ : -1; ++ ++ int64_t Call = ++ (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; ++ ++ int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) ++ ? static_cast(*(BFI->DeltaTaken)) ++ : -1; ++ ++ int64_t NumLoads = (BFI->NumLoads.hasValue()) ++ ? static_cast(*(BFI->NumLoads)) ++ : -1; ++ ++ int64_t NumStores = (BFI->NumStores.hasValue()) ++ ? static_cast(*(BFI->NumStores)) ++ : -1; ++ ++ int64_t BasicBlockSize = ++ (BFI->BasicBlockSize.hasValue()) ++ ? static_cast(*(BFI->BasicBlockSize)) ++ : -1; ++ ++ int64_t NumBasicBlocks = ++ (BFI->NumBasicBlocks.hasValue()) ++ ? static_cast(*(BFI->NumBasicBlocks)) ++ : -1; ++ ++ int64_t NumCalls = (BFI->NumCalls.hasValue()) ++ ? static_cast(*(BFI->NumCalls)) ++ : -1; ++ ++ int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) ++ ? static_cast(*(BFI->NumSelfCalls)) ++ : -1; ++ ++ int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) ++ ? static_cast(*(BFI->NumCallsExit)) ++ : -1; ++ ++ int64_t OperandRAType = (BFI->OperandRAType.hasValue()) ++ ? static_cast(*(BFI->OperandRAType)) ++ : -1; ++ ++ int64_t OperandRBType = (BFI->OperandRBType.hasValue()) ++ ? static_cast(*(BFI->OperandRBType)) ++ : -1; ++ ++ int64_t NumCallsInvoke = ++ (BFI->NumCallsInvoke.hasValue()) ++ ? static_cast(*(BFI->NumCallsInvoke)) ++ : -1; ++ ++ int64_t NumIndirectCalls = ++ (BFI->NumIndirectCalls.hasValue()) ++ ? static_cast(*(BFI->NumIndirectCalls)) ++ : -1; ++ ++ int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) ++ ? static_cast(*(BFI->NumTailCalls)) ++ : -1; ++ ++ Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," ++ << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr ++ << "," << LoopHeader << "," << ProcedureType << "," << Count ++ << "," << MissPredicted << "," << FallthroughCount << "," ++ << FallthroughMissPredicted << "," << NumOuterLoops << "," ++ << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth ++ << "," << LoopDepth << "," << LoopNumExitEdges << "," ++ << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," ++ << LoopNumLatches << "," << LoopNumBlocks << "," ++ << LoopNumBackEdges << "," << LocalExitingBlock << "," ++ << LocalLatchBlock << "," << LocalLoopHeader << "," << Call << "," ++ << DeltaTaken << "," << NumLoads << "," << NumStores << "," ++ << NumCalls << "," << OperandRAType << "," << OperandRBType << "," ++ << BasicBlockSize << "," << NumBasicBlocks << "," ++ << NumCallsInvoke << "," << NumIndirectCalls << "," ++ << NumTailCalls << "," << NumSelfCalls; ++ ++ if (FalseSuccessor && TrueSuccessor) { ++ dumpSuccessorFeatures(Printer, TrueSuccessor); ++ dumpSuccessorFeatures(Printer, FalseSuccessor); ++ ++ FalseSuccessor.reset(); ++ TrueSuccessor.reset(); ++ } ++ BFI.reset(); ++ ++ std::string BranchOffsetStr = (BranchOffset == -1) ? "None" : Twine::utohexstr(BranchOffset).str(); ++ ++ uint64_t fun_exec = Function.getExecutionCount(); ++ fun_exec = (fun_exec != UINT64_MAX) ? fun_exec : 0; ++ Printer << "," << Twine::utohexstr(Function.getAddress()) << "," ++ << fun_exec << "," << Function.getFunctionNumber() << "," ++ << Function.getOneName() << "," << Function.getPrintName() ++ << "," << BranchOffsetStr ++ << "\n"; ++ ++ //======================================================================== ++ ++ // this->BranchesInfoSet.push_back(std::move(BFI)); ++ } ++ } ++} ++ ++void FeatureMiner::dumpSuccessorFeatures(raw_ostream &Printer, ++ BBIPtr &Successor) { ++ int16_t BranchDominates = ++ (Successor->BranchDominates.hasValue()) ++ ? static_cast(*(Successor->BranchDominates)) ++ : -1; ++ ++ int16_t BranchPostdominates = ++ (Successor->BranchPostdominates.hasValue()) ++ ? static_cast(*(Successor->BranchPostdominates)) ++ : -1; ++ ++ int16_t LoopHeader = (Successor->LoopHeader.hasValue()) ++ ? static_cast(*(Successor->LoopHeader)) ++ : -1; ++ ++ int16_t Backedge = (Successor->Backedge.hasValue()) ++ ? static_cast(*(Successor->Backedge)) ++ : -1; ++ ++ int16_t Exit = ++ (Successor->Exit.hasValue()) ? static_cast(*(Successor->Exit)) : -1; ++ ++ int16_t Call = ++ (Successor->Call.hasValue()) ? static_cast(*(Successor->Call)) : -1; ++ ++ int32_t EndOpcode = (Successor->EndOpcode.hasValue()) ++ ? static_cast(*(Successor->EndOpcode)) ++ : -1; ++ ++ int64_t NumLoads = (Successor->NumLoads.hasValue()) ++ ? static_cast(*(Successor->NumLoads)) ++ : -1; ++ ++ int64_t NumStores = (Successor->NumStores.hasValue()) ++ ? static_cast(*(Successor->NumStores)) ++ : -1; ++ ++ int64_t BasicBlockSize = ++ (Successor->BasicBlockSize.hasValue()) ++ ? static_cast(*(Successor->BasicBlockSize)) ++ : -1; ++ ++ int64_t NumCalls = (Successor->NumCalls.hasValue()) ++ ? static_cast(*(Successor->NumCalls)) ++ : -1; ++ ++ int64_t NumCallsExit = (Successor->NumCallsExit.hasValue()) ++ ? static_cast(*(Successor->NumCallsExit)) ++ : -1; ++ ++ int64_t NumCallsInvoke = ++ (Successor->NumCallsInvoke.hasValue()) ++ ? static_cast(*(Successor->NumCallsInvoke)) ++ : -1; ++ ++ int64_t NumIndirectCalls = ++ (Successor->NumIndirectCalls.hasValue()) ++ ? static_cast(*(Successor->NumIndirectCalls)) ++ : -1; ++ ++ int64_t NumTailCalls = (Successor->NumTailCalls.hasValue()) ++ ? static_cast(*(Successor->NumTailCalls)) ++ : -1; ++ ++ Printer << "," << BranchDominates << "," << BranchPostdominates << "," ++ << EndOpcode << "," << Successor->EndOpcodeStr << "," << LoopHeader ++ << "," << Backedge << "," << Exit << "," << Call << "," ++ << Successor->FromFunName << "," ++ << Twine::utohexstr(Successor->FromBb) << "," << Successor->ToFunName ++ << "," << Twine::utohexstr(Successor->ToBb) << "," << NumLoads << "," ++ << NumStores << "," << BasicBlockSize << "," << NumCalls << "," ++ << NumCallsExit << "," << NumIndirectCalls << "," << NumCallsInvoke ++ << "," << NumTailCalls; ++} ++ ++void FeatureMiner::dumpFeatures(raw_ostream &Printer, uint64_t FunctionAddress, ++ uint64_t FunctionFrequency) { ++ ++ for (auto const &BFI : BranchesInfoSet) { ++ auto &FalseSuccessor = BFI->FalseSuccessor; ++ auto &TrueSuccessor = BFI->TrueSuccessor; ++ ++ if (!FalseSuccessor && !TrueSuccessor) ++ continue; ++ ++ int16_t ProcedureType = (BFI->ProcedureType.hasValue()) ++ ? static_cast(*(BFI->ProcedureType)) ++ : -1; ++ ++ int16_t Direction = ++ (BFI->Direction.hasValue()) ? static_cast(*(BFI->Direction)) : -1; ++ ++ int16_t LoopHeader = (BFI->LoopHeader.hasValue()) ++ ? static_cast(*(BFI->LoopHeader)) ++ : -1; ++ ++ int32_t Opcode = ++ (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; ++ ++ int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) ++ ? static_cast(*(BFI->CmpOpcode)) ++ : -1; ++ ++ int64_t Count = ++ (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; ++ ++ int64_t MissPredicted = (BFI->MissPredicted.hasValue()) ++ ? static_cast(*(BFI->MissPredicted)) ++ : -1; ++ ++ int64_t FallthroughCount = ++ (BFI->FallthroughCount.hasValue()) ++ ? static_cast(*(BFI->FallthroughCount)) ++ : -1; ++ ++ int64_t FallthroughMissPredicted = ++ (BFI->FallthroughMissPredicted.hasValue()) ++ ? static_cast(*(BFI->FallthroughMissPredicted)) ++ : -1; ++ ++ int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) ++ ? static_cast(*(BFI->NumOuterLoops)) ++ : -1; ++ int64_t TotalLoops = (BFI->TotalLoops.hasValue()) ++ ? static_cast(*(BFI->TotalLoops)) ++ : -1; ++ int64_t MaximumLoopDepth = ++ (BFI->MaximumLoopDepth.hasValue()) ++ ? static_cast(*(BFI->MaximumLoopDepth)) ++ : -1; ++ int64_t LoopDepth = (BFI->LoopDepth.hasValue()) ++ ? static_cast(*(BFI->LoopDepth)) ++ : -1; ++ int64_t LoopNumExitEdges = ++ (BFI->LoopNumExitEdges.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitEdges)) ++ : -1; ++ int64_t LoopNumExitBlocks = ++ (BFI->LoopNumExitBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitBlocks)) ++ : -1; ++ int64_t LoopNumExitingBlocks = ++ (BFI->LoopNumExitingBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumExitingBlocks)) ++ : -1; ++ int64_t LoopNumLatches = (BFI->LoopNumLatches.hasValue()) ++ ? static_cast(*(BFI->LoopNumLatches)) ++ : -1; ++ int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumBlocks)) ++ : -1; ++ int64_t LoopNumBackEdges = ++ (BFI->LoopNumBackEdges.hasValue()) ++ ? static_cast(*(BFI->LoopNumBackEdges)) ++ : -1; ++ ++ int64_t LocalExitingBlock = ++ (BFI->LocalExitingBlock.hasValue()) ++ ? static_cast(*(BFI->LocalExitingBlock)) ++ : -1; ++ ++ int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) ++ ? static_cast(*(BFI->LocalLatchBlock)) ++ : -1; ++ ++ int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) ++ ? static_cast(*(BFI->LocalLoopHeader)) ++ : -1; ++ ++ int64_t Call = ++ (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; ++ ++ int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) ++ ? static_cast(*(BFI->DeltaTaken)) ++ : -1; ++ ++ int64_t NumLoads = (BFI->NumLoads.hasValue()) ++ ? static_cast(*(BFI->NumLoads)) ++ : -1; ++ ++ int64_t NumStores = (BFI->NumStores.hasValue()) ++ ? static_cast(*(BFI->NumStores)) ++ : -1; ++ ++ int64_t BasicBlockSize = (BFI->BasicBlockSize.hasValue()) ++ ? static_cast(*(BFI->BasicBlockSize)) ++ : -1; ++ ++ int64_t BranchOffset = (BFI->BranchOffset.hasValue()) ++ ? static_cast(*(BFI->BranchOffset)): -1; ++ ++ int64_t NumBasicBlocks = (BFI->NumBasicBlocks.hasValue()) ++ ? static_cast(*(BFI->NumBasicBlocks)) ++ : -1; ++ ++ int64_t NumCalls = (BFI->NumCalls.hasValue()) ++ ? static_cast(*(BFI->NumCalls)) ++ : -1; ++ ++ int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) ++ ? static_cast(*(BFI->NumSelfCalls)) ++ : -1; ++ ++ int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) ++ ? static_cast(*(BFI->NumCallsExit)) ++ : -1; ++ ++ int64_t OperandRAType = (BFI->OperandRAType.hasValue()) ++ ? static_cast(*(BFI->OperandRAType)) ++ : -1; ++ ++ int64_t OperandRBType = (BFI->OperandRBType.hasValue()) ++ ? static_cast(*(BFI->OperandRBType)) ++ : -1; ++ ++ int64_t NumCallsInvoke = (BFI->NumCallsInvoke.hasValue()) ++ ? static_cast(*(BFI->NumCallsInvoke)) ++ : -1; ++ ++ int64_t NumIndirectCalls = ++ (BFI->NumIndirectCalls.hasValue()) ++ ? static_cast(*(BFI->NumIndirectCalls)) ++ : -1; ++ ++ int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) ++ ? static_cast(*(BFI->NumTailCalls)) ++ : -1; ++ ++ Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," ++ << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr << "," ++ << LoopHeader << "," << ProcedureType << "," << Count << "," ++ << MissPredicted << "," << FallthroughCount << "," ++ << FallthroughMissPredicted << "," << NumOuterLoops << "," ++ << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth ++ << "," << LoopDepth << "," << LoopNumExitEdges << "," ++ << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," ++ << LoopNumLatches << "," << LoopNumBlocks << "," << LoopNumBackEdges ++ << "," << LocalExitingBlock << "," << LocalLatchBlock << "," ++ << LocalLoopHeader << "," << Call << "," << DeltaTaken << "," ++ << NumLoads << "," << NumStores << "," << NumCalls << "," ++ << OperandRAType << "," << OperandRBType << "," << BasicBlockSize ++ << "," << NumBasicBlocks << "," << NumCallsInvoke << "," ++ << NumIndirectCalls << "," << NumTailCalls << "," << NumSelfCalls; ++ ++ if (FalseSuccessor && TrueSuccessor) { ++ dumpSuccessorFeatures(Printer, TrueSuccessor); ++ dumpSuccessorFeatures(Printer, FalseSuccessor); ++ } ++ ++ Printer << "," << Twine::utohexstr(FunctionAddress) << "," ++ << FunctionFrequency << "\n"; ++ } ++ BranchesInfoSet.clear(); ++} ++ ++void FeatureMiner::runOnFunctions(BinaryContext &BC) { ++ auto FileName = "features_new.csv"; ++ outs() << "BOLT-INFO: Starting feature miner pass\n"; ++ ++ std::error_code EC; ++// raw_fd_ostream Printer(FileName, EC, sys::fs::F_None); ++ raw_fd_ostream Printer(FileName, EC, sys::fs::OF_None); ++ ++ if (EC) { ++ errs() << "BOLT-WARNING: " << EC.message() << ", unable to open " ++ << FileName << " for output.\n"; ++ return; ++ } ++ ++ auto FILENAME = "profile_data_regular.fdata"; ++// raw_fd_ostream Printer2(FILENAME, EC, sys::fs::F_None); ++ raw_fd_ostream Printer2(FILENAME, EC, sys::fs::OF_None); ++ ++ if (EC) { ++ dbgs() << "BOLT-WARNING: " << EC.message() << ", unable to open" ++ << " " << FILENAME << " for output.\n"; ++ return; ++ } ++ ++ // CSV file header ++ Printer << "FUN_TYPE,OPCODE,OPCODE_STR,DIRECTION,CMP_OPCODE,CMP_OPCODE_STR," ++ "LOOP_HEADER,PROCEDURE_TYPE," ++ "COUNT_TAKEN,MISS_TAKEN,COUNT_NOT_TAKEN,MISS_NOT_TAKEN," ++ "NUM_OUTER_LOOPS,NUM_CALLS_EXIT,TOTAL_LOOPS,MAXIMUM_LOOP_DEPTH," ++ "LOOP_DEPTH,LOOP_NUM_EXIT_EDGES,LOOP_NUM_EXIT_BLOCKS," ++ "LOOP_NUM_EXITING_BLOCKS,LOOP_NUM_LATCHES,LOOP_NUM_BLOCKS," ++ "LOOP_NUM_BAKCEDGES,LOCAL_EXITING_BLOCK,LOCAL_LATCH_BLOCK," ++ "LOCAL_LOOP_HEADER,CALL,DELTA_TAKEN,NUM_LOADS,NUM_STORES," ++ "NUM_CALLS,OPERAND_RA_TYPE,OPERAND_RB_TYPE,BASIC_BLOCK_SIZE," ++ "NUM_BASIC_BLOCKS,NUM_CALLS_INVOKE,NUM_INDIRECT_CALLS," ++ "NUM_TAIL_CALLS,NUM_SELF_CALLS,TS_DOMINATES,TS_POSTDOMINATES," ++ "TS_END_OPCODE,TS_END_OPCODE_STR,TS_LOOP_HEADER,TS_BACKEDGE,TS_" ++ "EXIT,TS_CALL," ++ "TS_FROM_FUN_NAME,TS_FROM_BB,TS_TO_FUN_NAME,TS_TO_BB,TS_NUM_LOADS," ++ "TS_NUM_STORES,TS_BASIC_BLOCK_SIZE,TS_NUM_CALLS,TS_NUM_CALLS_EXIT," ++ "TS_NUM_INDIRECT_CALL,TS_NUM_CALLS_INVOKE,TS_NUM_TAIL_CALLS," ++ "FS_DOMINATES,FS_POSTDOMINATES,FS_END_OPCODE,FS_END_OPCODE_STR,FS_" ++ "LOOP_HEADER," ++ "FS_BACKEDGE,FS_EXIT,FS_CALL,FS_FROM_FUN_NAME,FS_FROM_BB," ++ "FS_TO_FUN_NAME,FS_TO_BB,FS_NUM_LOADS,FS_NUM_STORES," ++ "FS_BASIC_BLOCK_SIZE,FS_NUM_CALLS,FS_NUM_CALLS_EXIT," ++ "FS_NUM_INDIRECT_CALL,FS_NUM_CALLS_INVOKE,FS_NUM_TAIL_CALLS," ++ "FUN_ENTRY_ADDRESS,FUN_ENTRY_FREQUENCY" ++ ",FUN_UNIQUE_NUMBER,FUN_ONE_NAME,FUN_PRINT_NAME," ++ "BRANCH_ADDRESS\n"; ++ ++ auto &BFs = BC.getBinaryFunctions(); ++ SBI = std::make_unique(); ++ for (auto &BFI : BFs) { ++ BinaryFunction &Function = BFI.second; ++ ++ if (Function.empty() || (Function.hasValidProfile() && opts::IncludeValidProfile)) ++ continue; ++ ++ if (!Function.isLoopFree()) { ++ const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); ++ SBI->findLoopEdgesInfo(LoopsInfo); ++ } ++ extractFeatures(Function, BC, Printer); ++ ++ SBI->clear(); ++ ++ // dumpProfileData(Function, Printer2); ++ } ++ ++ outs() << "BOLT-INFO: Dumping two-way conditional branches' features" ++ << " at " << FileName << "\n"; ++} ++ ++/*void FeatureMiner::dumpProfileData(BinaryFunction &Function, ++ raw_ostream &Printer) { ++ ++ BinaryContext &BC = Function.getBinaryContext(); ++ ++ std::string FromFunName = Function.getPrintName(); ++ for (auto &BB : Function) { ++ auto LastInst = BB.getLastNonPseudoInstr(); ++ ++ for (auto &Inst : BB) { ++ if (!BC.MIB->isCall(Inst) && !BC.MIB->isBranch(Inst) && ++ LastInst != (&Inst)) ++ continue; ++ ++ auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); ++ ++ if (!Offset) ++ continue; ++ ++ uint64_t TakenFreqEdge = 0; ++ auto FromBb = Offset.get(); ++ std::string ToFunName; ++ uint32_t ToBb; ++ ++ if (BC.MIB->isCall(Inst)) { ++ auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst); ++ if (!CalleeSymbol) ++ continue; ++ ++ ToFunName = CalleeSymbol->getName(); ++ ToBb = 0; ++ ++ if (BC.MIB->getConditionalTailCall(Inst)) { ++ ++ if (BC.MIB->hasAnnotation(Inst, "CTCTakenCount")) { ++ auto CountAnnt = ++ BC.MIB->tryGetAnnotationAs(Inst, "CTCTakenCount"); ++ if (CountAnnt) { ++ TakenFreqEdge = (*CountAnnt); ++ } ++ } ++ } else { ++ if (BC.MIB->hasAnnotation(Inst, "Count")) { ++ auto CountAnnt = ++ BC.MIB->tryGetAnnotationAs(Inst, "Count"); ++ if (CountAnnt) { ++ TakenFreqEdge = (*CountAnnt); ++ } ++ } ++ } ++ ++ if (TakenFreqEdge > 0) ++ Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) ++ << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) << " " ++ << 0 << " " << TakenFreqEdge << "\n"; ++ } else { ++ for (BinaryBasicBlock *SuccBB : BB.successors()) { ++ TakenFreqEdge = BB.getBranchInfo(*SuccBB).Count; ++ BinaryFunction *ToFun = SuccBB->getFunction(); ++ ToFunName = ToFun->getPrintName(); ++ ToBb = SuccBB->getInputOffset(); ++ ++ if (TakenFreqEdge > 0) ++ Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) ++ << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) ++ << " " << 0 << " " << TakenFreqEdge << "\n"; ++ } ++ } ++ } ++ } ++} ++*/ ++ ++} // namespace bolt ++} // namespace llvm +\ No newline at end of file +diff --git a/bolt/lib/Passes/StaticBranchInfo.cpp b/bolt/lib/Passes/StaticBranchInfo.cpp +new file mode 100644 +index 000000000..13426b397 +--- /dev/null ++++ b/bolt/lib/Passes/StaticBranchInfo.cpp +@@ -0,0 +1,162 @@ ++//===------ Passes/StaticBranchInfo.cpp -----------------------------------===// ++// ++// The LLVM Compiler Infrastructure ++// ++// This file is distributed under the University of Illinois Open Source ++// License. See LICENSE.TXT for details. ++// ++//===----------------------------------------------------------------------===// ++// ++// This is an auxiliary class to the feature miner, static branch probability ++// and frequency passes. This class is responsible for finding loop info (loop ++// back edges, loop exit edges and loop headers) of a function. It also finds ++// basic block info (if a block contains store and call instructions) and if a ++// basic block contains a call to the exit. ++// ++//===----------------------------------------------------------------------===// ++ ++// #include "Passes/StaticBranchInfo.h" ++// #include "BinaryBasicBlock.h" ++#include "bolt/Passes/StaticBranchInfo.h" ++#include "bolt/Core/BinaryBasicBlock.h" ++ ++namespace llvm { ++namespace bolt { ++ ++void StaticBranchInfo::findLoopEdgesInfo(const BinaryLoopInfo &LoopsInfo) { ++ // Traverse discovered loops ++ std::stack Loops; ++ for (BinaryLoop *BL : LoopsInfo) ++ Loops.push(BL); ++ ++ while (!Loops.empty()) { ++ BinaryLoop *Loop = Loops.top(); ++ Loops.pop(); ++ BinaryBasicBlock *LoopHeader = Loop->getHeader(); ++ LoopHeaders.insert(LoopHeader); ++ ++ // Add nested loops in the stack. ++ for (BinaryLoop::iterator I = Loop->begin(), E = Loop->end(); I != E; ++I) { ++ Loops.push(*I); ++ } ++ ++ SmallVector Latches; ++ Loop->getLoopLatches(Latches); ++ ++ // Find back edges. ++ for (BinaryBasicBlock *Latch : Latches) { ++ for (BinaryBasicBlock *Succ : Latch->successors()) { ++ if (Succ == LoopHeader) { ++ Edge CFGEdge = std::make_pair(Latch->getLabel(), Succ->getLabel()); ++ BackEdges.insert(CFGEdge); ++ } ++ } ++ } ++ ++ // Find exit edges. ++ SmallVector AuxExitEdges; ++ Loop->getExitEdges(AuxExitEdges); ++ for (BinaryLoop::Edge &Exit : AuxExitEdges) { ++ ExitEdges.insert(Exit); ++ } ++ } ++} ++ ++void StaticBranchInfo::findBasicBlockInfo(const BinaryFunction &Function, ++ BinaryContext &BC) { ++ for (auto &BB : Function) { ++ for (auto &Inst : BB) { ++ if (BC.MIB->isCall(Inst)) ++ CallSet.insert(&BB); ++ else if (BC.MIB->isStore(Inst)) ++ StoreSet.insert(&BB); ++ } ++ } ++} ++ ++bool StaticBranchInfo::isBackEdge(const Edge &CFGEdge) const { ++ return BackEdges.count(CFGEdge); ++} ++ ++bool StaticBranchInfo::isBackEdge(const BinaryBasicBlock *SrcBB, ++ const BinaryBasicBlock *DstBB) const { ++ const Edge CFGEdge = std::make_pair(SrcBB->getLabel(), DstBB->getLabel()); ++ return isBackEdge(CFGEdge); ++} ++ ++bool StaticBranchInfo::isExitEdge(const BinaryLoop::Edge &CFGEdge) const { ++ return ExitEdges.count(CFGEdge); ++} ++ ++bool StaticBranchInfo::isExitEdge(const BinaryBasicBlock *SrcBB, ++ const BinaryBasicBlock *DstBB) const { ++// const BinaryLoop::Edge CFGEdge = std::make_pair(SrcBB, DstBB); ++ const BinaryLoop::Edge CFGEdge = std::make_pair(const_cast(SrcBB), const_cast(DstBB)); ++ return isExitEdge(CFGEdge); ++} ++ ++bool StaticBranchInfo::isLoopHeader(const BinaryBasicBlock *BB) const { ++ return LoopHeaders.count(BB); ++} ++ ++bool StaticBranchInfo::hasCallInst(const BinaryBasicBlock *BB) const { ++ return CallSet.count(BB); ++} ++ ++bool StaticBranchInfo::hasStoreInst(const BinaryBasicBlock *BB) const { ++ return StoreSet.count(BB); ++} ++ ++bool StaticBranchInfo::callToExit(BinaryBasicBlock *BB, ++ BinaryContext &BC) const { ++ auto &currBB = *BB; ++ for (auto &Inst : currBB) { ++ if (BC.MIB->isCall(Inst)) { ++ if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { ++ StringRef CalleeName = CalleeSymbol->getName(); ++ if (CalleeName == "__cxa_throw@PLT" || ++ CalleeName == "_Unwind_Resume@PLT" || ++ CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || ++ CalleeName == "abort@PLT") ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++unsigned StaticBranchInfo::countBackEdges(BinaryBasicBlock *BB) const { ++ unsigned CountEdges = 0; ++ ++ for (BinaryBasicBlock *SuccBB : BB->successors()) { ++ const Edge CFGEdge = std::make_pair(BB->getLabel(), SuccBB->getLabel()); ++ if (BackEdges.count(CFGEdge)) ++ ++CountEdges; ++ } ++ ++ return CountEdges; ++} ++ ++unsigned StaticBranchInfo::countExitEdges(BinaryBasicBlock *BB) const { ++ unsigned CountEdges = 0; ++ ++ for (BinaryBasicBlock *SuccBB : BB->successors()) { ++ const BinaryLoop::Edge CFGEdge = std::make_pair(BB, SuccBB); ++ if (ExitEdges.count(CFGEdge)) ++ ++CountEdges; ++ } ++ ++ return CountEdges; ++} ++ ++void StaticBranchInfo::clear() { ++ LoopHeaders.clear(); ++ BackEdges.clear(); ++ ExitEdges.clear(); ++ CallSet.clear(); ++ StoreSet.clear(); ++} ++ ++} // namespace bolt ++} // namespace llvm +\ No newline at end of file +-- +2.33.0 + diff --git a/0005-Add-block-correction-optimization.patch b/0005-Add-block-correction-optimization.patch new file mode 100644 index 0000000..32a861e --- /dev/null +++ b/0005-Add-block-correction-optimization.patch @@ -0,0 +1,2325 @@ +From f2a5570a0821739a9e674e989ad1f734cc159570 Mon Sep 17 00:00:00 2001 +From: h00502206 +Date: Tue, 4 Jun 2024 21:52:50 +0800 +Subject: [PATCH] Add block correction optimization + +--- + bolt/CMakeLists.txt | 3 + + .../bolt/include/bolt/Core/BinaryBasicBlock.h | 34 +- + .../bolt/Core/BinaryBasicBlockFeature.h | 287 +++++ + .../bolt/include/bolt/Passes/FeatureMiner.h | 36 +- + .../include/bolt/Passes/StaticBranchInfo.h | 9 +- + .../bolt/include/bolt/Profile/DataReader.h | 74 +- + .../bolt/lib/Core/BinaryBasicBlockFeature.cpp | 25 + + bolt/lib/Core/CMakeLists.txt | 1 + + bolt/lib/Passes/CMakeLists.txt | 2 +- + bolt/lib/Passes/FeatureMiner.cpp | 1137 +++++------------ + .../bolt/lib/Passes/StaticBranchInfo.cpp | 6 +- + bolt/lib/Profile/CMakeLists.txt | 1 + + bolt/lib/Profile/DataReader.cpp | 126 +- + .../bolt/lib/Rewrite/RewriteInstance.cpp | 7 + + 14 files changed, 885 insertions(+), 863 deletions(-) + create mode 100644 bolt/include/bolt/Core/BinaryBasicBlockFeature.h + create mode 100644 bolt/lib/Core/BinaryBasicBlockFeature.cpp + +diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt +index 3de930496..09dabcdee 100644 +--- a/bolt/CMakeLists.txt ++++ b/bolt/CMakeLists.txt +@@ -9,6 +9,9 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|aarch64") + set(BOLT_ENABLE_RUNTIME ON) + endif() + ++ ++add_compile_options(-fexceptions) ++ + set(BOLT_CLANG_EXE "" CACHE FILEPATH "Path to clang executable for the target \ + architecture for use in BOLT tests") + set(BOLT_LLD_EXE "" CACHE FILEPATH "Path to lld executable for the target \ +diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h +index 0a82f467e..bfeb174de 100644 +--- a/bolt/include/bolt/Core/BinaryBasicBlock.h ++++ b/bolt/include/bolt/Core/BinaryBasicBlock.h +@@ -15,6 +15,7 @@ + #ifndef BOLT_CORE_BINARY_BASIC_BLOCK_H + #define BOLT_CORE_BINARY_BASIC_BLOCK_H + ++#include "bolt/Core/BinaryBasicBlockFeature.h" + #include "bolt/Core/FunctionLayout.h" + #include "bolt/Core/MCPlus.h" + #include "llvm/ADT/GraphTraits.h" +@@ -25,6 +26,7 @@ + #include "llvm/Support/raw_ostream.h" + #include + #include ++#include + + namespace llvm { + class MCCodeEmitter; +@@ -62,6 +64,13 @@ public: + + using BranchInfoType = SmallVector; + ++ std::set ChildrenSet; ++ ++ std::set ParentSet; ++ ++ BinaryBasicBlockFeature block_features; ++ ++ + private: + /// Vector of all instructions in the block. + InstructionListType Instructions; +@@ -384,9 +393,11 @@ public: + /// corresponding to a jump condition which could be true or false. + /// Return nullptr if the basic block does not have a conditional jump. + BinaryBasicBlock *getConditionalSuccessor(bool Condition) { +- if (succ_size() != 2) +- return nullptr; +- return Successors[Condition == true ? 0 : 1]; ++ if (succ_size() == 2) ++ return Successors[Condition == true ? 0 : 1]; ++ if (succ_size() == 1) ++ return Successors[0]; ++ return nullptr; + } + + const BinaryBasicBlock *getConditionalSuccessor(bool Condition) const { +@@ -407,6 +418,11 @@ public: + return const_cast(this)->getFallthrough(); + } + ++ /// Return branch info corresponding to only branch. ++ const BinaryBranchInfo &getOnlyBranchInfo() const { ++ return BranchInfo[0]; ++ }; ++ + /// Return branch info corresponding to a taken branch. + const BinaryBranchInfo &getTakenBranchInfo() const { + assert(BranchInfo.size() == 2 && +@@ -807,6 +823,16 @@ public: + OutputAddressRange.second = Address; + } + ++ /// Sets features of this BB. ++ void setFeatures(BinaryBasicBlockFeature BBF) { ++ block_features = BBF; ++ } ++ ++ /// Gets numberic features of this BB. ++ BinaryBasicBlockFeature getFeatures() { ++ return block_features; ++ } ++ + /// Gets the memory address range of this BB in the input binary. + std::pair getInputAddressRange() const { + return InputRange; +@@ -974,7 +1000,7 @@ private: + #if defined(LLVM_ON_UNIX) + /// Keep the size of the BinaryBasicBlock within a reasonable size class + /// (jemalloc bucket) on Linux +-static_assert(sizeof(BinaryBasicBlock) <= 256, ""); ++static_assert(sizeof(BinaryBasicBlock) <= 2048, ""); + #endif + + bool operator<(const BinaryBasicBlock &LHS, const BinaryBasicBlock &RHS); +diff --git a/bolt/include/bolt/Core/BinaryBasicBlockFeature.h b/bolt/include/bolt/Core/BinaryBasicBlockFeature.h +new file mode 100644 +index 000000000..0f123bd49 +--- /dev/null ++++ b/bolt/include/bolt/Core/BinaryBasicBlockFeature.h +@@ -0,0 +1,287 @@ ++//===- bolt/Core/BinaryBasicBlockFeature.h - Low-level basic block -----*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// Features of BinaryBasicBlock ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef BOLT_CORE_BINARY_BASIC_BLOCK_FEATURE_H ++#define BOLT_CORE_BINARY_BASIC_BLOCK_FEATURE_H ++ ++#include "bolt/Core/FunctionLayout.h" ++#include "bolt/Core/MCPlus.h" ++#include "llvm/ADT/GraphTraits.h" ++#include "llvm/ADT/StringRef.h" ++#include "llvm/MC/MCInst.h" ++#include "llvm/MC/MCSymbol.h" ++#include "llvm/Support/ErrorOr.h" ++#include "llvm/Support/raw_ostream.h" ++#include ++#include ++ ++namespace llvm { ++ ++namespace bolt { ++ ++class BinaryBasicBlockFeature { ++ ++public: ++ ++ int32_t Opcode; ++ ++ int16_t Direction; ++ ++ int32_t CmpOpcode; ++ ++ int16_t LoopHeader; ++ ++ int16_t ProcedureType; ++ ++ int64_t Count; ++ ++ int64_t FallthroughCount; ++ ++ int64_t TotalLoops; ++ ++ int64_t LoopDepth; ++ ++ int64_t LoopNumBlocks; ++ ++ int64_t LocalExitingBlock; ++ ++ int64_t LocalLatchBlock; ++ ++ int64_t LocalLoopHeader; ++ ++ int64_t Call; ++ ++ int64_t DeltaTaken; ++ ++ int64_t NumLoads; ++ ++ int64_t NumCalls; ++ ++ int64_t OperandRAType; ++ ++ int64_t OperandRBType; ++ ++ int64_t BasicBlockSize; ++ ++ int64_t NumBasicBlocks; ++ ++ int64_t HasIndirectCalls; ++ ++ std::vector EndOpcode_vec; ++ ++ std::vector LoopHeader_vec; ++ ++ std::vector Backedge_vec; ++ ++ std::vector Exit_vec; ++ ++ std::vector Call_vec; ++ ++ std::vector BasicBlockSize_vec; ++ ++ std::vector InferenceFeatures; ++ ++ uint64_t FuncExec; ++ ++ int32_t ParentChildNum; ++ ++ int32_t ParentCount; ++ ++ int32_t ChildParentNum; ++ ++ int32_t ChildCount; ++ ++public: ++ ++ void setOpcode(const int32_t &BlockOpcode) { ++ Opcode = BlockOpcode; ++ } ++ ++ void setDirection(const int16_t &BlockDirection) { ++ Direction = BlockDirection; ++ } ++ ++ void setCmpOpcode(const int32_t &BlockCmpOpcode) { ++ CmpOpcode = BlockCmpOpcode; ++ } ++ ++ void setLoopHeader(const int16_t &BlockLoopHeader) { ++ LoopHeader = BlockLoopHeader; ++ } ++ ++ void setProcedureType(const int16_t &BlockProcedureType) { ++ ProcedureType = BlockProcedureType; ++ } ++ ++ void setCount(const int64_t &BlockCount) { ++ Count = BlockCount; ++ } ++ ++ void setFallthroughCount(const int64_t &BlockFallthroughCount) { ++ FallthroughCount = BlockFallthroughCount; ++ } ++ ++ void setTotalLoops(const int64_t &BlockTotalLoops) { ++ TotalLoops = BlockTotalLoops; ++ } ++ ++ void setLoopDepth(const int64_t &BlockLoopDepth) { ++ LoopDepth = BlockLoopDepth; ++ } ++ ++ void setLoopNumBlocks(const int64_t &BlockLoopNumBlocks) { ++ LoopNumBlocks = BlockLoopNumBlocks; ++ } ++ ++ void setLocalExitingBlock(const int64_t &BlockLocalExitingBlock) { ++ LocalExitingBlock = BlockLocalExitingBlock; ++ } ++ ++ void setLocalLatchBlock(const int64_t &BlockLocalLatchBlock) { ++ LocalLatchBlock = BlockLocalLatchBlock; ++ } ++ ++ void setLocalLoopHeader(const int64_t &BlockLocalLoopHeader) { ++ LocalLoopHeader = BlockLocalLoopHeader; ++ } ++ ++ void setDeltaTaken(const int64_t &BlockDeltaTaken) { ++ DeltaTaken = BlockDeltaTaken; ++ } ++ ++ void setNumLoads(const int64_t &BlockNumLoads) { ++ NumLoads = BlockNumLoads; ++ } ++ ++ void setNumCalls(const int64_t &BlockNumCalls) { ++ NumCalls = BlockNumCalls; ++ } ++ ++ void setOperandRAType(const int64_t &BlockOperandRAType) { ++ OperandRAType = BlockOperandRAType; ++ } ++ ++ void setOperandRBType(const int64_t &BlockOperandRBType) { ++ OperandRBType = BlockOperandRBType; ++ } ++ ++ void setBasicBlockSize(const int64_t &BlockBasicBlockSize) { ++ BasicBlockSize = BlockBasicBlockSize; ++ } ++ ++ void setNumBasicBlocks(const int64_t &BlockNumBasicBlocks) { ++ NumBasicBlocks = BlockNumBasicBlocks; ++ } ++ ++ void setHasIndirectCalls(const int64_t &BlockHasIndirectCalls) { ++ HasIndirectCalls = BlockHasIndirectCalls; ++ } ++ ++ void setEndOpcodeVec(const int32_t &EndOpcode) { ++ EndOpcode_vec.push_back(EndOpcode); ++ } ++ ++ void setLoopHeaderVec(const int16_t &LoopHeader) { ++ LoopHeader_vec.push_back(LoopHeader); ++ } ++ ++ void setBackedgeVec(const int16_t &Backedge) { ++ Backedge_vec.push_back(Backedge); ++ } ++ ++ void setExitVec(const int16_t &Exit) { ++ Exit_vec.push_back(Exit); ++ } ++ ++ void setCallVec(const int16_t &Call) { ++ Call_vec.push_back(Call); ++ } ++ ++ void setBasicBlockSizeVec(const int64_t &BasicBlockSize) { ++ BasicBlockSize_vec.push_back(BasicBlockSize); ++ } ++ ++ void setFunExec(const uint64_t &BlockFuncExec) { ++ FuncExec = BlockFuncExec; ++ } ++ ++ void setParentChildNum(const int32_t &BlockParentChildNum) { ++ ParentChildNum = BlockParentChildNum; ++ } ++ ++ void setParentCount(const int32_t &BlockParentCount) { ++ ParentCount = BlockParentCount; ++ } ++ ++ void setChildParentNum(const int32_t &BlockChildParentNum) { ++ ChildParentNum = BlockChildParentNum; ++ } ++ ++ void setChildCount(const int32_t &BlockChildCount) { ++ ChildCount = BlockChildCount; ++ } ++ ++ ++ ++ void setInferenceFeatures() { ++ ++ if (Count == -1 || FallthroughCount == -1) { ++ return; ++ } ++ if (ParentChildNum == -1 && ParentCount == -1 && ChildParentNum == -1 && ChildCount == -1){ ++ return; ++ } ++ ++ InferenceFeatures.push_back(static_cast(Direction)); ++ InferenceFeatures.push_back(static_cast(LoopHeader)); ++ InferenceFeatures.push_back(static_cast(ProcedureType)); ++ InferenceFeatures.push_back(static_cast(OperandRAType)); ++ InferenceFeatures.push_back(static_cast(OperandRBType)); ++ InferenceFeatures.push_back(static_cast(LoopHeader_vec[0])); ++ InferenceFeatures.push_back(static_cast(Backedge_vec[0])); ++ InferenceFeatures.push_back(static_cast(Exit_vec[0])); ++ InferenceFeatures.push_back(static_cast(LoopHeader_vec[1])); ++ InferenceFeatures.push_back(static_cast(Call_vec[0])); ++ InferenceFeatures.push_back(static_cast(LocalExitingBlock)); ++ InferenceFeatures.push_back(static_cast(HasIndirectCalls)); ++ InferenceFeatures.push_back(static_cast(LocalLatchBlock)); ++ InferenceFeatures.push_back(static_cast(LocalLoopHeader)); ++ InferenceFeatures.push_back(static_cast(Opcode)); ++ InferenceFeatures.push_back(static_cast(CmpOpcode)); ++ InferenceFeatures.push_back(static_cast(EndOpcode_vec[0])); ++ InferenceFeatures.push_back(static_cast(EndOpcode_vec[1])); ++ InferenceFeatures.push_back(static_cast(FuncExec)); ++ InferenceFeatures.push_back(static_cast(NumBasicBlocks)); ++ InferenceFeatures.push_back(static_cast(BasicBlockSize)); ++ InferenceFeatures.push_back(static_cast(BasicBlockSize_vec[0])); ++ InferenceFeatures.push_back(static_cast(BasicBlockSize_vec[1])); ++ InferenceFeatures.push_back(static_cast(LoopNumBlocks)); ++ InferenceFeatures.push_back(static_cast(NumLoads)); ++ InferenceFeatures.push_back(static_cast(NumCalls)); ++ InferenceFeatures.push_back(static_cast(TotalLoops)); ++ InferenceFeatures.push_back(static_cast(DeltaTaken)); ++ InferenceFeatures.push_back(static_cast(LoopDepth)); ++ InferenceFeatures.push_back(static_cast(ParentChildNum)); ++ InferenceFeatures.push_back(static_cast(ParentCount)); ++ InferenceFeatures.push_back(static_cast(ChildParentNum)); ++ InferenceFeatures.push_back(static_cast(ChildCount)); ++ } ++ ++ std::vector getInferenceFeatures() { ++ return InferenceFeatures; ++ } ++ ++}; ++} ++} ++ ++#endif +\ No newline at end of file +diff --git a/bolt/include/bolt/Passes/FeatureMiner.h b/bolt/include/bolt/Passes/FeatureMiner.h +index 916e5515d..b03666ebf 100644 +--- a/bolt/include/bolt/Passes/FeatureMiner.h ++++ b/bolt/include/bolt/Passes/FeatureMiner.h +@@ -14,19 +14,12 @@ + #ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ + #define LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ + +-// #include "BinaryContext.h" +-// #include "BinaryFunction.h" +-// #include "BinaryLoop.h" +-// #include "DominatorAnalysis.h" +-// #include "Passes/BinaryPasses.h" +-// #include "Passes/StaticBranchInfo.h" + #include "bolt/Core/BinaryData.h" + #include "bolt/Core/BinaryFunction.h" + #include "bolt/Core/BinaryLoop.h" + #include "bolt/Passes/DominatorAnalysis.h" + #include "bolt/Passes/BinaryPasses.h" + #include "bolt/Passes/StaticBranchInfo.h" +- + #include "llvm/ADT/DenseMap.h" + #include "llvm/ADT/Optional.h" + #include "llvm/ADT/StringRef.h" +@@ -36,6 +29,7 @@ + #include + #include + #include ++#include + + namespace llvm { + namespace bolt { +@@ -43,7 +37,6 @@ namespace bolt { + class FeatureMiner : public BinaryFunctionPass { + private: + std::unique_ptr SBI; +- + /// BasicBlockInfo - This structure holds feature information about the target + /// BasicBlock of either the taken or the fallthrough paths of a given branch. + struct BasicBlockInfo { +@@ -57,7 +50,7 @@ private: + Optional NumLoads; + Optional NumStores; + Optional EndOpcode; // 0 = NOTHING +- StringRef EndOpcodeStr = "UNDEF"; ++ std::string EndOpcodeStr = "UNDEF"; + Optional BasicBlockSize; + std::string FromFunName = "UNDEF"; + uint32_t FromBb; +@@ -76,8 +69,8 @@ private: + /// BranchFeaturesInfo - This structure holds feature information about each + /// two-way branch from the program. + struct BranchFeaturesInfo { +- StringRef OpcodeStr = "UNDEF"; +- StringRef CmpOpcodeStr = "UNDEF"; ++ std::string OpcodeStr = "UNDEF"; ++ std::string CmpOpcodeStr = "UNDEF"; + bool Simple = 0; + + Optional Opcode; +@@ -130,28 +123,28 @@ private: + }; + + typedef std::unique_ptr BFIPtr; ++ + std::vector BranchesInfoSet; + ++ + /// getProcedureType - Determines which category the function falls into: + /// Leaf, Non-leaf or Calls-self. + int8_t getProcedureType(BinaryFunction &Function, BinaryContext &BC); + + /// addSuccessorInfo - Discovers feature information for the target successor + /// basic block, and inserts it into the static branch info container. +- void addSuccessorInfo(DominatorAnalysis &DA, +- DominatorAnalysis &PDA, BFIPtr const &BFI, +- BinaryFunction &Function, BinaryContext &BC, +- MCInst &Inst, BinaryBasicBlock &BB, bool Succ); ++ void addSuccessorInfo(BFIPtr const &BFI, BinaryFunction &Function, ++ BinaryContext &BC, BinaryBasicBlock &BB, bool SuccType); + + /// extractFeatures - Extracts the feature information for each two-way branch + /// from the program. + void extractFeatures(BinaryFunction &Function, +- BinaryContext &BC, +- raw_ostream &Printer); ++ BinaryContext &BC); + ++ void generateInstFeatures(BinaryContext &BC, BinaryBasicBlock &BB, BFIPtr const &BFI, int Index); + /// dumpSuccessorFeatures - Dumps the feature information about the target + /// BasicBlock of either the taken or the fallthrough paths of a given branch. +- void dumpSuccessorFeatures(raw_ostream &Printer, BBIPtr &Successor); ++ void generateSuccessorFeatures(BBIPtr &Successor, BinaryBasicBlockFeature *BBF); + + /// dumpFeatures - Dumps the feature information about each two-way branch + /// from the program. +@@ -167,12 +160,17 @@ public: + explicit FeatureMiner(const cl::opt &PrintPass) + : BinaryFunctionPass(PrintPass) {} + ++ std::ofstream trainPrinter; ++ ++ + const char *getName() const override { return "feature-miner"; } + + void runOnFunctions(BinaryContext &BC) override; ++ void inferenceFeatures(BinaryFunction &Function); ++ void generateProfileFeatures(BinaryBasicBlock *BB, BinaryBasicBlockFeature *BBF); + }; + + } // namespace bolt + } // namespace llvm + +-#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ */ +\ No newline at end of file ++#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_FEATUREMINER_H_ */ +diff --git a/bolt/include/bolt/Passes/StaticBranchInfo.h b/bolt/include/bolt/Passes/StaticBranchInfo.h +index 1713d3367..54a1f7cff 100644 +--- a/bolt/include/bolt/Passes/StaticBranchInfo.h ++++ b/bolt/include/bolt/Passes/StaticBranchInfo.h +@@ -18,16 +18,11 @@ + #ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ + #define LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ + +-// #include "BinaryContext.h" +-// #include "BinaryFunction.h" +-// #include "BinaryLoop.h" + #include "bolt/Core/BinaryContext.h" + #include "bolt/Core/BinaryFunction.h" + #include "bolt/Core/BinaryLoop.h" +- + #include "llvm/MC/MCSymbol.h" +-// add new include +-#include ++#include + + namespace llvm { + namespace bolt { +@@ -113,4 +108,4 @@ public: + } // namespace bolt + } // namespace llvm + +-#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ */ +\ No newline at end of file ++#endif /* LLVM_TOOLS_LLVM_BOLT_PASSES_STATICBRANCHINFO_H_ */ +diff --git a/bolt/include/bolt/Profile/DataReader.h b/bolt/include/bolt/Profile/DataReader.h +index 60cedfeb3..7e2c318fc 100644 +--- a/bolt/include/bolt/Profile/DataReader.h ++++ b/bolt/include/bolt/Profile/DataReader.h +@@ -21,6 +21,7 @@ + #include "llvm/Support/ErrorOr.h" + #include "llvm/Support/MemoryBuffer.h" + #include "llvm/Support/raw_ostream.h" ++#include + #include + #include + +@@ -43,6 +44,13 @@ inline raw_ostream &operator<<(raw_ostream &OS, const LBREntry &LBR) { + return OS; + } + ++extern "C" { ++typedef void *(*CreateONNXRunnerFunc)(const char *); ++typedef void (*DeleteONNXRunnerFunc)(void *); ++typedef float (*RunONNXModelFunc)(void *, std::vector &, ++ std::vector &, std::vector &); ++} ++ + struct Location { + bool IsSymbol; + StringRef Name; +@@ -262,7 +270,8 @@ struct FuncSampleData { + class DataReader : public ProfileReaderBase { + public: + explicit DataReader(StringRef Filename) +- : ProfileReaderBase(Filename), Diag(errs()) {} ++ : ProfileReaderBase(Filename), Diag(errs()), onnxRunner(nullptr), ++ lib_handle(nullptr), handleOnnxRuntime(nullptr) {} + + StringRef getReaderName() const override { return "branch profile reader"; } + +@@ -281,7 +290,70 @@ public: + /// Return all event names used to collect this profile + virtual StringSet<> getEventNames() const override { return EventNames; } + ++ ~DataReader() { ++ // delete onnxrunner; ++ if (onnxRunner && lib_handle && handleOnnxRuntime) { ++ DeleteONNXRunnerFunc deleteONNXRunner = ++ (DeleteONNXRunnerFunc)dlsym(lib_handle, "deleteONNXRunner"); ++ deleteONNXRunner(onnxRunner); ++ dlclose(lib_handle); ++ dlclose(handleOnnxRuntime); ++ } ++ } ++ ++ /// Initialize the onnxruntime model. ++ void initializeONNXRunner(const std::string &modelPath) { ++ if (!onnxRunner && !lib_handle && !handleOnnxRuntime) { ++ handleOnnxRuntime = ++ dlopen("libonnxruntime.so.1.16.3", RTLD_LAZY | RTLD_GLOBAL); ++ lib_handle = ++ dlopen("libONNXRunner.so", RTLD_LAZY); ++ CreateONNXRunnerFunc createONNXRunner = ++ (CreateONNXRunnerFunc)dlsym(lib_handle, "createONNXRunner"); ++ onnxRunner = createONNXRunner(modelPath.c_str()); ++ } ++ } ++ ++ /// Inference step for predicting the BB counts based on the BB features. ++ float ONNXInference(std::vector &input_string, ++ std::vector &input_int64, ++ std::vector &input_float) { ++ if (onnxRunner && lib_handle) { ++ RunONNXModelFunc runONNXModel = ++ (RunONNXModelFunc)dlsym(lib_handle, "runONNXModel"); ++ float model_pred = ++ runONNXModel(onnxRunner, input_string, input_int64, input_float); ++ return model_pred; ++ } ++ return -1.0; ++ } ++ + protected: ++ /// The onnxruntime model pointer read from the input model path. ++ void *onnxRunner; ++ ++ /// The library handle of the ai4compiler framwork. ++ void *lib_handle; ++ ++ /// The library handle of the onnxruntime. ++ void *handleOnnxRuntime; ++ ++ /// The annotating threshold for the model prediction. ++ float threshold = 0.95; ++ ++ /// Return the annotating threshold for the model prediction. ++ float getThreshold() const { return threshold; } ++ ++ /// The counting value of the total modified BB-count number. ++ uint64_t modified_BB_total = 0; ++ ++ /// Add the total modified BB-count number by the BB modifiied number within ++ /// the funciton. ++ void addModifiedBBTotal(uint64_t &value) { modified_BB_total += value; } ++ ++ /// Return the counting value of the total modified BB-count number. ++ uint64_t getModifiedBBTotal() const { return modified_BB_total; } ++ + /// Read profile information available for the function. + void readProfile(BinaryFunction &BF); + +diff --git a/bolt/lib/Core/BinaryBasicBlockFeature.cpp b/bolt/lib/Core/BinaryBasicBlockFeature.cpp +new file mode 100644 +index 000000000..81d8c7546 +--- /dev/null ++++ b/bolt/lib/Core/BinaryBasicBlockFeature.cpp +@@ -0,0 +1,25 @@ ++//===- bolt/Core/BinaryBasicBlockFeature.cpp - Low-level basic block -------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements the BinaryBasicBlock class. ++// ++//===----------------------------------------------------------------------===// ++ ++#include "bolt/Core/BinaryBasicBlock.h" ++#include "bolt/Core/BinaryBasicBlockFeature.h" ++ ++ ++#define DEBUG_TYPE "bolt" ++ ++namespace llvm { ++namespace bolt { ++ ++ ++ ++} // namespace bolt ++} // namespace llvm +\ No newline at end of file +diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt +index 501b5eb6e..25e9fb020 100644 +--- a/bolt/lib/Core/CMakeLists.txt ++++ b/bolt/lib/Core/CMakeLists.txt +@@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS + + add_llvm_library(LLVMBOLTCore + BinaryBasicBlock.cpp ++ BinaryBasicBlockFeature.cpp + BinaryContext.cpp + BinaryData.cpp + BinaryEmitter.cpp +diff --git a/bolt/lib/Passes/CMakeLists.txt b/bolt/lib/Passes/CMakeLists.txt +index 901ff614c..9f5abd101 100644 +--- a/bolt/lib/Passes/CMakeLists.txt ++++ b/bolt/lib/Passes/CMakeLists.txt +@@ -12,9 +12,9 @@ add_llvm_library(LLVMBOLTPasses + DataflowAnalysis.cpp + DataflowInfoManager.cpp + ExtTSPReorderAlgorithm.cpp +- FeatureMiner.cpp + FrameAnalysis.cpp + FrameOptimizer.cpp ++ FeatureMiner.cpp + HFSort.cpp + HFSortPlus.cpp + IdenticalCodeFolding.cpp +diff --git a/bolt/lib/Passes/FeatureMiner.cpp b/bolt/lib/Passes/FeatureMiner.cpp +index 680222906..abee69745 100644 +--- a/bolt/lib/Passes/FeatureMiner.cpp ++++ b/bolt/lib/Passes/FeatureMiner.cpp +@@ -11,15 +11,10 @@ + // https://dl.acm.org/doi/10.1145/239912.239923 + //===----------------------------------------------------------------------===// + +-// #include "Passes/FeatureMiner.h" +-// #include "Passes/DataflowInfoManager.h" +-// #include "llvm/Support/CommandLine.h" +-// #include "llvm/Support/Options.h" + #include "bolt/Passes/FeatureMiner.h" + #include "bolt/Passes/DataflowInfoManager.h" ++#include "bolt/Passes/StaticBranchInfo.h" + #include "llvm/Support/CommandLine.h" +- +-// add new include + #include "llvm/Support/FileSystem.h" + + #undef DEBUG_TYPE +@@ -29,24 +24,7 @@ using namespace llvm; + using namespace bolt; + + namespace opts { +- +-extern cl::OptionCategory InferenceCategory; +- +-cl::opt VespaUseDFS( +- "vespa-dfs", +- cl::desc("use DFS ordering when using -gen-features option"), +- cl::init(false), +- cl::ReallyHidden, +- cl::ZeroOrMore, +- cl::cat(InferenceCategory)); +- +-cl::opt IncludeValidProfile( +- "beetle-valid-profile-info", +- cl::desc("include valid profile information."), +- cl::init(false), +- cl::ReallyHidden, +- cl::ZeroOrMore, +- cl::cat(InferenceCategory)); ++extern cl::opt BlockCorrection; + + } // namespace opts + +@@ -75,50 +53,19 @@ int8_t FeatureMiner::getProcedureType(BinaryFunction &Function, + return ProcedureType; // leaf type + } + +-void FeatureMiner::addSuccessorInfo(DominatorAnalysis &DA, +- DominatorAnalysis &PDA, +- BFIPtr const &BFI, BinaryFunction &Function, +- BinaryContext &BC, MCInst &Inst, +- BinaryBasicBlock &BB, bool SuccType) { ++void FeatureMiner::addSuccessorInfo(BFIPtr const &BFI, BinaryFunction &Function, ++ BinaryContext &BC, BinaryBasicBlock &BB, bool SuccType) { + + BinaryBasicBlock *Successor = BB.getConditionalSuccessor(SuccType); + + if (!Successor) + return; + +- unsigned NumLoads{0}; +- unsigned NumStores{0}; +- unsigned NumCallsExit{0}; + unsigned NumCalls{0}; +- unsigned NumCallsInvoke{0}; +- unsigned NumTailCalls{0}; +- unsigned NumIndirectCalls{0}; + + for (auto &Inst : BB) { +- if (BC.MIB->isLoad(Inst)) { +- ++NumLoads; +- } else if (BC.MIB->isStore(Inst)) { +- ++NumStores; +- } else if (BC.MIB->isCall(Inst)) { ++ if (BC.MIB->isCall(Inst)) { + ++NumCalls; +- +- if (BC.MIB->isIndirectCall(Inst)) +- ++NumIndirectCalls; +- +- if (BC.MIB->isInvoke(Inst)) +- ++NumCallsInvoke; +- +- if (BC.MIB->isTailCall(Inst)) +- ++NumTailCalls; +- +- if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { +- StringRef CalleeName = CalleeSymbol->getName(); +- if (CalleeName == "__cxa_throw@PLT" || +- CalleeName == "_Unwind_Resume@PLT" || +- CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || +- CalleeName == "abort@PLT") +- ++NumCallsExit; +- } + } + } + +@@ -138,117 +85,78 @@ void FeatureMiner::addSuccessorInfo(DominatorAnalysis &DA, + SuccBBInfo->Backedge = SBI->isBackEdge(&BB, Successor); + + MCInst *SuccInst = Successor->getTerminatorBefore(nullptr); ++ + // Store information about the branch type ending sucessor basic block + SuccBBInfo->EndOpcode = (SuccInst && BC.MIA->isBranch(*SuccInst)) + ? SuccInst->getOpcode() + : 0; // 0 = NOTHING +- if (SuccBBInfo->EndOpcode != 0) +- SuccBBInfo->EndOpcodeStr = BC.MII->getName(SuccInst->getOpcode()); +- else +- SuccBBInfo->EndOpcodeStr = "NOTHING"; + + // Check if the successor basic block contains + // a procedure call and store it. + SuccBBInfo->Call = (NumCalls > 0) ? 1 // Contains a call instruction + : 0; // Does not contain a call instruction + +- SuccBBInfo->NumStores = NumStores; +- SuccBBInfo->NumLoads = NumLoads; +- SuccBBInfo->NumCallsExit = NumCallsExit; +- SuccBBInfo->NumCalls = NumCalls; +- +- SuccBBInfo->NumCallsInvoke = NumCallsInvoke; +- SuccBBInfo->NumIndirectCalls = NumIndirectCalls; +- SuccBBInfo->NumTailCalls = NumTailCalls; +- +- auto InstSucc = Successor->getLastNonPseudoInstr(); +- if (InstSucc) { +- // Check if the source basic block dominates its +- // target basic block and store it. +- SuccBBInfo->BranchDominates = (DA.doesADominateB(Inst, *InstSucc) == true) +- ? 1 // Dominates +- : 0; // Does not dominate +- +- // Check if the target basic block postdominates +- // the source basic block and store it. +- SuccBBInfo->BranchPostdominates = +- (PDA.doesADominateB(*InstSucc, Inst) == true) +- ? 1 // Postdominates +- : 0; // Does not postdominate +- } + +- /// The follwoing information is used as an identifier only for +- /// the purpose of matching the inferred probabilities with the branches +- /// in the binary. +- SuccBBInfo->FromFunName = Function.getPrintName(); +- SuccBBInfo->FromBb = BB.getInputOffset(); +- BinaryFunction *ToFun = Successor->getFunction(); +- SuccBBInfo->ToFunName = ToFun->getPrintName(); +- SuccBBInfo->ToBb = Successor->getInputOffset(); +- +- auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); +- if (Offset) { +- uint32_t TargetOffset = Successor->getInputOffset(); +- uint32_t BranchOffset = Offset.get(); +- BFI->BranchOffset = BranchOffset; +- if (BranchOffset != UINT32_MAX && TargetOffset != UINT32_MAX) { +- int64_t Delta = TargetOffset - BranchOffset; +- BFI->DeltaTaken = std::abs(Delta); +- } +- } ++ uint32_t Offset = BB.getEndOffset(); + + if (SuccType) { + BFI->TrueSuccessor = std::move(SuccBBInfo); +- + // Check if the taken branch is a forward +- // or a backwards branch and store it. ++ // or a backwards branch and store it + BFI->Direction = (Function.isForwardBranch(&BB, Successor) == true) +- ? 1 // Forward branch +- : 0; // Backwards branch +- +- auto TakenBranchInfo = BB.getTakenBranchInfo(); +- BFI->Count = TakenBranchInfo.Count; +- BFI->MissPredicted = TakenBranchInfo.MispredictedCount; ++ ? 1 // Forward branch ++ : 0; // Backwards branch ++ ++ auto OnlyBranchInfo = BB.getOnlyBranchInfo(); ++ BFI->Count = OnlyBranchInfo.Count; ++ ++ if (Offset) { ++ uint32_t TargetOffset = Successor->getInputOffset(); ++ uint32_t BranchOffset = Offset; ++ if (BranchOffset != UINT32_MAX && TargetOffset != UINT32_MAX) { ++ int64_t Delta = static_cast(TargetOffset) - static_cast(BranchOffset); ++ BFI->DeltaTaken = std::abs(Delta); ++ } ++ } + } else { ++ if (BB.succ_size() == 2) { ++ auto FallthroughBranchInfo = BB.getFallthroughBranchInfo(); ++ BFI->FallthroughCount = FallthroughBranchInfo.Count; ++ } else { ++ auto OnlyBranchInfo = BB.getOnlyBranchInfo(); ++ BFI->FallthroughCount = OnlyBranchInfo.Count; ++ } + BFI->FalseSuccessor = std::move(SuccBBInfo); +- +- auto FallthroughBranchInfo = BB.getFallthroughBranchInfo(); +- BFI->FallthroughCount = FallthroughBranchInfo.Count; +- BFI->FallthroughMissPredicted = FallthroughBranchInfo.MispredictedCount; + } + } + +-void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC, +- raw_ostream &Printer) { ++void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC) { + int8_t ProcedureType = getProcedureType(Function, BC); +-// auto Info = DataflowInfoManager(BC, Function, nullptr, nullptr); + auto Info = DataflowInfoManager(Function, nullptr, nullptr); +- auto &DA = Info.getDominatorAnalysis(); +- auto &PDA = Info.getPostDominatorAnalysis(); + const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); +- bool Simple = Function.isSimple(); + +-// const auto &Order = opts::VespaUseDFS ? Function.dfs() : Function.getLayout(); ++ bool Simple = Function.isSimple(); + const auto &Order = Function.dfs(); ++ std::string Function_name = Function.getPrintName(); ++ + + for (auto *BBA : Order) { +- ++ + auto &BB = *BBA; +- unsigned NumOuterLoops{0}; ++ ++ BinaryBasicBlockFeature BBF = BB.getFeatures(); ++ + unsigned TotalLoops{0}; +- unsigned MaximumLoopDepth{0}; + unsigned LoopDepth{0}; +- unsigned LoopNumExitEdges{0}; +- unsigned LoopNumExitBlocks{0}; +- unsigned LoopNumExitingBlocks{0}; +- unsigned LoopNumLatches{0}; + unsigned LoopNumBlocks{0}; +- unsigned LoopNumBackEdges{0}; + + bool LocalExitingBlock{false}; + bool LocalLatchBlock{false}; + bool LocalLoopHeader{false}; + ++ ++ generateProfileFeatures(&BB, &BBF); ++ + BinaryLoop *Loop = LoopsInfo.getLoopFor(&BB); + if (Loop) { + SmallVector ExitingBlocks; +@@ -263,414 +171,301 @@ void FeatureMiner::extractFeatures(BinaryFunction &Function, BinaryContext &BC, + SmallVector Latches; + Loop->getLoopLatches(Latches); + +- NumOuterLoops = LoopsInfo.OuterLoops; + TotalLoops = LoopsInfo.TotalLoops; +- MaximumLoopDepth = LoopsInfo.MaximumDepth; + LoopDepth = Loop->getLoopDepth(); +- LoopNumExitEdges = ExitEdges.size(); +- LoopNumExitBlocks = ExitBlocks.size(); +- LoopNumExitingBlocks = ExitingBlocks.size(); +- LoopNumLatches = Latches.size(); + LoopNumBlocks = Loop->getNumBlocks(); +- LoopNumBackEdges = Loop->getNumBackEdges(); +- + LocalExitingBlock = Loop->isLoopExiting(&BB); + LocalLatchBlock = Loop->isLoopLatch(&BB); + LocalLoopHeader = ((Loop->getHeader() == (&BB)) ? 1 : 0); + } + + unsigned NumLoads{0}; +- unsigned NumStores{0}; +- unsigned NumCallsExit{0}; + unsigned NumCalls{0}; +- unsigned NumCallsInvoke{0}; +- unsigned NumTailCalls{0}; + unsigned NumIndirectCalls{0}; +- unsigned NumSelfCalls{0}; + + for (auto &Inst : BB) { + if (BC.MIB->isLoad(Inst)) { + ++NumLoads; +- } else if (BC.MIB->isStore(Inst)) { +- ++NumStores; + } else if (BC.MIB->isCall(Inst)) { + ++NumCalls; +- + if (BC.MIB->isIndirectCall(Inst)) + ++NumIndirectCalls; +- +- if (BC.MIB->isInvoke(Inst)) +- ++NumCallsInvoke; +- +- if (BC.MIB->isTailCall(Inst)) +- ++NumTailCalls; +- +- if (const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst)) { +- StringRef CalleeName = CalleeSymbol->getName(); +- if (CalleeName == "__cxa_throw@PLT" || +- CalleeName == "_Unwind_Resume@PLT" || +- CalleeName == "__cxa_rethrow@PLT" || CalleeName == "exit@PLT" || +- CalleeName == "abort@PLT") +- ++NumCallsExit; +- else if (CalleeName == Function.getPrintName()) { +- ++NumSelfCalls; +- } +- } + } + } + + int Index = -2; + bool LoopHeader = SBI->isLoopHeader(&BB); ++ ++ BFIPtr BFI = std::make_unique(); ++ ++ ++ BFI->TotalLoops = TotalLoops; ++ BFI->LoopDepth = LoopDepth; ++ BFI->LoopNumBlocks = LoopNumBlocks; ++ BFI->LocalExitingBlock = LocalExitingBlock; ++ BFI->LocalLatchBlock = LocalLatchBlock; ++ BFI->LocalLoopHeader = LocalLoopHeader; ++ BFI->NumCalls = NumCalls; ++ BFI->BasicBlockSize = BB.size(); ++ BFI->NumBasicBlocks = Function.size(); ++ ++ BFI->NumLoads = NumLoads; ++ BFI->NumIndirectCalls = NumIndirectCalls; ++ BFI->LoopHeader = LoopHeader; ++ BFI->ProcedureType = ProcedureType; ++ ++ // Adding taken successor info. ++ addSuccessorInfo(BFI, Function, BC, BB, true); ++ // Adding fall through successor info. ++ addSuccessorInfo(BFI, Function, BC, BB, false); ++ ++ MCInst ConditionalInst; ++ bool hasConditionalBranch = false; ++ MCInst UnconditionalInst; ++ bool hasUnconditionalBranch = false; ++ + for (auto &Inst : BB) { + ++Index; +- +- if (!BC.MIA->isConditionalBranch(Inst)) ++ if (!BC.MIA->isConditionalBranch(Inst) && !BC.MIA->isUnconditionalBranch(Inst)) + continue; + +- BFIPtr BFI = std::make_unique(); +- +- BFI->Simple = Simple; +- BFI->NumOuterLoops = NumOuterLoops; +- BFI->TotalLoops = TotalLoops; +- BFI->MaximumLoopDepth = MaximumLoopDepth; +- BFI->LoopDepth = LoopDepth; +- BFI->LoopNumExitEdges = LoopNumExitEdges; +- BFI->LoopNumExitBlocks = LoopNumExitBlocks; +- BFI->LoopNumExitingBlocks = LoopNumExitingBlocks; +- BFI->LoopNumLatches = LoopNumLatches; +- BFI->LoopNumBlocks = LoopNumBlocks; +- BFI->LoopNumBackEdges = LoopNumBackEdges; +- +- BFI->LocalExitingBlock = LocalExitingBlock; +- BFI->LocalLatchBlock = LocalLatchBlock; +- BFI->LocalLoopHeader = LocalLoopHeader; +- +- BFI->Call = ((NumCalls > 0) ? 1 : 0); +- BFI->NumCalls = NumCalls; +- +- BFI->BasicBlockSize = BB.size(); +- BFI->NumBasicBlocks = Function.size(); +- BFI->NumSelfCalls = NumSelfCalls; +- +- BFI->NumLoads = NumLoads; +- BFI->NumStores = NumStores; +- BFI->NumCallsExit = NumCallsExit; +- +- BFI->NumCallsInvoke = NumCallsInvoke; +- BFI->NumIndirectCalls = NumIndirectCalls; +- BFI->NumTailCalls = NumTailCalls; +- +- // Check if branch's basic block is a loop header and store it. +- BFI->LoopHeader = LoopHeader; +- +- // Adding taken successor info. +- addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, true); +- // Adding fall through successor info. +- addSuccessorInfo(DA, PDA, BFI, Function, BC, Inst, BB, false); +- +- // Holds the branch opcode info. +- BFI->Opcode = Inst.getOpcode(); +- BFI->OpcodeStr = BC.MII->getName(Inst.getOpcode()); +- +- // Holds the branch's procedure type. +- BFI->ProcedureType = ProcedureType; +- +- BFI->CmpOpcode = 0; +- if (Index > -1) { +- auto Cmp = BB.begin() + Index; +- +- if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { +- // Holding the branch comparison opcode info. +- BFI->CmpOpcode = (*Cmp).getOpcode(); ++ generateInstFeatures(BC, BB, BFI, Index); + +- BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); +- +- auto getOperandType = [&](const MCOperand &Operand) -> int32_t { +- if (Operand.isReg()) +- return 0; +- else if (Operand.isImm()) +- return 1; +- // else if (Operand.isFPImm()) +- else if (Operand.isSFPImm()) +- return 2; +- else if (Operand.isExpr()) +- return 3; +- else +- return -1; +- }; +- +- const auto InstInfo = BC.MII->get((*Cmp).getOpcode()); +- unsigned NumDefs = InstInfo.getNumDefs(); +- int32_t NumPrimeOperands = +- MCPlus::getNumPrimeOperands(*Cmp) - NumDefs; +- switch (NumPrimeOperands) { +- case 6: { +- int32_t RBType = getOperandType((*Cmp).getOperand(NumDefs)); +- int32_t RAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); +- +- if (RBType == 0 && RAType == 0) { +- BFI->OperandRBType = RBType; +- BFI->OperandRAType = RAType; +- } else if (RBType == 0 && (RAType == 1 || RAType == 2)) { +- RAType = getOperandType((*Cmp).getOperand(NumPrimeOperands - 1)); +- +- if (RAType != 1 && RAType != 2) { +- RAType = -1; +- } +- +- BFI->OperandRBType = RBType; +- BFI->OperandRAType = RAType; +- } else { +- BFI->OperandRAType = -1; +- BFI->OperandRBType = -1; +- } +- break; +- } +- case 2: +- BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); +- BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); +- break; +- case 3: +- BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); +- BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 2)); +- break; +- case 1: +- BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs)); +- break; +- default: +- BFI->OperandRAType = -1; +- BFI->OperandRBType = -1; +- break; +- } +- +- } else { +- Index -= 1; +- for (int Idx = Index; Idx > -1; Idx--) { +- auto Cmp = BB.begin() + Idx; +- if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { +- // Holding the branch comparison opcode info. +- BFI->CmpOpcode = (*Cmp).getOpcode(); +- BFI->CmpOpcodeStr = BC.MII->getName((*Cmp).getOpcode()); +- break; +- } +- } +- } ++ if (BC.MIA->isConditionalBranch(Inst)) { ++ ConditionalInst = Inst; ++ hasConditionalBranch = true; + } ++ ++ ++ if (BC.MIA->isUnconditionalBranch(Inst)){ ++ UnconditionalInst = Inst; ++ hasUnconditionalBranch = true; ++ } ++ ++ } + +- //======================================================================== +- +- auto &FalseSuccessor = BFI->FalseSuccessor; +- auto &TrueSuccessor = BFI->TrueSuccessor; ++ if (hasConditionalBranch) { ++ BFI->Opcode = ConditionalInst.getOpcode(); + +- if (!FalseSuccessor && !TrueSuccessor) +- continue; ++ } else { ++ if (hasUnconditionalBranch) { ++ BFI->Opcode = UnconditionalInst.getOpcode(); + +- int64_t BranchOffset = +- (BFI->BranchOffset.hasValue()) +- ? static_cast(*(BFI->BranchOffset)) +- : -1; +- if(BranchOffset == -1) +- continue; ++ } else { ++ auto Inst = BB.getLastNonPseudoInstr(); ++ BFI->Opcode = Inst->getOpcode(); ++ generateInstFeatures(BC, BB, BFI, Index); ++ } ++ } ++ ++ auto &FalseSuccessor = BFI->FalseSuccessor; ++ auto &TrueSuccessor = BFI->TrueSuccessor; + +- int16_t ProcedureType = (BFI->ProcedureType.hasValue()) ++ int16_t ProcedureType = (BFI->ProcedureType.hasValue()) + ? static_cast(*(BFI->ProcedureType)) + : -1; + +- int16_t Direction = (BFI->Direction.hasValue()) +- ? static_cast(*(BFI->Direction)) +- : -1; ++ int64_t Count = (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; + +- int16_t LoopHeader = (BFI->LoopHeader.hasValue()) +- ? static_cast(*(BFI->LoopHeader)) +- : -1; +- +- int32_t Opcode = +- (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; +- +- int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) +- ? static_cast(*(BFI->CmpOpcode)) +- : -1; +- +- int64_t Count = +- (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; +- +- int64_t MissPredicted = (BFI->MissPredicted.hasValue()) +- ? static_cast(*(BFI->MissPredicted)) +- : -1; +- +- int64_t FallthroughCount = +- (BFI->FallthroughCount.hasValue()) ++ int64_t FallthroughCount = (BFI->FallthroughCount.hasValue()) + ? static_cast(*(BFI->FallthroughCount)) + : -1; + +- int64_t FallthroughMissPredicted = +- (BFI->FallthroughMissPredicted.hasValue()) +- ? static_cast(*(BFI->FallthroughMissPredicted)) +- : -1; +- +- int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) +- ? static_cast(*(BFI->NumOuterLoops)) +- : -1; +- int64_t TotalLoops = (BFI->TotalLoops.hasValue()) +- ? static_cast(*(BFI->TotalLoops)) ++ int16_t LoopHeaderValid = (BFI->LoopHeader.hasValue()) ++ ? static_cast(*(BFI->LoopHeader)) + : -1; +- int64_t MaximumLoopDepth = +- (BFI->MaximumLoopDepth.hasValue()) +- ? static_cast(*(BFI->MaximumLoopDepth)) +- : -1; +- int64_t LoopDepth = (BFI->LoopDepth.hasValue()) ++ ++ int64_t TotalLoopsValid = (BFI->TotalLoops.hasValue()) ++ ? static_cast(*(BFI->TotalLoops)) ++ : -1; ++ int64_t LoopDepthValid = (BFI->LoopDepth.hasValue()) + ? static_cast(*(BFI->LoopDepth)) + : -1; +- int64_t LoopNumExitEdges = +- (BFI->LoopNumExitEdges.hasValue()) +- ? static_cast(*(BFI->LoopNumExitEdges)) +- : -1; +- int64_t LoopNumExitBlocks = +- (BFI->LoopNumExitBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumExitBlocks)) +- : -1; +- int64_t LoopNumExitingBlocks = +- (BFI->LoopNumExitingBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumExitingBlocks)) +- : -1; +- int64_t LoopNumLatches = +- (BFI->LoopNumLatches.hasValue()) +- ? static_cast(*(BFI->LoopNumLatches)) +- : -1; +- int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumBlocks)) +- : -1; +- int64_t LoopNumBackEdges = +- (BFI->LoopNumBackEdges.hasValue()) +- ? static_cast(*(BFI->LoopNumBackEdges)) +- : -1; ++ int64_t LoopNumBlocksValid = (BFI->LoopNumBlocks.hasValue()) ++ ? static_cast(*(BFI->LoopNumBlocks)) ++ : -1; ++ int64_t LocalExitingBlockValid = ++ (BFI->LocalExitingBlock.hasValue()) ++ ? static_cast(*(BFI->LocalExitingBlock)) ++ : -1; + +- int64_t LocalExitingBlock = +- (BFI->LocalExitingBlock.hasValue()) +- ? static_cast(*(BFI->LocalExitingBlock)) +- : -1; ++ int64_t LocalLatchBlockValid = (BFI->LocalLatchBlock.hasValue()) ++ ? static_cast(*(BFI->LocalLatchBlock)) ++ : -1; + +- int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) +- ? static_cast(*(BFI->LocalLatchBlock)) +- : -1; ++ int64_t LocalLoopHeaderValid = (BFI->LocalLoopHeader.hasValue()) ++ ? static_cast(*(BFI->LocalLoopHeader)) ++ : -1; + +- int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) +- ? static_cast(*(BFI->LocalLoopHeader)) +- : -1; ++ int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) ++ ? static_cast(*(BFI->CmpOpcode)) ++ : -1; + +- int64_t Call = +- (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; ++ int64_t OperandRAType = (BFI->OperandRAType.hasValue()) ++ ? static_cast(*(BFI->OperandRAType)) ++ : 10; + +- int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) +- ? static_cast(*(BFI->DeltaTaken)) +- : -1; ++ int64_t OperandRBType = (BFI->OperandRBType.hasValue()) ++ ? static_cast(*(BFI->OperandRBType)) ++ : 10; ++ int16_t Direction = (BFI->Direction.hasValue()) ++ ? static_cast(*(BFI->Direction)) ++ : -1; + +- int64_t NumLoads = (BFI->NumLoads.hasValue()) +- ? static_cast(*(BFI->NumLoads)) +- : -1; ++ int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) ++ ? static_cast(*(BFI->DeltaTaken)) ++ : -1; + +- int64_t NumStores = (BFI->NumStores.hasValue()) +- ? static_cast(*(BFI->NumStores)) +- : -1; ++ int64_t NumLoadsValid = (BFI->NumLoads.hasValue()) ++ ? static_cast(*(BFI->NumLoads)) ++ : -1; + +- int64_t BasicBlockSize = +- (BFI->BasicBlockSize.hasValue()) +- ? static_cast(*(BFI->BasicBlockSize)) +- : -1; ++ int64_t BasicBlockSize = ++ (BFI->BasicBlockSize.hasValue()) ++ ? static_cast(*(BFI->BasicBlockSize)) ++ : -1; + +- int64_t NumBasicBlocks = +- (BFI->NumBasicBlocks.hasValue()) +- ? static_cast(*(BFI->NumBasicBlocks)) +- : -1; ++ int64_t NumBasicBlocks = ++ (BFI->NumBasicBlocks.hasValue()) ++ ? static_cast(*(BFI->NumBasicBlocks)) ++ : -1; + +- int64_t NumCalls = (BFI->NumCalls.hasValue()) ++ int64_t NumCallsValid = (BFI->NumCalls.hasValue()) + ? static_cast(*(BFI->NumCalls)) + : -1; + +- int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) +- ? static_cast(*(BFI->NumSelfCalls)) +- : -1; +- +- int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) +- ? static_cast(*(BFI->NumCallsExit)) +- : -1; +- +- int64_t OperandRAType = (BFI->OperandRAType.hasValue()) +- ? static_cast(*(BFI->OperandRAType)) +- : -1; +- +- int64_t OperandRBType = (BFI->OperandRBType.hasValue()) +- ? static_cast(*(BFI->OperandRBType)) +- : -1; +- +- int64_t NumCallsInvoke = +- (BFI->NumCallsInvoke.hasValue()) +- ? static_cast(*(BFI->NumCallsInvoke)) +- : -1; +- +- int64_t NumIndirectCalls = ++ int64_t NumIndirectCallsValid = + (BFI->NumIndirectCalls.hasValue()) + ? static_cast(*(BFI->NumIndirectCalls)) + : -1; + +- int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) +- ? static_cast(*(BFI->NumTailCalls)) +- : -1; +- +- Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," +- << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr +- << "," << LoopHeader << "," << ProcedureType << "," << Count +- << "," << MissPredicted << "," << FallthroughCount << "," +- << FallthroughMissPredicted << "," << NumOuterLoops << "," +- << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth +- << "," << LoopDepth << "," << LoopNumExitEdges << "," +- << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," +- << LoopNumLatches << "," << LoopNumBlocks << "," +- << LoopNumBackEdges << "," << LocalExitingBlock << "," +- << LocalLatchBlock << "," << LocalLoopHeader << "," << Call << "," +- << DeltaTaken << "," << NumLoads << "," << NumStores << "," +- << NumCalls << "," << OperandRAType << "," << OperandRBType << "," +- << BasicBlockSize << "," << NumBasicBlocks << "," +- << NumCallsInvoke << "," << NumIndirectCalls << "," +- << NumTailCalls << "," << NumSelfCalls; +- +- if (FalseSuccessor && TrueSuccessor) { +- dumpSuccessorFeatures(Printer, TrueSuccessor); +- dumpSuccessorFeatures(Printer, FalseSuccessor); +- +- FalseSuccessor.reset(); +- TrueSuccessor.reset(); +- } +- BFI.reset(); +- +- std::string BranchOffsetStr = (BranchOffset == -1) ? "None" : Twine::utohexstr(BranchOffset).str(); ++ int64_t HasIndirectCalls = (NumIndirectCallsValid > 0) ? 1 : 0; ++ ++ int32_t Opcode = (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; ++ ++ uint64_t fun_exec = Function.getExecutionCount(); ++ fun_exec = (fun_exec != UINT64_MAX) ? fun_exec : 0; ++ ++ BBF.setDirection(Direction); ++ BBF.setDeltaTaken(DeltaTaken); ++ BBF.setOpcode(Opcode); ++ BBF.setCmpOpcode(CmpOpcode); ++ BBF.setOperandRAType(OperandRAType); ++ BBF.setOperandRBType(OperandRBType); ++ BBF.setFunExec(fun_exec); ++ BBF.setTotalLoops(TotalLoopsValid); ++ BBF.setLoopDepth(LoopDepthValid); ++ BBF.setLoopNumBlocks(LoopNumBlocksValid); ++ BBF.setLocalExitingBlock(LocalExitingBlockValid); ++ BBF.setLocalLatchBlock(LocalLatchBlockValid); ++ BBF.setLocalLoopHeader(LocalLoopHeaderValid); ++ BBF.setNumCalls(NumCallsValid); ++ BBF.setBasicBlockSize(BasicBlockSize); ++ BBF.setNumBasicBlocks(NumBasicBlocks); ++ BBF.setNumLoads(NumLoadsValid); ++ BBF.setHasIndirectCalls(HasIndirectCalls); ++ BBF.setLoopHeader(LoopHeaderValid); ++ BBF.setProcedureType(ProcedureType); ++ BBF.setCount(Count); ++ BBF.setFallthroughCount(FallthroughCount); ++ ++ ++ generateSuccessorFeatures(TrueSuccessor, &BBF); ++ generateSuccessorFeatures(FalseSuccessor, &BBF); ++ ++ FalseSuccessor.reset(); ++ TrueSuccessor.reset(); ++ ++ BBF.setInferenceFeatures(); ++ BB.setFeatures(BBF); ++ ++ BFI.reset(); ++ } ++} + +- uint64_t fun_exec = Function.getExecutionCount(); +- fun_exec = (fun_exec != UINT64_MAX) ? fun_exec : 0; +- Printer << "," << Twine::utohexstr(Function.getAddress()) << "," +- << fun_exec << "," << Function.getFunctionNumber() << "," +- << Function.getOneName() << "," << Function.getPrintName() +- << "," << BranchOffsetStr +- << "\n"; ++void FeatureMiner::generateInstFeatures(BinaryContext &BC, BinaryBasicBlock &BB, BFIPtr const &BFI, int Index) { ++ ++ // Holds the branch opcode info. ++ ++ BFI->CmpOpcode = 0; ++ if (Index > -1) { ++ auto Cmp = BB.begin() + Index; ++ if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { ++ // Holding the branch comparison opcode info. ++ BFI->CmpOpcode = (*Cmp).getOpcode(); ++ auto getOperandType = [&](const MCOperand &Operand) -> int32_t { ++ if (Operand.isReg()) ++ return 0; ++ else if (Operand.isImm()) ++ return 1; ++ else if (Operand.isSFPImm()) ++ return 2; ++ else if (Operand.isExpr()) ++ return 3; ++ else ++ return -1; ++ }; ++ ++ const auto InstInfo = BC.MII->get((*Cmp).getOpcode()); ++ unsigned NumDefs = InstInfo.getNumDefs(); ++ int32_t NumPrimeOperands = ++ MCPlus::getNumPrimeOperands(*Cmp) - NumDefs; ++ switch (NumPrimeOperands) { ++ case 6: { ++ int32_t RBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ int32_t RAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); ++ ++ if (RBType == 0 && RAType == 0) { ++ BFI->OperandRBType = RBType; ++ BFI->OperandRAType = RAType; ++ } else if (RBType == 0 && (RAType == 1 || RAType == 2)) { ++ RAType = getOperandType((*Cmp).getOperand(NumPrimeOperands - 1)); ++ ++ if (RAType != 1 && RAType != 2) { ++ RAType = -1; ++ } + +- //======================================================================== ++ BFI->OperandRBType = RBType; ++ BFI->OperandRAType = RAType; ++ } else { ++ BFI->OperandRAType = -1; ++ BFI->OperandRBType = -1; ++ } ++ break; ++ } ++ case 2: ++ BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 1)); ++ break; ++ case 3: ++ BFI->OperandRBType = getOperandType((*Cmp).getOperand(NumDefs)); ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs + 2)); ++ break; ++ case 1: ++ BFI->OperandRAType = getOperandType((*Cmp).getOperand(NumDefs)); ++ break; ++ default: ++ BFI->OperandRAType = -1; ++ BFI->OperandRBType = -1; ++ break; ++ } + +- // this->BranchesInfoSet.push_back(std::move(BFI)); ++ } else { ++ Index -= 1; ++ for (int Idx = Index; Idx > -1; Idx--) { ++ auto Cmp = BB.begin() + Idx; ++ if (BC.MII->get((*Cmp).getOpcode()).isCompare()) { ++ // Holding the branch comparison opcode info. ++ BFI->CmpOpcode = (*Cmp).getOpcode(); ++ break; ++ } ++ } + } + } + } + +-void FeatureMiner::dumpSuccessorFeatures(raw_ostream &Printer, +- BBIPtr &Successor) { +- int16_t BranchDominates = +- (Successor->BranchDominates.hasValue()) +- ? static_cast(*(Successor->BranchDominates)) +- : -1; ++void FeatureMiner::generateSuccessorFeatures(BBIPtr &Successor, BinaryBasicBlockFeature *BBF) { + +- int16_t BranchPostdominates = +- (Successor->BranchPostdominates.hasValue()) +- ? static_cast(*(Successor->BranchPostdominates)) +- : -1; + + int16_t LoopHeader = (Successor->LoopHeader.hasValue()) + ? static_cast(*(Successor->LoopHeader)) +@@ -690,378 +485,86 @@ void FeatureMiner::dumpSuccessorFeatures(raw_ostream &Printer, + ? static_cast(*(Successor->EndOpcode)) + : -1; + +- int64_t NumLoads = (Successor->NumLoads.hasValue()) +- ? static_cast(*(Successor->NumLoads)) +- : -1; +- +- int64_t NumStores = (Successor->NumStores.hasValue()) +- ? static_cast(*(Successor->NumStores)) +- : -1; +- + int64_t BasicBlockSize = + (Successor->BasicBlockSize.hasValue()) + ? static_cast(*(Successor->BasicBlockSize)) + : -1; + +- int64_t NumCalls = (Successor->NumCalls.hasValue()) +- ? static_cast(*(Successor->NumCalls)) +- : -1; +- +- int64_t NumCallsExit = (Successor->NumCallsExit.hasValue()) +- ? static_cast(*(Successor->NumCallsExit)) +- : -1; +- +- int64_t NumCallsInvoke = +- (Successor->NumCallsInvoke.hasValue()) +- ? static_cast(*(Successor->NumCallsInvoke)) +- : -1; +- +- int64_t NumIndirectCalls = +- (Successor->NumIndirectCalls.hasValue()) +- ? static_cast(*(Successor->NumIndirectCalls)) +- : -1; ++ BBF->setEndOpcodeVec(EndOpcode); ++ BBF->setLoopHeaderVec(LoopHeader); ++ BBF->setBackedgeVec(Backedge); ++ BBF->setExitVec(Exit); ++ BBF->setCallVec(Call); ++ BBF->setBasicBlockSizeVec(BasicBlockSize); + +- int64_t NumTailCalls = (Successor->NumTailCalls.hasValue()) +- ? static_cast(*(Successor->NumTailCalls)) +- : -1; +- +- Printer << "," << BranchDominates << "," << BranchPostdominates << "," +- << EndOpcode << "," << Successor->EndOpcodeStr << "," << LoopHeader +- << "," << Backedge << "," << Exit << "," << Call << "," +- << Successor->FromFunName << "," +- << Twine::utohexstr(Successor->FromBb) << "," << Successor->ToFunName +- << "," << Twine::utohexstr(Successor->ToBb) << "," << NumLoads << "," +- << NumStores << "," << BasicBlockSize << "," << NumCalls << "," +- << NumCallsExit << "," << NumIndirectCalls << "," << NumCallsInvoke +- << "," << NumTailCalls; + } + +-void FeatureMiner::dumpFeatures(raw_ostream &Printer, uint64_t FunctionAddress, +- uint64_t FunctionFrequency) { +- +- for (auto const &BFI : BranchesInfoSet) { +- auto &FalseSuccessor = BFI->FalseSuccessor; +- auto &TrueSuccessor = BFI->TrueSuccessor; +- +- if (!FalseSuccessor && !TrueSuccessor) +- continue; +- +- int16_t ProcedureType = (BFI->ProcedureType.hasValue()) +- ? static_cast(*(BFI->ProcedureType)) +- : -1; +- +- int16_t Direction = +- (BFI->Direction.hasValue()) ? static_cast(*(BFI->Direction)) : -1; +- +- int16_t LoopHeader = (BFI->LoopHeader.hasValue()) +- ? static_cast(*(BFI->LoopHeader)) +- : -1; +- +- int32_t Opcode = +- (BFI->Opcode.hasValue()) ? static_cast(*(BFI->Opcode)) : -1; +- +- int32_t CmpOpcode = (BFI->CmpOpcode.hasValue()) +- ? static_cast(*(BFI->CmpOpcode)) +- : -1; +- +- int64_t Count = +- (BFI->Count.hasValue()) ? static_cast(*(BFI->Count)) : -1; +- +- int64_t MissPredicted = (BFI->MissPredicted.hasValue()) +- ? static_cast(*(BFI->MissPredicted)) +- : -1; +- +- int64_t FallthroughCount = +- (BFI->FallthroughCount.hasValue()) +- ? static_cast(*(BFI->FallthroughCount)) +- : -1; +- +- int64_t FallthroughMissPredicted = +- (BFI->FallthroughMissPredicted.hasValue()) +- ? static_cast(*(BFI->FallthroughMissPredicted)) +- : -1; +- +- int64_t NumOuterLoops = (BFI->NumOuterLoops.hasValue()) +- ? static_cast(*(BFI->NumOuterLoops)) +- : -1; +- int64_t TotalLoops = (BFI->TotalLoops.hasValue()) +- ? static_cast(*(BFI->TotalLoops)) +- : -1; +- int64_t MaximumLoopDepth = +- (BFI->MaximumLoopDepth.hasValue()) +- ? static_cast(*(BFI->MaximumLoopDepth)) +- : -1; +- int64_t LoopDepth = (BFI->LoopDepth.hasValue()) +- ? static_cast(*(BFI->LoopDepth)) +- : -1; +- int64_t LoopNumExitEdges = +- (BFI->LoopNumExitEdges.hasValue()) +- ? static_cast(*(BFI->LoopNumExitEdges)) +- : -1; +- int64_t LoopNumExitBlocks = +- (BFI->LoopNumExitBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumExitBlocks)) +- : -1; +- int64_t LoopNumExitingBlocks = +- (BFI->LoopNumExitingBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumExitingBlocks)) +- : -1; +- int64_t LoopNumLatches = (BFI->LoopNumLatches.hasValue()) +- ? static_cast(*(BFI->LoopNumLatches)) +- : -1; +- int64_t LoopNumBlocks = (BFI->LoopNumBlocks.hasValue()) +- ? static_cast(*(BFI->LoopNumBlocks)) +- : -1; +- int64_t LoopNumBackEdges = +- (BFI->LoopNumBackEdges.hasValue()) +- ? static_cast(*(BFI->LoopNumBackEdges)) +- : -1; +- +- int64_t LocalExitingBlock = +- (BFI->LocalExitingBlock.hasValue()) +- ? static_cast(*(BFI->LocalExitingBlock)) +- : -1; +- +- int64_t LocalLatchBlock = (BFI->LocalLatchBlock.hasValue()) +- ? static_cast(*(BFI->LocalLatchBlock)) +- : -1; +- +- int64_t LocalLoopHeader = (BFI->LocalLoopHeader.hasValue()) +- ? static_cast(*(BFI->LocalLoopHeader)) +- : -1; +- +- int64_t Call = +- (BFI->Call.hasValue()) ? static_cast(*(BFI->Call)) : -1; +- +- int64_t DeltaTaken = (BFI->DeltaTaken.hasValue()) +- ? static_cast(*(BFI->DeltaTaken)) +- : -1; +- +- int64_t NumLoads = (BFI->NumLoads.hasValue()) +- ? static_cast(*(BFI->NumLoads)) +- : -1; +- +- int64_t NumStores = (BFI->NumStores.hasValue()) +- ? static_cast(*(BFI->NumStores)) +- : -1; +- +- int64_t BasicBlockSize = (BFI->BasicBlockSize.hasValue()) +- ? static_cast(*(BFI->BasicBlockSize)) +- : -1; +- +- int64_t BranchOffset = (BFI->BranchOffset.hasValue()) +- ? static_cast(*(BFI->BranchOffset)): -1; +- +- int64_t NumBasicBlocks = (BFI->NumBasicBlocks.hasValue()) +- ? static_cast(*(BFI->NumBasicBlocks)) +- : -1; +- +- int64_t NumCalls = (BFI->NumCalls.hasValue()) +- ? static_cast(*(BFI->NumCalls)) +- : -1; +- +- int64_t NumSelfCalls = (BFI->NumSelfCalls.hasValue()) +- ? static_cast(*(BFI->NumSelfCalls)) +- : -1; +- +- int64_t NumCallsExit = (BFI->NumCallsExit.hasValue()) +- ? static_cast(*(BFI->NumCallsExit)) +- : -1; +- +- int64_t OperandRAType = (BFI->OperandRAType.hasValue()) +- ? static_cast(*(BFI->OperandRAType)) +- : -1; +- +- int64_t OperandRBType = (BFI->OperandRBType.hasValue()) +- ? static_cast(*(BFI->OperandRBType)) +- : -1; +- +- int64_t NumCallsInvoke = (BFI->NumCallsInvoke.hasValue()) +- ? static_cast(*(BFI->NumCallsInvoke)) +- : -1; ++void FeatureMiner::runOnFunctions(BinaryContext &BC){ + +- int64_t NumIndirectCalls = +- (BFI->NumIndirectCalls.hasValue()) +- ? static_cast(*(BFI->NumIndirectCalls)) +- : -1; +- +- int64_t NumTailCalls = (BFI->NumTailCalls.hasValue()) +- ? static_cast(*(BFI->NumTailCalls)) +- : -1; +- +- Printer << BFI->Simple << "," << Opcode << "," << BFI->OpcodeStr << "," +- << Direction << "," << CmpOpcode << "," << BFI->CmpOpcodeStr << "," +- << LoopHeader << "," << ProcedureType << "," << Count << "," +- << MissPredicted << "," << FallthroughCount << "," +- << FallthroughMissPredicted << "," << NumOuterLoops << "," +- << NumCallsExit << "," << TotalLoops << "," << MaximumLoopDepth +- << "," << LoopDepth << "," << LoopNumExitEdges << "," +- << LoopNumExitBlocks << "," << LoopNumExitingBlocks << "," +- << LoopNumLatches << "," << LoopNumBlocks << "," << LoopNumBackEdges +- << "," << LocalExitingBlock << "," << LocalLatchBlock << "," +- << LocalLoopHeader << "," << Call << "," << DeltaTaken << "," +- << NumLoads << "," << NumStores << "," << NumCalls << "," +- << OperandRAType << "," << OperandRBType << "," << BasicBlockSize +- << "," << NumBasicBlocks << "," << NumCallsInvoke << "," +- << NumIndirectCalls << "," << NumTailCalls << "," << NumSelfCalls; +- +- if (FalseSuccessor && TrueSuccessor) { +- dumpSuccessorFeatures(Printer, TrueSuccessor); +- dumpSuccessorFeatures(Printer, FalseSuccessor); +- } +- +- Printer << "," << Twine::utohexstr(FunctionAddress) << "," +- << FunctionFrequency << "\n"; +- } +- BranchesInfoSet.clear(); + } + +-void FeatureMiner::runOnFunctions(BinaryContext &BC) { +- auto FileName = "features_new.csv"; +- outs() << "BOLT-INFO: Starting feature miner pass\n"; +- +- std::error_code EC; +-// raw_fd_ostream Printer(FileName, EC, sys::fs::F_None); +- raw_fd_ostream Printer(FileName, EC, sys::fs::OF_None); +- +- if (EC) { +- errs() << "BOLT-WARNING: " << EC.message() << ", unable to open " +- << FileName << " for output.\n"; +- return; +- } +- +- auto FILENAME = "profile_data_regular.fdata"; +-// raw_fd_ostream Printer2(FILENAME, EC, sys::fs::F_None); +- raw_fd_ostream Printer2(FILENAME, EC, sys::fs::OF_None); +- +- if (EC) { +- dbgs() << "BOLT-WARNING: " << EC.message() << ", unable to open" +- << " " << FILENAME << " for output.\n"; +- return; +- } ++void FeatureMiner::inferenceFeatures(BinaryFunction &Function){ + +- // CSV file header +- Printer << "FUN_TYPE,OPCODE,OPCODE_STR,DIRECTION,CMP_OPCODE,CMP_OPCODE_STR," +- "LOOP_HEADER,PROCEDURE_TYPE," +- "COUNT_TAKEN,MISS_TAKEN,COUNT_NOT_TAKEN,MISS_NOT_TAKEN," +- "NUM_OUTER_LOOPS,NUM_CALLS_EXIT,TOTAL_LOOPS,MAXIMUM_LOOP_DEPTH," +- "LOOP_DEPTH,LOOP_NUM_EXIT_EDGES,LOOP_NUM_EXIT_BLOCKS," +- "LOOP_NUM_EXITING_BLOCKS,LOOP_NUM_LATCHES,LOOP_NUM_BLOCKS," +- "LOOP_NUM_BAKCEDGES,LOCAL_EXITING_BLOCK,LOCAL_LATCH_BLOCK," +- "LOCAL_LOOP_HEADER,CALL,DELTA_TAKEN,NUM_LOADS,NUM_STORES," +- "NUM_CALLS,OPERAND_RA_TYPE,OPERAND_RB_TYPE,BASIC_BLOCK_SIZE," +- "NUM_BASIC_BLOCKS,NUM_CALLS_INVOKE,NUM_INDIRECT_CALLS," +- "NUM_TAIL_CALLS,NUM_SELF_CALLS,TS_DOMINATES,TS_POSTDOMINATES," +- "TS_END_OPCODE,TS_END_OPCODE_STR,TS_LOOP_HEADER,TS_BACKEDGE,TS_" +- "EXIT,TS_CALL," +- "TS_FROM_FUN_NAME,TS_FROM_BB,TS_TO_FUN_NAME,TS_TO_BB,TS_NUM_LOADS," +- "TS_NUM_STORES,TS_BASIC_BLOCK_SIZE,TS_NUM_CALLS,TS_NUM_CALLS_EXIT," +- "TS_NUM_INDIRECT_CALL,TS_NUM_CALLS_INVOKE,TS_NUM_TAIL_CALLS," +- "FS_DOMINATES,FS_POSTDOMINATES,FS_END_OPCODE,FS_END_OPCODE_STR,FS_" +- "LOOP_HEADER," +- "FS_BACKEDGE,FS_EXIT,FS_CALL,FS_FROM_FUN_NAME,FS_FROM_BB," +- "FS_TO_FUN_NAME,FS_TO_BB,FS_NUM_LOADS,FS_NUM_STORES," +- "FS_BASIC_BLOCK_SIZE,FS_NUM_CALLS,FS_NUM_CALLS_EXIT," +- "FS_NUM_INDIRECT_CALL,FS_NUM_CALLS_INVOKE,FS_NUM_TAIL_CALLS," +- "FUN_ENTRY_ADDRESS,FUN_ENTRY_FREQUENCY" +- ",FUN_UNIQUE_NUMBER,FUN_ONE_NAME,FUN_PRINT_NAME," +- "BRANCH_ADDRESS\n"; +- +- auto &BFs = BC.getBinaryFunctions(); + SBI = std::make_unique(); +- for (auto &BFI : BFs) { +- BinaryFunction &Function = BFI.second; + +- if (Function.empty() || (Function.hasValidProfile() && opts::IncludeValidProfile)) +- continue; +- +- if (!Function.isLoopFree()) { +- const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); +- SBI->findLoopEdgesInfo(LoopsInfo); +- } +- extractFeatures(Function, BC, Printer); +- +- SBI->clear(); ++ if (Function.empty()) ++ return; + +- // dumpProfileData(Function, Printer2); ++ if (!Function.isLoopFree()) { ++ const BinaryLoopInfo &LoopsInfo = Function.getLoopInfo(); ++ SBI->findLoopEdgesInfo(LoopsInfo); + } + +- outs() << "BOLT-INFO: Dumping two-way conditional branches' features" +- << " at " << FileName << "\n"; +-} +- +-/*void FeatureMiner::dumpProfileData(BinaryFunction &Function, +- raw_ostream &Printer) { +- + BinaryContext &BC = Function.getBinaryContext(); ++ extractFeatures(Function, BC); + +- std::string FromFunName = Function.getPrintName(); +- for (auto &BB : Function) { +- auto LastInst = BB.getLastNonPseudoInstr(); +- +- for (auto &Inst : BB) { +- if (!BC.MIB->isCall(Inst) && !BC.MIB->isBranch(Inst) && +- LastInst != (&Inst)) +- continue; +- +- auto Offset = BC.MIB->tryGetAnnotationAs(Inst, "Offset"); +- +- if (!Offset) +- continue; +- +- uint64_t TakenFreqEdge = 0; +- auto FromBb = Offset.get(); +- std::string ToFunName; +- uint32_t ToBb; +- +- if (BC.MIB->isCall(Inst)) { +- auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst); +- if (!CalleeSymbol) +- continue; +- +- ToFunName = CalleeSymbol->getName(); +- ToBb = 0; ++ SBI->clear(); ++} + +- if (BC.MIB->getConditionalTailCall(Inst)) { ++void FeatureMiner::generateProfileFeatures(BinaryBasicBlock *BB, BinaryBasicBlockFeature *BBF) { ++ int32_t parentChildNum, parentCount, childParentNum, childCount; + +- if (BC.MIB->hasAnnotation(Inst, "CTCTakenCount")) { +- auto CountAnnt = +- BC.MIB->tryGetAnnotationAs(Inst, "CTCTakenCount"); +- if (CountAnnt) { +- TakenFreqEdge = (*CountAnnt); +- } +- } +- } else { +- if (BC.MIB->hasAnnotation(Inst, "Count")) { +- auto CountAnnt = +- BC.MIB->tryGetAnnotationAs(Inst, "Count"); +- if (CountAnnt) { +- TakenFreqEdge = (*CountAnnt); +- } +- } +- } ++ if (BB->ParentSet.size() == 0) { ++ parentChildNum = -1; ++ parentCount = -1; ++ } else { ++ parentChildNum = std::numeric_limits::max(); ++ parentCount = 0; ++ for (BinaryBasicBlock *parent: BB->ParentSet) { ++ if (parent->ChildrenSet.size() < parentChildNum) { ++ parentChildNum = parent->ChildrenSet.size(); ++ parentCount = parent->getExecutionCount(); ++ } else if (parent->ChildrenSet.size() == parentChildNum && parent->getExecutionCount() > parentCount) { ++ parentCount = parent->getExecutionCount(); ++ } ++ } ++ } + +- if (TakenFreqEdge > 0) +- Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) +- << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) << " " +- << 0 << " " << TakenFreqEdge << "\n"; +- } else { +- for (BinaryBasicBlock *SuccBB : BB.successors()) { +- TakenFreqEdge = BB.getBranchInfo(*SuccBB).Count; +- BinaryFunction *ToFun = SuccBB->getFunction(); +- ToFunName = ToFun->getPrintName(); +- ToBb = SuccBB->getInputOffset(); +- +- if (TakenFreqEdge > 0) +- Printer << "1 " << FromFunName << " " << Twine::utohexstr(FromBb) +- << " 1 " << ToFunName << " " << Twine::utohexstr(ToBb) +- << " " << 0 << " " << TakenFreqEdge << "\n"; +- } ++ if (BB->ChildrenSet.size() == 0) { ++ childParentNum = -1; ++ childCount = -1; ++ } else { ++ childParentNum = std::numeric_limits::max(); ++ childCount = 0; ++ for (BinaryBasicBlock *child: BB->ChildrenSet) { ++ if (child->ParentSet.size() < childParentNum) { ++ childParentNum = child->ParentSet.size(); ++ childCount = child->getExecutionCount(); ++ } else if (child->ParentSet.size() == childParentNum && child->getExecutionCount() > childCount) { ++ childCount = child->getExecutionCount(); + } + } + } ++ ++ int64_t parentCountCatch = parentCount > 0 ? 1 : 0; ++ int64_t childCountCatch = childCount > 0 ? 1 : 0; ++ ++ BBF->setParentChildNum(parentChildNum); ++ BBF->setParentCount(parentCountCatch); ++ BBF->setChildParentNum(childParentNum); ++ BBF->setChildCount(childCountCatch); ++ + } +-*/ + + } // namespace bolt + } // namespace llvm +\ No newline at end of file +diff --git a/bolt/lib/Passes/StaticBranchInfo.cpp b/bolt/lib/Passes/StaticBranchInfo.cpp +index 13426b397..5a3e0ec9c 100644 +--- a/bolt/lib/Passes/StaticBranchInfo.cpp ++++ b/bolt/lib/Passes/StaticBranchInfo.cpp +@@ -15,10 +15,9 @@ + // + //===----------------------------------------------------------------------===// + +-// #include "Passes/StaticBranchInfo.h" +-// #include "BinaryBasicBlock.h" + #include "bolt/Passes/StaticBranchInfo.h" + #include "bolt/Core/BinaryBasicBlock.h" ++#include "bolt/Core/BinaryLoop.h" + + namespace llvm { + namespace bolt { +@@ -90,7 +89,6 @@ bool StaticBranchInfo::isExitEdge(const BinaryLoop::Edge &CFGEdge) const { + + bool StaticBranchInfo::isExitEdge(const BinaryBasicBlock *SrcBB, + const BinaryBasicBlock *DstBB) const { +-// const BinaryLoop::Edge CFGEdge = std::make_pair(SrcBB, DstBB); + const BinaryLoop::Edge CFGEdge = std::make_pair(const_cast(SrcBB), const_cast(DstBB)); + return isExitEdge(CFGEdge); + } +@@ -159,4 +157,4 @@ void StaticBranchInfo::clear() { + } + + } // namespace bolt +-} // namespace llvm +\ No newline at end of file ++} // namespace llvm +diff --git a/bolt/lib/Profile/CMakeLists.txt b/bolt/lib/Profile/CMakeLists.txt +index f4397e331..2273f4572 100644 +--- a/bolt/lib/Profile/CMakeLists.txt ++++ b/bolt/lib/Profile/CMakeLists.txt +@@ -11,6 +11,7 @@ add_llvm_library(LLVMBOLTProfile + Support + ) + ++ + target_link_libraries(LLVMBOLTProfile + PRIVATE + LLVMBOLTCore +diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp +index a51201d3b..2ae715049 100644 +--- a/bolt/lib/Profile/DataReader.cpp ++++ b/bolt/lib/Profile/DataReader.cpp +@@ -11,33 +11,48 @@ + // + //===----------------------------------------------------------------------===// + +-#include "bolt/Profile/DataReader.h" + #include "bolt/Core/BinaryFunction.h" ++#include "bolt/Passes/FeatureMiner.h" + #include "bolt/Passes/MCF.h" ++#include "bolt/Profile/DataReader.h" + #include "bolt/Utils/Utils.h" + #include "llvm/Support/CommandLine.h" + #include "llvm/Support/Debug.h" + #include "llvm/Support/Errc.h" ++#include ++#include ++#include + #include ++#include + +-#undef DEBUG_TYPE ++#undef DEBUG_TYPE + #define DEBUG_TYPE "bolt-prof" + + using namespace llvm; + + namespace opts { +- ++extern cl::opt BlockCorrection; + extern cl::OptionCategory BoltCategory; + extern llvm::cl::opt Verbosity; + +-static cl::opt +-DumpData("dump-data", +- cl::desc("dump parsed bolt data for debugging"), +- cl::Hidden, +- cl::cat(BoltCategory)); ++static cl::opt InputModelFilename("model-path", ++ cl::desc(""), ++ cl::Optional, ++ cl::cat(BoltCategory)); ++ ++static cl::opt DumpData("dump-data", ++ cl::desc("dump parsed bolt data for debugging"), ++ cl::Hidden, cl::cat(BoltCategory)); + + } // namespace opts + ++extern "C" { ++typedef void *(*CreateONNXRunnerFunc)(const char *); ++typedef void (*DeleteONNXRunnerFunc)(void *); ++typedef float (*RunONNXModelFunc)(void *, std::vector &, ++ std::vector &, std::vector &); ++} ++ + namespace llvm { + namespace bolt { + +@@ -322,6 +337,16 @@ Error DataReader::readProfilePreCFG(BinaryContext &BC) { + } + + Error DataReader::readProfile(BinaryContext &BC) { ++ ++ if (opts::BlockCorrection) { ++ if (opts::InputModelFilename.empty()) { ++ outs() << "llvm-bolt: expected -model-path= option.\n"; ++ exit(1); ++ } else { ++ DataReader::initializeONNXRunner(opts::InputModelFilename); ++ } ++ } ++ + for (auto &BFI : BC.getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; + readProfile(Function); +@@ -333,6 +358,12 @@ Error DataReader::readProfile(BinaryContext &BC) { + ++NumUnused; + BC.setNumUnusedProfiledObjects(NumUnused); + ++ if (opts::BlockCorrection) { ++ uint64_t modified_total = DataReader::getModifiedBBTotal(); ++ outs() << "BOLT-INFO: total modified CFG BB count number is " ++ << modified_total << ".\n"; ++ } ++ + return Error::success(); + } + +@@ -564,11 +595,75 @@ float DataReader::evaluateProfileData(BinaryFunction &BF, + return MatchRatio; + } + ++void generateChildrenParentCount(BinaryBasicBlock *BB) { ++ typedef GraphTraits GraphT; ++ ++ for (typename GraphT::ChildIteratorType CI = GraphT::child_begin(BB), ++ E = GraphT::child_end(BB); ++ CI != E; ++CI) { ++ typename GraphT::NodeRef Child = *CI; ++ BB->ChildrenSet.insert(Child); ++ Child->ParentSet.insert(BB); ++ } ++} ++ ++void generateChildrenParentCount(BinaryFunction &BF) { ++ for (BinaryBasicBlock &BB : BF) { ++ generateChildrenParentCount(&BB); ++ } ++} ++ ++uint64_t estimateBBCount(DataReader *dataReaderRef, BinaryBasicBlock *BB, ++ float threshold) { ++ uint64_t modified = 0; ++ std::vector input_string; ++ std::vector input_int64; ++ std::vector input_float; ++ ++ BinaryBasicBlockFeature BBF = BB->getFeatures(); ++ input_int64 = BBF.getInferenceFeatures(); ++ ++ if (input_int64.empty()) { ++ return 0; ++ } ++ ++ float model_pred = ++ dataReaderRef->ONNXInference(input_string, input_int64, input_float); ++ if (BB->getExecutionCount() == 0 && model_pred >= threshold) { ++ uint64_t min_neighbor_count = std::numeric_limits::max(); ++ for (BinaryBasicBlock *parent : BB->ParentSet) { ++ if (parent->getExecutionCount() > 0 && ++ parent->getExecutionCount() < min_neighbor_count) ++ min_neighbor_count = parent->getExecutionCount(); ++ } ++ for (BinaryBasicBlock *child : BB->ChildrenSet) { ++ if (child->getExecutionCount() > 0 && ++ child->getExecutionCount() < min_neighbor_count) ++ min_neighbor_count = child->getExecutionCount(); ++ } ++ if (min_neighbor_count != std::numeric_limits::max()) { ++ BB->setExecutionCount(min_neighbor_count); ++ modified = 1; ++ } ++ } ++ return modified; ++} ++ ++uint64_t estimateBBCount(DataReader *dataReaderRef, BinaryFunction &BF, ++ float threshold) { ++ uint64_t modified_total_func = 0; ++ const auto &Order = BF.dfs(); ++ for (auto *BBA : Order) { ++ auto &BB = *BBA; ++ modified_total_func += estimateBBCount(dataReaderRef, &BB, threshold); ++ } ++ return modified_total_func; ++} ++ + void DataReader::readSampleData(BinaryFunction &BF) { + FuncSampleData *SampleDataOrErr = getFuncSampleData(BF.getNames()); + if (!SampleDataOrErr) + return; +- + // Basic samples mode territory (without LBR info) + // First step is to assign BB execution count based on samples from perf + BF.ProfileMatchRatio = 1.0f; +@@ -609,6 +704,17 @@ void DataReader::readSampleData(BinaryFunction &BF) { + + BF.ExecutionCount = TotalEntryCount; + ++ if (opts::BlockCorrection) { ++ generateChildrenParentCount(BF); ++ std::unique_ptr FM = ++ std::make_unique(opts::BlockCorrection); ++ FM->inferenceFeatures(BF); ++ ++ float threshold = DataReader::getThreshold(); ++ uint64_t modified_total_func = estimateBBCount(this, BF, threshold); ++ DataReader::addModifiedBBTotal(modified_total_func); ++ } ++ + estimateEdgeCounts(BF); + } + +@@ -1324,7 +1430,7 @@ std::vector fetchMapEntriesRegex( + return AllData; + } + +-} ++} // namespace + + bool DataReader::mayHaveProfileData(const BinaryFunction &Function) { + if (getBranchData(Function) || getMemData(Function)) +diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp +index fd21724c5..3bed5c16e 100644 +--- a/bolt/lib/Rewrite/RewriteInstance.cpp ++++ b/bolt/lib/Rewrite/RewriteInstance.cpp +@@ -16,6 +16,7 @@ + #include "bolt/Core/ParallelUtilities.h" + #include "bolt/Core/Relocation.h" + #include "bolt/Passes/CacheMetrics.h" ++#include "bolt/Passes/FeatureMiner.h" + #include "bolt/Passes/ReorderFunctions.h" + #include "bolt/Profile/BoltAddressTranslation.h" + #include "bolt/Profile/DataAggregator.h" +@@ -104,6 +105,12 @@ cl::opt DumpDotAll( + "enable '-print-loops' for color-coded blocks"), + cl::Hidden, cl::cat(BoltCategory)); + ++cl::opt BlockCorrection( ++ "block-correction", ++ cl::desc("capture features useful for ML model to inference the count on the binary basic block" ++ " and correct them on CFG."), ++ cl::ZeroOrMore, cl::cat(BoltOptCategory)); ++ + static cl::list + ForceFunctionNames("funcs", + cl::CommaSeparated, +-- +2.33.0 + diff --git a/llvm-bolt.spec b/llvm-bolt.spec index 52c1c3c..b45c348 100644 --- a/llvm-bolt.spec +++ b/llvm-bolt.spec @@ -15,7 +15,7 @@ Name: llvm-bolt Version: %{bolt_version} -Release: 2 +Release: 3 Summary: BOLT is a post-link optimizer developed to speed up large applications License: Apache 2.0 URL: https://github.com/llvm/llvm-project/tree/main/bolt @@ -25,6 +25,8 @@ Source0: https://github.com/llvm/llvm-project/releases/download/llvmorg-% Patch1: 0001-AArch64-fix-bug-55005-handle-DW_CFA_GNU_NegateRAState.patch Patch2: 0002-AArch64-Add-AArch64-support-for-hugify.patch Patch3: 0003-AArch64-Add-AArch64-support-for-inline.patch +Patch4: 0004-Added-open-source-code-related-to-feature-extracting.patch +Patch5: 0005-Add-block-correction-optimization.patch BuildRequires: gcc BuildRequires: gcc-c++ @@ -140,6 +142,12 @@ rm -f %{buildroot}/%{_builddir}/%{bolt_srcdir}/lib/lib*.a %doc %{install_docdir} %changelog +* Wed Jun 5 2024 Zhou Zeping 15.0.7-3 +- Type:Update +- ID:NA +- SUG:NA +- DESC: Add CFG block count correction optimization. + * Thu Sep 7 2023 Xiong Zhou 15.0.7-2 - Type:Update - ID:NA