diff options
Diffstat (limited to 'include/llvm/CodeGen')
73 files changed, 3950 insertions, 3183 deletions
diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h index b2cc7047b3..5f2bbd62a0 100644 --- a/include/llvm/CodeGen/Analysis.h +++ b/include/llvm/CodeGen/Analysis.h @@ -17,19 +17,18 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/ISDOpcodes.h" -#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/Support/CallSite.h" namespace llvm { - class GlobalVariable; -class TargetLowering; class TargetLoweringBase; class SDNode; class SDValue; class SelectionDAG; +class TargetLowering; +struct EVT; /// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence /// of insertvalue or extractvalue indices that identify a member, return diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index 4bda0f1603..d96d81072d 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -16,13 +16,16 @@ #ifndef LLVM_CODEGEN_ASMPRINTER_H #define LLVM_CODEGEN_ASMPRINTER_H +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/IR/InlineAsm.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" namespace llvm { + class AsmPrinterHandler; class BlockAddress; + class ByteStreamer; class GCStrategy; class Constant; class ConstantArray; @@ -41,9 +44,11 @@ namespace llvm { class MCAsmInfo; class MCCFIInstruction; class MCContext; + class MCInst; class MCInstrInfo; class MCSection; class MCStreamer; + class MCSubtargetInfo; class MCSymbol; class MDNode; class DwarfDebug; @@ -110,20 +115,28 @@ namespace llvm { /// function. MachineLoopInfo *LI; + struct HandlerInfo { + AsmPrinterHandler *Handler; + const char *TimerName, *TimerGroupName; + HandlerInfo(AsmPrinterHandler *Handler, const char *TimerName, + const char *TimerGroupName) + : Handler(Handler), TimerName(TimerName), + TimerGroupName(TimerGroupName) {} + }; + /// Handlers - a vector of all debug/EH info emitters we should use. + /// This vector maintains ownership of the emitters. + SmallVector<HandlerInfo, 1> Handlers; + /// DD - If the target supports dwarf debug info, this pointer is non-null. DwarfDebug *DD; - /// DE - If the target supports dwarf exception info, this pointer is - /// non-null. - DwarfException *DE; - protected: explicit AsmPrinter(TargetMachine &TM, MCStreamer &Streamer); public: virtual ~AsmPrinter(); - const DwarfDebug *getDwarfDebug() const { return DD; } + DwarfDebug *getDwarfDebug() { return DD; } /// isVerbose - Return true if assembly output should contain comments. /// @@ -139,12 +152,20 @@ namespace llvm { /// getDataLayout - Return information about data layout. const DataLayout &getDataLayout() const; + /// getSubtargetInfo - Return information about subtarget. + const MCSubtargetInfo &getSubtargetInfo() const; + + void EmitToStreamer(MCStreamer &S, const MCInst &Inst); + /// getTargetTriple - Return the target triple string. StringRef getTargetTriple() const; /// getCurrentSection() - Return the current section we are emitting to. const MCSection *getCurrentSection() const; + void getNameWithPrefix(SmallVectorImpl<char> &Name, + const GlobalValue *GV) const; + MCSymbol *getSymbol(const GlobalValue *GV) const; //===------------------------------------------------------------------===// @@ -153,20 +174,20 @@ namespace llvm { /// getAnalysisUsage - Record analysis usage. /// - void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; /// doInitialization - Set up the AsmPrinter when we are working on a new /// module. If your pass overrides this, it must make sure to explicitly /// call this implementation. - bool doInitialization(Module &M); + bool doInitialization(Module &M) override; /// doFinalization - Shut down the asmprinter. If you override this in your /// pass, you must make sure to call it explicitly. - bool doFinalization(Module &M); + bool doFinalization(Module &M) override; /// runOnMachineFunction - Emit the specified function out to the /// OutStreamer. - virtual bool runOnMachineFunction(MachineFunction &MF) { + bool runOnMachineFunction(MachineFunction &MF) override { SetupMachineFunction(MF); EmitFunctionHeader(); EmitFunctionBody(); @@ -189,7 +210,7 @@ namespace llvm { /// function. void EmitFunctionBody(); - void emitPrologLabel(const MachineInstr &MI); + void emitCFIInstruction(const MachineInstr &MI); enum CFIMoveType { CFI_M_None, @@ -200,11 +221,6 @@ namespace llvm { bool needsSEHMoves(); - /// needsRelocationsForDwarfStringPool - Specifies whether the object format - /// expects to use relocations to refer to debug entries. Alternatively we - /// emit section offsets in bytes from the start of the string pool. - bool needsRelocationsForDwarfStringPool() const; - /// EmitConstantPool - Print to the current output stream assembly /// representations of the constants in the constant pool MCP. This is /// used to print out constants which have been "spilled to memory" by @@ -270,6 +286,9 @@ namespace llvm { llvm_unreachable("EmitInstruction not implemented"); } + /// GetCPISymbol - Return the symbol for the specified constant pool entry. + virtual MCSymbol *GetCPISymbol(unsigned CPID) const; + virtual void EmitFunctionEntryLabel(); virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV); @@ -298,27 +317,21 @@ namespace llvm { /// GetTempSymbol - Return the MCSymbol corresponding to the assembler /// temporary label with the specified stem and unique ID. - MCSymbol *GetTempSymbol(StringRef Name, unsigned ID) const; + MCSymbol *GetTempSymbol(Twine Name, unsigned ID) const; /// GetTempSymbol - Return an assembler temporary label with the specified /// stem. - MCSymbol *GetTempSymbol(StringRef Name) const; - + MCSymbol *GetTempSymbol(Twine Name) const; - /// GetSymbolWithGlobalValueBase - Return the MCSymbol for a symbol with - /// global value name as its base, with the specified suffix, and where the - /// symbol is forced to have private linkage if ForcePrivate is true. - MCSymbol *GetSymbolWithGlobalValueBase(const GlobalValue *GV, - StringRef Suffix, - bool ForcePrivate = true) const; + /// Return the MCSymbol for a private symbol with global value name as its + /// base, with the specified suffix. + MCSymbol *getSymbolWithGlobalValueBase(const GlobalValue *GV, + StringRef Suffix) const; /// GetExternalSymbolSymbol - Return the MCSymbol for the specified /// ExternalSymbol. MCSymbol *GetExternalSymbolSymbol(StringRef Sym) const; - /// GetCPISymbol - Return the symbol for the specified constant pool entry. - MCSymbol *GetCPISymbol(unsigned CPID) const; - /// GetJTISymbol - Return the symbol for the specified jump table entry. MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const; @@ -418,7 +431,7 @@ namespace llvm { virtual unsigned getISAEncoding() { return 0; } /// EmitDwarfRegOp - Emit dwarf register operation. - virtual void EmitDwarfRegOp(const MachineLocation &MLoc, + virtual void EmitDwarfRegOp(ByteStreamer &BS, const MachineLocation &MLoc, bool Indirect) const; //===------------------------------------------------------------------===// @@ -460,6 +473,15 @@ namespace llvm { unsigned AsmVariant, const char *ExtraCode, raw_ostream &OS); + /// Let the target do anything it needs to do after emitting inlineasm. + /// This callback can be used restore the original mode in case the + /// inlineasm contains directives to switch modes. + /// \p StartInfo - the original subtarget info before inline asm + /// \p EndInfo - the final subtarget info after parsing the inline asm, + /// or NULL if the value is unknown. + virtual void emitInlineAsmEnd(const MCSubtargetInfo &StartInfo, + const MCSubtargetInfo *EndInfo) const; + private: /// Private state for PrintSpecial() // Assign a unique ID to this machine instruction. diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index a18f433bda..50bbb0d6b5 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -18,14 +18,14 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/CallingConv.h" #include "llvm/Target/TargetCallingConv.h" namespace llvm { - class TargetRegisterInfo; - class TargetMachine; - class CCState; +class CCState; +class MVT; +class TargetMachine; +class TargetRegisterInfo; /// CCValAssign - Represent assignment of one arg/retval to a location. class CCValAssign { @@ -39,6 +39,7 @@ public: VExt, // The value is vector-widened in the location. // FIXME: Not implemented yet. Code that uses AExt to mean // vector-widen should be fixed to use VExt instead. + FPExt, // The floating-point value is fp-extended in the location. Indirect // The location contains pointer to the value. // TODO: a subset of the value is in the location. }; @@ -347,6 +348,15 @@ public: return AllocateStack(Size, Align); } + /// Version of AllocateStack with list of extra registers to be shadowed. + /// Note that, unlike AllocateReg, this shadows ALL of the shadow registers. + unsigned AllocateStack(unsigned Size, unsigned Align, + const uint16_t *ShadowRegs, unsigned NumShadowRegs) { + for (unsigned i = 0; i < NumShadowRegs; ++i) + MarkAllocated(ShadowRegs[i]); + return AllocateStack(Size, Align); + } + // HandleByVal - Allocate a stack slot large enough to pass an argument by // value. The size and alignment information of the argument is encoded in its // parameter attribute. @@ -394,6 +404,11 @@ public: ByValRegs.clear(); } + // Rewind byval registers tracking info. + void rewindByValRegsInfo() { + InRegsParamsProceed = 0; + } + ParmContext getCallOrPrologue() const { return CallOrPrologue; } private: diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h index bc8dce3335..02a4bb5537 100644 --- a/include/llvm/CodeGen/CommandFlags.h +++ b/include/llvm/CodeGen/CommandFlags.h @@ -19,6 +19,7 @@ #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include <string> using namespace llvm; @@ -85,9 +86,6 @@ FileType("filetype", cl::init(TargetMachine::CGFT_AssemblyFile), "Emit nothing, for performance testing"), clEnumValEnd)); -cl::opt<bool> DisableDotLoc("disable-dot-loc", cl::Hidden, - cl::desc("Do not use .loc entries")); - cl::opt<bool> DisableCFI("disable-cfi", cl::Hidden, cl::desc("Do not use .cfi_* directives")); @@ -210,4 +208,30 @@ cl::opt<std::string> StartAfter("start-after", cl::value_desc("pass-name"), cl::init("")); +// Common utility function tightly tied to the options listed here. Initializes +// a TargetOptions object with CodeGen flags and returns it. +static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { + TargetOptions Options; + Options.LessPreciseFPMADOption = EnableFPMAD; + Options.NoFramePointerElim = DisableFPElim; + Options.AllowFPOpFusion = FuseFPOps; + Options.UnsafeFPMath = EnableUnsafeFPMath; + Options.NoInfsFPMath = EnableNoInfsFPMath; + Options.NoNaNsFPMath = EnableNoNaNsFPMath; + Options.HonorSignDependentRoundingFPMathOption = + EnableHonorSignDependentRoundingFPMath; + Options.UseSoftFloat = GenerateSoftFloatCalls; + if (FloatABIForCalls != FloatABI::Default) + Options.FloatABIType = FloatABIForCalls; + Options.NoZerosInBSS = DontPlaceZerosInBSS; + Options.GuaranteedTailCallOpt = EnableGuaranteedTailCallOpt; + Options.DisableTailCalls = DisableTailCalls; + Options.StackAlignmentOverride = OverrideStackAlignment; + Options.TrapFuncName = TrapFuncName; + Options.PositionIndependentExecutable = EnablePIE; + Options.EnableSegmentedStacks = SegmentedStacks; + Options.UseInitArray = UseInitArray; + return Options; +} + #endif diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h index e8a4a2d2d8..2899fe1ab4 100644 --- a/include/llvm/CodeGen/EdgeBundles.h +++ b/include/llvm/CodeGen/EdgeBundles.h @@ -55,8 +55,8 @@ public: void view() const; private: - virtual bool runOnMachineFunction(MachineFunction&); - virtual void getAnalysisUsage(AnalysisUsage&) const; + bool runOnMachineFunction(MachineFunction&) override; + void getAnalysisUsage(AnalysisUsage&) const override; }; /// Specialize WriteGraph, the standard implementation won't work. diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h index 1e0ef6b545..aeffbd41ca 100644 --- a/include/llvm/CodeGen/FastISel.h +++ b/include/llvm/CodeGen/FastISel.h @@ -17,22 +17,22 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/ValueTypes.h" namespace llvm { class AllocaInst; class Constant; class ConstantFP; +class DataLayout; class FunctionLoweringInfo; class Instruction; class LoadInst; +class MVT; class MachineConstantPool; +class MachineFrameInfo; class MachineFunction; class MachineInstr; -class MachineFrameInfo; class MachineRegisterInfo; -class DataLayout; class TargetInstrInfo; class TargetLibraryInfo; class TargetLowering; @@ -51,9 +51,9 @@ protected: MachineRegisterInfo &MRI; MachineFrameInfo &MFI; MachineConstantPool &MCP; - DebugLoc DL; + DebugLoc DbgLoc; const TargetMachine &TM; - const DataLayout &TD; + const DataLayout &DL; const TargetInstrInfo &TII; const TargetLowering &TLI; const TargetRegisterInfo &TRI; @@ -87,7 +87,7 @@ public: void startNewBlock(); /// Return current debug location information. - DebugLoc getCurDebugLoc() const { return DL; } + DebugLoc getCurDebugLoc() const { return DbgLoc; } /// Do "fast" instruction selection for function arguments and append machine /// instructions to the current block. Return true if it is successful. diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index 50d320f8e8..06e7aaaf22 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -21,7 +21,6 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -41,6 +40,8 @@ class MachineBasicBlock; class MachineFunction; class MachineModuleInfo; class MachineRegisterInfo; +class SelectionDAG; +class MVT; class TargetLowering; class Value; @@ -125,7 +126,7 @@ public: /// set - Initialize this FunctionLoweringInfo with the given Function /// and its associated MachineFunction. /// - void set(const Function &Fn, MachineFunction &MF); + void set(const Function &Fn, MachineFunction &MF, SelectionDAG *DAG); /// clear - Clear out all the function-specific state. This returns this /// FunctionLoweringInfo to an empty state, ready to be used for a diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h index 1070d29f73..ea9454259e 100644 --- a/include/llvm/CodeGen/GCMetadata.h +++ b/include/llvm/CodeGen/GCMetadata.h @@ -35,8 +35,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" -#include "llvm/Support/DebugLoc.h" namespace llvm { class AsmPrinter; diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h index 456d2dcb51..bb170c85cb 100644 --- a/include/llvm/CodeGen/GCs.h +++ b/include/llvm/CodeGen/GCs.h @@ -17,13 +17,13 @@ namespace llvm { class GCStrategy; class GCMetadataPrinter; - + /// FIXME: Collector instances are not useful on their own. These no longer /// serve any purpose except to link in the plugins. - + /// Creates an ocaml-compatible garbage collector. void linkOcamlGC(); - + /// Creates an ocaml-compatible metadata printer. void linkOcamlGCPrinter(); @@ -32,7 +32,7 @@ namespace llvm { /// Creates an erlang-compatible metadata printer. void linkErlangGCPrinter(); - + /// Creates a shadow stack garbage collector. This collector requires no code /// generator support. void linkShadowStackGC(); diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 48a0523dc6..89b0908625 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -603,7 +603,7 @@ namespace ISD { /// This corresponds to "load atomic" instruction. ATOMIC_LOAD, - /// OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val) + /// OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) /// This corresponds to "store atomic" instruction. ATOMIC_STORE, @@ -702,6 +702,8 @@ namespace ISD { LAST_LOADEXT_TYPE }; + NodeType getExtForLoadExtType(LoadExtType); + //===--------------------------------------------------------------------===// /// ISD::CondCode enum - These are ordered carefully to make the bitfields /// below work out, when considering SETFALSE (something that never exists diff --git a/include/llvm/CodeGen/IntrinsicLowering.h b/include/llvm/CodeGen/IntrinsicLowering.h index 68389dde49..9e6ab7d459 100644 --- a/include/llvm/CodeGen/IntrinsicLowering.h +++ b/include/llvm/CodeGen/IntrinsicLowering.h @@ -24,13 +24,13 @@ namespace llvm { class DataLayout; class IntrinsicLowering { - const DataLayout& TD; + const DataLayout& DL; bool Warned; public: - explicit IntrinsicLowering(const DataLayout &td) : - TD(td), Warned(false) {} + explicit IntrinsicLowering(const DataLayout &DL) : + DL(DL), Warned(false) {} /// AddPrototypes - This method, if called, causes all of the prototypes /// that might be needed by an intrinsic lowering implementation to be diff --git a/include/llvm/CodeGen/JITCodeEmitter.h b/include/llvm/CodeGen/JITCodeEmitter.h index 9a73214186..bb0df2e277 100644 --- a/include/llvm/CodeGen/JITCodeEmitter.h +++ b/include/llvm/CodeGen/JITCodeEmitter.h @@ -51,7 +51,7 @@ class Function; /// occurred, more memory is allocated, and we reemit the code into it. /// class JITCodeEmitter : public MachineCodeEmitter { - virtual void anchor(); + void anchor() override; public: virtual ~JITCodeEmitter() {} @@ -59,15 +59,15 @@ public: /// about to be code generated. This initializes the BufferBegin/End/Ptr /// fields. /// - virtual void startFunction(MachineFunction &F) = 0; + void startFunction(MachineFunction &F) override = 0; /// finishFunction - This callback is invoked when the specified function has /// finished code generation. If a buffer overflow has occurred, this method /// returns true (the callee is required to try again), otherwise it returns /// false. /// - virtual bool finishFunction(MachineFunction &F) = 0; - + bool finishFunction(MachineFunction &F) override = 0; + /// allocIndirectGV - Allocates and fills storage for an indirect /// GlobalValue, and returns the address. virtual void *allocIndirectGV(const GlobalValue *GV, @@ -248,12 +248,12 @@ public: /// emitLabel - Emits a label - virtual void emitLabel(MCSymbol *Label) = 0; + void emitLabel(MCSymbol *Label) override = 0; /// allocateSpace - Allocate a block of space in the current output buffer, /// returning null (and setting conditions to indicate buffer overflow) on /// failure. Alignment is the alignment in bytes of the buffer desired. - virtual void *allocateSpace(uintptr_t Size, unsigned Alignment) { + void *allocateSpace(uintptr_t Size, unsigned Alignment) override { emitAlignment(Alignment); void *Result; @@ -278,18 +278,18 @@ public: /// StartMachineBasicBlock - This should be called by the target when a new /// basic block is about to be emitted. This way the MCE knows where the /// start of the block is, and can implement getMachineBasicBlockAddress. - virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) = 0; - + void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0; + /// getCurrentPCValue - This returns the address that the next emitted byte /// will be output to. /// - virtual uintptr_t getCurrentPCValue() const { + uintptr_t getCurrentPCValue() const override { return (uintptr_t)CurBufferPtr; } /// getCurrentPCOffset - Return the offset from the start of the emitted /// buffer that we are currently writing to. - uintptr_t getCurrentPCOffset() const { + uintptr_t getCurrentPCOffset() const override { return CurBufferPtr-BufferBegin; } @@ -298,38 +298,39 @@ public: /// creates jump tables or constant pools in memory on the fly while the /// object code emitters rely on a linker to have real addresses and should /// use relocations instead. - bool earlyResolveAddresses() const { return true; } + bool earlyResolveAddresses() const override { return true; } /// addRelocation - Whenever a relocatable address is needed, it should be /// noted with this interface. - virtual void addRelocation(const MachineRelocation &MR) = 0; - + void addRelocation(const MachineRelocation &MR) override = 0; + /// FIXME: These should all be handled with relocations! /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in /// the constant pool that was last emitted with the emitConstantPool method. /// - virtual uintptr_t getConstantPoolEntryAddress(unsigned Index) const = 0; + uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0; /// getJumpTableEntryAddress - Return the address of the jump table with index /// 'Index' in the function that last called initJumpTableInfo. /// - virtual uintptr_t getJumpTableEntryAddress(unsigned Index) const = 0; - + uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0; + /// getMachineBasicBlockAddress - Return the address of the specified /// MachineBasicBlock, only usable after the label for the MBB has been /// emitted. /// - virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const= 0; + uintptr_t + getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0; /// getLabelAddress - Return the address of the specified Label, only usable /// after the Label has been emitted. /// - virtual uintptr_t getLabelAddress(MCSymbol *Label) const = 0; - + uintptr_t getLabelAddress(MCSymbol *Label) const override = 0; + /// Specifies the MachineModuleInfo object. This is used for exception handling /// purposes. - virtual void setModuleInfo(MachineModuleInfo* Info) = 0; + void setModuleInfo(MachineModuleInfo* Info) override = 0; /// getLabelLocations - Return the label locations map of the label IDs to /// their address. diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h index d454347d0b..d566da8a59 100644 --- a/include/llvm/CodeGen/LatencyPriorityQueue.h +++ b/include/llvm/CodeGen/LatencyPriorityQueue.h @@ -47,21 +47,21 @@ namespace llvm { LatencyPriorityQueue() : Picker(this) { } - bool isBottomUp() const { return false; } + bool isBottomUp() const override { return false; } - void initNodes(std::vector<SUnit> &sunits) { + void initNodes(std::vector<SUnit> &sunits) override { SUnits = &sunits; NumNodesSolelyBlocking.resize(SUnits->size(), 0); } - void addNode(const SUnit *SU) { + void addNode(const SUnit *SU) override { NumNodesSolelyBlocking.resize(SUnits->size(), 0); } - void updateNode(const SUnit *SU) { + void updateNode(const SUnit *SU) override { } - void releaseState() { + void releaseState() override { SUnits = 0; } @@ -75,21 +75,21 @@ namespace llvm { return NumNodesSolelyBlocking[NodeNum]; } - bool empty() const { return Queue.empty(); } + bool empty() const override { return Queue.empty(); } - virtual void push(SUnit *U); + void push(SUnit *U) override; - virtual SUnit *pop(); + SUnit *pop() override; - virtual void remove(SUnit *SU); + void remove(SUnit *SU) override; - virtual void dump(ScheduleDAG* DAG) const; + void dump(ScheduleDAG* DAG) const override; // scheduledNode - As nodes are scheduled, we look to see if there are any // successor nodes that have a single unscheduled predecessor. If so, that // single predecessor has a higher priority, since scheduling it will make // the node available. - void scheduledNode(SUnit *Node); + void scheduledNode(SUnit *Node) override; private: void AdjustPriorityOfUnscheduledPreds(SUnit *SU); diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h index 26563a6055..e0593f80ff 100644 --- a/include/llvm/CodeGen/LexicalScopes.h +++ b/include/llvm/CodeGen/LexicalScopes.h @@ -21,9 +21,9 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Metadata.h" -#include "llvm/Support/DebugLoc.h" -#include "llvm/Support/ValueHandle.h" +#include "llvm/IR/ValueHandle.h" #include <utility> namespace llvm { @@ -44,32 +44,35 @@ typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange; /// class LexicalScopes { public: - LexicalScopes() : MF(NULL), CurrentFnLexicalScope(NULL) { } - virtual ~LexicalScopes(); + LexicalScopes() : MF(NULL), CurrentFnLexicalScope(NULL) {} + ~LexicalScopes(); - /// initialize - Scan machine function and constuct lexical scope nest. - virtual void initialize(const MachineFunction &); + /// initialize - Scan machine function and constuct lexical scope nest, resets + /// the instance if necessary. + void initialize(const MachineFunction &); /// releaseMemory - release memory. - virtual void releaseMemory(); - + void reset(); + /// empty - Return true if there is any lexical scope information available. bool empty() { return CurrentFnLexicalScope == NULL; } - /// isCurrentFunctionScope - Return true if given lexical scope represents + /// isCurrentFunctionScope - Return true if given lexical scope represents /// current function. - bool isCurrentFunctionScope(const LexicalScope *LS) { + bool isCurrentFunctionScope(const LexicalScope *LS) { return LS == CurrentFnLexicalScope; } /// getCurrentFunctionScope - Return lexical scope for the current function. - LexicalScope *getCurrentFunctionScope() const { return CurrentFnLexicalScope;} + LexicalScope *getCurrentFunctionScope() const { + return CurrentFnLexicalScope; + } /// getMachineBasicBlocks - Populate given set using machine basic blocks /// which have machine instructions that belong to lexical scope identified by /// DebugLoc. void getMachineBasicBlocks(DebugLoc DL, - SmallPtrSet<const MachineBasicBlock*, 4> &MBBs); + SmallPtrSet<const MachineBasicBlock *, 4> &MBBs); /// dominates - Return true if DebugLoc's lexical scope dominates at least one /// machine instruction's lexical scope in a given machine basic block. @@ -104,7 +107,6 @@ public: void dump(); private: - /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If /// not available then create new lexical scope. LexicalScope *getOrCreateLexicalScope(DebugLoc DL); @@ -123,8 +125,9 @@ private: void extractLexicalScopes(SmallVectorImpl<InsnRange> &MIRanges, DenseMap<const MachineInstr *, LexicalScope *> &M); void constructScopeNest(LexicalScope *Scope); - void assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges, - DenseMap<const MachineInstr *, LexicalScope *> &M); + void + assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges, + DenseMap<const MachineInstr *, LexicalScope *> &M); private: const MachineFunction *MF; @@ -133,10 +136,11 @@ private: /// contained LexicalScope*s. DenseMap<const MDNode *, LexicalScope *> LexicalScopeMap; - /// InlinedLexicalScopeMap - Tracks inlined function scopes in current function. + /// InlinedLexicalScopeMap - Tracks inlined function scopes in current + /// function. DenseMap<DebugLoc, LexicalScope *> InlinedLexicalScopeMap; - /// AbstractScopeMap - These scopes are not included LexicalScopeMap. + /// AbstractScopeMap - These scopes are not included LexicalScopeMap. /// AbstractScopes owns its LexicalScope*s. DenseMap<const MDNode *, LexicalScope *> AbstractScopeMap; @@ -153,26 +157,23 @@ private: /// LexicalScope - This class is used to track scope information. /// class LexicalScope { - virtual void anchor(); public: LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A) - : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A), - LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0) { + : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A), LastInsn(0), + FirstInsn(0), DFSIn(0), DFSOut(0) { if (Parent) Parent->addChild(this); } - virtual ~LexicalScope() {} - // Accessors. - LexicalScope *getParent() const { return Parent; } - const MDNode *getDesc() const { return Desc; } - const MDNode *getInlinedAt() const { return InlinedAtLocation; } - const MDNode *getScopeNode() const { return Desc; } - bool isAbstractScope() const { return AbstractScope; } + LexicalScope *getParent() const { return Parent; } + const MDNode *getDesc() const { return Desc; } + const MDNode *getInlinedAt() const { return InlinedAtLocation; } + const MDNode *getScopeNode() const { return Desc; } + bool isAbstractScope() const { return AbstractScope; } SmallVectorImpl<LexicalScope *> &getChildren() { return Children; } - SmallVectorImpl<InsnRange> &getRanges() { return Ranges; } + SmallVectorImpl<InsnRange> &getRanges() { return Ranges; } /// addChild - Add a child scope. void addChild(LexicalScope *S) { Children.push_back(S); } @@ -189,7 +190,7 @@ public: /// extendInsnRange - Extend the current instruction range covered by /// this scope. void extendInsnRange(const MachineInstr *MI) { - assert (FirstInsn && "MI Range is not open!"); + assert(FirstInsn && "MI Range is not open!"); LastInsn = MI; if (Parent) Parent->extendInsnRange(MI); @@ -199,7 +200,7 @@ public: /// until now. This is used when a new scope is encountered while walking /// machine instructions. void closeInsnRange(LexicalScope *NewScope = NULL) { - assert (LastInsn && "Last insn missing!"); + assert(LastInsn && "Last insn missing!"); Ranges.push_back(InsnRange(FirstInsn, LastInsn)); FirstInsn = NULL; LastInsn = NULL; @@ -219,28 +220,28 @@ public: } // Depth First Search support to walk and manipulate LexicalScope hierarchy. - unsigned getDFSOut() const { return DFSOut; } - void setDFSOut(unsigned O) { DFSOut = O; } - unsigned getDFSIn() const { return DFSIn; } - void setDFSIn(unsigned I) { DFSIn = I; } + unsigned getDFSOut() const { return DFSOut; } + void setDFSOut(unsigned O) { DFSOut = O; } + unsigned getDFSIn() const { return DFSIn; } + void setDFSIn(unsigned I) { DFSIn = I; } /// dump - print lexical scope. void dump(unsigned Indent = 0) const; private: - LexicalScope *Parent; // Parent to this scope. - AssertingVH<const MDNode> Desc; // Debug info descriptor. - AssertingVH<const MDNode> InlinedAtLocation; // Location at which this - // scope is inlined. - bool AbstractScope; // Abstract Scope - SmallVector<LexicalScope *, 4> Children; // Scopes defined in scope. - // Contents not owned. + LexicalScope *Parent; // Parent to this scope. + AssertingVH<const MDNode> Desc; // Debug info descriptor. + AssertingVH<const MDNode> InlinedAtLocation; // Location at which this + // scope is inlined. + bool AbstractScope; // Abstract Scope + SmallVector<LexicalScope *, 4> Children; // Scopes defined in scope. + // Contents not owned. SmallVector<InsnRange, 4> Ranges; - const MachineInstr *LastInsn; // Last instruction of this scope. - const MachineInstr *FirstInsn; // First instruction of this scope. - unsigned DFSIn, DFSOut; // In & Out Depth use to determine - // scope nesting. + const MachineInstr *LastInsn; // Last instruction of this scope. + const MachineInstr *FirstInsn; // First instruction of this scope. + unsigned DFSIn, DFSOut; // In & Out Depth use to determine + // scope nesting. }; } // end llvm namespace diff --git a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h index 722b0de6f7..c3046da90b 100644 --- a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h +++ b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h @@ -18,7 +18,6 @@ #include "llvm/CodeGen/GCs.h" #include <cstdlib> -#ifndef ANDROID_TARGET_BUILD namespace { struct ForceAsmWriterLinking { ForceAsmWriterLinking() { @@ -35,6 +34,5 @@ namespace { } } ForceAsmWriterLinking; // Force link by creating a global definition. } -#endif #endif // LLVM_CODEGEN_LINKALLASMWRITERCOMPONENTS_H diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index 3a9fef6fbd..41d126ac98 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -173,7 +173,7 @@ namespace llvm { } bool operator<(const Segment &Other) const { - return start < Other.start || (start == Other.start && end < Other.end); + return std::tie(start, end) < std::tie(Other.start, Other.end); } bool operator==(const Segment &Other) const { return start == Other.start && end == Other.end; @@ -552,8 +552,7 @@ namespace llvm { bool operator<(const LiveInterval& other) const { const SlotIndex &thisIndex = beginIndex(); const SlotIndex &otherIndex = other.beginIndex(); - return thisIndex < otherIndex || - (thisIndex == otherIndex && reg < other.reg); + return std::tie(thisIndex, reg) < std::tie(otherIndex, other.reg); } void print(raw_ostream &OS) const; diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h index d8437f09aa..54925938ec 100644 --- a/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h @@ -45,6 +45,7 @@ namespace llvm { class TargetInstrInfo; class TargetRegisterClass; class VirtRegMap; + class MachineBlockFrequencyInfo; class LiveIntervals : public MachineFunctionPass { MachineFunction* MF; @@ -100,7 +101,9 @@ namespace llvm { virtual ~LiveIntervals(); // Calculate the spill weight to assign to a single instruction. - static float getSpillWeight(bool isDef, bool isUse, BlockFrequency freq); + static float getSpillWeight(bool isDef, bool isUse, + const MachineBlockFrequencyInfo *MBFI, + const MachineInstr *Instr); LiveInterval &getInterval(unsigned Reg) { if (hasInterval(Reg)) @@ -252,14 +255,14 @@ namespace llvm { VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; } - virtual void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void releaseMemory(); + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; /// runOnMachineFunction - pass entry point - virtual bool runOnMachineFunction(MachineFunction&); + bool runOnMachineFunction(MachineFunction&) override; /// print - Implement the dump method. - virtual void print(raw_ostream &O, const Module* = 0) const; + void print(raw_ostream &O, const Module* = 0) const override; /// intervalIsInOneMBB - If LI is confined to a single basic block, return /// a pointer to that block. If LI is live in to or out of any block, diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h new file mode 100644 index 0000000000..c93eaf5b13 --- /dev/null +++ b/include/llvm/CodeGen/LivePhysRegs.h @@ -0,0 +1,146 @@ +//===- llvm/CodeGen/LivePhysRegs.h - Live Physical Register Set -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the LivePhysRegs utility for tracking liveness of +// physical registers. This can be used for ad-hoc liveness tracking after +// register allocation. You can start with the live-ins/live-outs at the +// beginning/end of a block and update the information while walking the +// instructions inside the block. This implementation tracks the liveness on a +// sub-register granularity. +// +// We assume that the high bits of a physical super-register are not preserved +// unless the instruction has an implicit-use operand reading the super- +// register. +// +// X86 Example: +// %YMM0<def> = ... +// %XMM0<def> = ... (Kills %XMM0, all %XMM0s sub-registers, and %YMM0) +// +// %YMM0<def> = ... +// %XMM0<def> = ..., %YMM0<imp-use> (%YMM0 and all its sub-registers are alive) +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_LIVE_PHYS_REGS_H +#define LLVM_CODEGEN_LIVE_PHYS_REGS_H + +#include "llvm/ADT/SparseSet.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include <cassert> + +namespace llvm { + +class MachineInstr; + +/// \brief A set of live physical registers with functions to track liveness +/// when walking backward/forward through a basic block. +class LivePhysRegs { + const TargetRegisterInfo *TRI; + SparseSet<unsigned> LiveRegs; + + LivePhysRegs(const LivePhysRegs&) LLVM_DELETED_FUNCTION; + LivePhysRegs &operator=(const LivePhysRegs&) LLVM_DELETED_FUNCTION; +public: + /// \brief Constructs a new empty LivePhysRegs set. + LivePhysRegs() : TRI(0), LiveRegs() {} + + /// \brief Constructs and initialize an empty LivePhysRegs set. + LivePhysRegs(const TargetRegisterInfo *TRI) : TRI(TRI) { + assert(TRI && "Invalid TargetRegisterInfo pointer."); + LiveRegs.setUniverse(TRI->getNumRegs()); + } + + /// \brief Clear and initialize the LivePhysRegs set. + void init(const TargetRegisterInfo *_TRI) { + assert(_TRI && "Invalid TargetRegisterInfo pointer."); + TRI = _TRI; + LiveRegs.clear(); + LiveRegs.setUniverse(TRI->getNumRegs()); + } + + /// \brief Clears the LivePhysRegs set. + void clear() { LiveRegs.clear(); } + + /// \brief Returns true if the set is empty. + bool empty() const { return LiveRegs.empty(); } + + /// \brief Adds a physical register and all its sub-registers to the set. + void addReg(unsigned Reg) { + assert(TRI && "LivePhysRegs is not initialized."); + assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); + for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); + SubRegs.isValid(); ++SubRegs) + LiveRegs.insert(*SubRegs); + } + + /// \brief Removes a physical register, all its sub-registers, and all its + /// super-registers from the set. + void removeReg(unsigned Reg) { + assert(TRI && "LivePhysRegs is not initialized."); + assert(Reg <= TRI->getNumRegs() && "Expected a physical register."); + for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); + SubRegs.isValid(); ++SubRegs) + LiveRegs.erase(*SubRegs); + for (MCSuperRegIterator SuperRegs(Reg, TRI, /*IncludeSelf=*/false); + SuperRegs.isValid(); ++SuperRegs) + LiveRegs.erase(*SuperRegs); + } + + /// \brief Removes physical registers clobbered by the regmask operand @p MO. + void removeRegsInMask(const MachineOperand &MO); + + /// \brief Returns true if register @p Reg is contained in the set. This also + /// works if only the super register of @p Reg has been defined, because we + /// always add also all sub-registers to the set. + bool contains(unsigned Reg) const { return LiveRegs.count(Reg); } + + /// \brief Simulates liveness when stepping backwards over an + /// instruction(bundle): Remove Defs, add uses. This is the recommended way of + /// calculating liveness. + void stepBackward(const MachineInstr &MI); + + /// \brief Simulates liveness when stepping forward over an + /// instruction(bundle): Remove killed-uses, add defs. This is the not + /// recommended way, because it depends on accurate kill flags. If possible + /// use stepBackwards() instead of this function. + void stepForward(const MachineInstr &MI); + + /// \brief Adds all live-in registers of basic block @p MBB. + void addLiveIns(const MachineBasicBlock *MBB) { + for (MachineBasicBlock::livein_iterator LI = MBB->livein_begin(), + LE = MBB->livein_end(); LI != LE; ++LI) + addReg(*LI); + } + + /// \brief Adds all live-out registers of basic block @p MBB. + void addLiveOuts(const MachineBasicBlock *MBB) { + for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(), + SE = MBB->succ_end(); SI != SE; ++SI) + addLiveIns(*SI); + } + + typedef SparseSet<unsigned>::const_iterator const_iterator; + const_iterator begin() const { return LiveRegs.begin(); } + const_iterator end() const { return LiveRegs.end(); } + + /// \brief Prints the currently live registers to @p OS. + void print(raw_ostream &OS) const; + + /// \brief Dumps the currently live registers to the debug output. + void dump() const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const LivePhysRegs& LR) { + LR.print(OS); + return OS; +} + +} // namespace llvm + +#endif // LLVM_CODEGEN_LIVE_PHYS_REGS_H diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h index 7edf67cc24..4ce39e31d1 100644 --- a/include/llvm/CodeGen/LiveRangeEdit.h +++ b/include/llvm/CodeGen/LiveRangeEdit.h @@ -99,7 +99,7 @@ private: /// MachineRegisterInfo callback to notify when new virtual /// registers are created. - void MRI_NoteNewVirtualRegister(unsigned VReg); + void MRI_NoteNewVirtualRegister(unsigned VReg) override; public: /// Create a LiveRangeEdit for breaking down parent into smaller pieces. diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h index 7a3e9e8347..28b819bb6f 100644 --- a/include/llvm/CodeGen/LiveRegMatrix.h +++ b/include/llvm/CodeGen/LiveRegMatrix.h @@ -59,9 +59,9 @@ class LiveRegMatrix : public MachineFunctionPass { BitVector RegMaskUsable; // MachineFunctionPass boilerplate. - virtual void getAnalysisUsage(AnalysisUsage&) const; - virtual bool runOnMachineFunction(MachineFunction&); - virtual void releaseMemory(); + void getAnalysisUsage(AnalysisUsage&) const override; + bool runOnMachineFunction(MachineFunction&) override; + void releaseMemory() override; public: static char ID; LiveRegMatrix(); diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h deleted file mode 100644 index 02b9c55cc6..0000000000 --- a/include/llvm/CodeGen/LiveRegUnits.h +++ /dev/null @@ -1,88 +0,0 @@ -//===-- llvm/CodeGen/LiveRegUnits.h - Live register unit set ----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements a Set of live register units. This can be used for ad -// hoc liveness tracking after register allocation. You can start with the -// live-ins/live-outs at the beginning/end of a block and update the information -// while walking the instructions inside the block. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_LIVEREGUNITS_H -#define LLVM_CODEGEN_LIVEREGUNITS_H - -#include "llvm/ADT/SparseSet.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include <cassert> - -namespace llvm { - -class MachineInstr; - -/// A set of live register units with functions to track liveness when walking -/// backward/forward through a basic block. -class LiveRegUnits { - SparseSet<unsigned> LiveUnits; - - LiveRegUnits(const LiveRegUnits&) LLVM_DELETED_FUNCTION; - LiveRegUnits &operator=(const LiveRegUnits&) LLVM_DELETED_FUNCTION; -public: - /// \brief Constructs a new empty LiveRegUnits set. - LiveRegUnits() {} - - void init(const TargetRegisterInfo *TRI) { - LiveUnits.clear(); - LiveUnits.setUniverse(TRI->getNumRegs()); - } - - void clear() { LiveUnits.clear(); } - - bool empty() const { return LiveUnits.empty(); } - - /// \brief Adds a register to the set. - void addReg(unsigned Reg, const MCRegisterInfo &MCRI) { - for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits) - LiveUnits.insert(*RUnits); - } - - /// \brief Removes a register from the set. - void removeReg(unsigned Reg, const MCRegisterInfo &MCRI) { - for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits) - LiveUnits.erase(*RUnits); - } - - /// \brief Removes registers clobbered by the regmask operand @p Op. - void removeRegsInMask(const MachineOperand &Op, const MCRegisterInfo &MCRI); - - /// \brief Returns true if register @p Reg (or one of its super register) is - /// contained in the set. - bool contains(unsigned Reg, const MCRegisterInfo &MCRI) const { - for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits) { - if (LiveUnits.count(*RUnits)) - return true; - } - return false; - } - - /// \brief Simulates liveness when stepping backwards over an - /// instruction(bundle): Remove Defs, add uses. - void stepBackward(const MachineInstr &MI, const MCRegisterInfo &MCRI); - - /// \brief Simulates liveness when stepping forward over an - /// instruction(bundle): Remove killed-uses, add defs. - void stepForward(const MachineInstr &MI, const MCRegisterInfo &MCRI); - - /// \brief Adds all registers in the live-in list of block @p BB. - void addLiveIns(const MachineBasicBlock *MBB, const MCRegisterInfo &MCRI); -}; - -} // namespace llvm - -#endif diff --git a/include/llvm/CodeGen/LiveStackAnalysis.h b/include/llvm/CodeGen/LiveStackAnalysis.h index 92c35f784d..ac32a9c590 100644 --- a/include/llvm/CodeGen/LiveStackAnalysis.h +++ b/include/llvm/CodeGen/LiveStackAnalysis.h @@ -85,14 +85,14 @@ namespace llvm { VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; } - virtual void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void releaseMemory(); + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; /// runOnMachineFunction - pass entry point - virtual bool runOnMachineFunction(MachineFunction&); + bool runOnMachineFunction(MachineFunction&) override; /// print - Implement the dump method. - virtual void print(raw_ostream &O, const Module* = 0) const; + void print(raw_ostream &O, const Module* = 0) const override; }; } diff --git a/include/llvm/CodeGen/LiveVariables.h b/include/llvm/CodeGen/LiveVariables.h index dc735f7b50..a4a5fcc31e 100644 --- a/include/llvm/CodeGen/LiveVariables.h +++ b/include/llvm/CodeGen/LiveVariables.h @@ -177,7 +177,7 @@ private: // Intermediate data structures void analyzePHINodes(const MachineFunction& Fn); public: - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; /// RegisterDefIsDead - Return true if the specified instruction defines the /// specified register, but that definition is dead. @@ -258,10 +258,10 @@ public: (void)Removed; return true; } - - void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void releaseMemory() { + void getAnalysisUsage(AnalysisUsage &AU) const override; + + void releaseMemory() override { VirtRegInfo.clear(); } diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index 7717809e0d..5e86e756d5 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -363,6 +363,9 @@ public: /// void addSuccessor(MachineBasicBlock *succ, uint32_t weight = 0); + /// Set successor weight of a given iterator. + void setSuccWeight(succ_iterator I, uint32_t weight); + /// removeSuccessor - Remove successor from the successors list of this /// MachineBasicBlock. The Predecessors list of succ is automatically updated. /// @@ -500,7 +503,7 @@ public: /// /// If I points to a bundle of instructions, they are all erased. iterator erase(iterator I) { - return erase(I, llvm::next(I)); + return erase(I, std::next(I)); } /// Remove an instruction from the instruction list and delete it. @@ -539,7 +542,7 @@ public: void splice(iterator Where, MachineBasicBlock *Other, iterator From) { // The range splice() doesn't allow noop moves, but this one does. if (Where != From) - splice(Where, Other, From, llvm::next(From)); + splice(Where, Other, From, std::next(From)); } /// Take a block of instructions from MBB 'Other' in the range [From, To), @@ -608,6 +611,9 @@ public: void dump() const; void print(raw_ostream &OS, SlotIndexes* = 0) const; + // Printing method used by LoopInfo. + void printAsOperand(raw_ostream &OS, bool PrintType = true); + /// getNumber - MachineBasicBlocks are uniquely numbered at the function /// level, unless they're not in a MachineFunction yet, in which case this /// will return -1. @@ -655,8 +661,6 @@ private: raw_ostream& operator<<(raw_ostream &OS, const MachineBasicBlock &MBB); -void WriteAsOperand(raw_ostream &, const MachineBasicBlock*, bool t); - // This is useful when building IndexedMaps keyed on basic block pointers. struct MBB2NumberFunctor : public std::unary_function<const MachineBasicBlock*, unsigned> { @@ -746,11 +750,11 @@ public: MachineInstrSpan(MachineBasicBlock::iterator I) : MBB(*I->getParent()), I(I), - B(I == MBB.begin() ? MBB.end() : llvm::prior(I)), - E(llvm::next(I)) {} + B(I == MBB.begin() ? MBB.end() : std::prev(I)), + E(std::next(I)) {} MachineBasicBlock::iterator begin() { - return B == MBB.end() ? MBB.begin() : llvm::next(B); + return B == MBB.end() ? MBB.begin() : std::next(B); } MachineBasicBlock::iterator end() { return E; } bool empty() { return begin() == end(); } diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index a9c7bf7dbc..f3ef87cbbc 100644 --- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -1,4 +1,4 @@ -//====----- MachineBlockFrequencyInfo.h - MachineBlock Frequency Analysis ----====// +//====-- MachineBlockFrequencyInfo.h - MBB Frequency Analysis -*- C++ -*--====// // // The LLVM Compiler Infrastructure // @@ -28,9 +28,9 @@ class BlockFrequencyImpl; /// MachineBlockFrequencyInfo pass uses BlockFrequencyImpl implementation to estimate /// machine basic block frequencies. class MachineBlockFrequencyInfo : public MachineFunctionPass { - - BlockFrequencyImpl<MachineBasicBlock, MachineFunction, - MachineBranchProbabilityInfo> *MBFI; + typedef BlockFrequencyImpl<MachineBasicBlock, MachineFunction, + MachineBranchProbabilityInfo> ImplType; + std::unique_ptr<ImplType> MBFI; public: static char ID; @@ -39,9 +39,11 @@ public: ~MachineBlockFrequencyInfo(); - void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnMachineFunction(MachineFunction &F) override; - bool runOnMachineFunction(MachineFunction &F); + void releaseMemory() override; /// getblockFreq - Return block frequency. Return 0 if we don't have the /// information. Please note that initial frequency is equal to 1024. It means @@ -49,6 +51,21 @@ public: /// the other block frequencies. We do this to avoid using of floating points. /// BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const; + + const MachineFunction *getFunction() const; + void view() const; + + // Print the block frequency Freq to OS using the current functions entry + // frequency to convert freq into a relative decimal form. + raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const; + + // Convenience method that attempts to look up the frequency associated with + // BB and print it to OS. + raw_ostream &printBlockFreq(raw_ostream &OS, + const MachineBasicBlock *MBB) const; + + uint64_t getEntryFreq() const; + }; } diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h index c59948fd65..7ba749559c 100644 --- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h +++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h @@ -40,7 +40,7 @@ public: initializeMachineBranchProbabilityInfoPass(Registry); } - void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); } @@ -60,7 +60,8 @@ public: uint32_t getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const; // A 'Hot' edge is an edge which probability is >= 80%. - bool isEdgeHot(MachineBasicBlock *Src, MachineBasicBlock *Dst) const; + bool isEdgeHot(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; // Return a hot successor for the block BB or null if there isn't one. // NB: This routine's complexity is linear on the number of successors. @@ -72,14 +73,15 @@ public: // NB: This routine's complexity is linear on the number of successors of // Src. Querying sequentially for each successor's probability is a quadratic // query pattern. - BranchProbability getEdgeProbability(MachineBasicBlock *Src, - MachineBasicBlock *Dst) const; + BranchProbability getEdgeProbability(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; // Print value between 0 (0% probability) and 1 (100% probability), // however the value is never equal to 0, and can be 1 only iff SRC block // has only one successor. - raw_ostream &printEdgeProbability(raw_ostream &OS, MachineBasicBlock *Src, - MachineBasicBlock *Dst) const; + raw_ostream &printEdgeProbability(raw_ostream &OS, + const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; }; } diff --git a/include/llvm/CodeGen/MachineCodeEmitter.h b/include/llvm/CodeGen/MachineCodeEmitter.h index 9e41e6e9c1..f729cedde4 100644 --- a/include/llvm/CodeGen/MachineCodeEmitter.h +++ b/include/llvm/CodeGen/MachineCodeEmitter.h @@ -17,8 +17,8 @@ #ifndef LLVM_CODEGEN_MACHINECODEEMITTER_H #define LLVM_CODEGEN_MACHINECODEEMITTER_H +#include "llvm/IR/DebugLoc.h" #include "llvm/Support/DataTypes.h" -#include "llvm/Support/DebugLoc.h" #include <string> namespace llvm { diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h index e41d206da6..f1ae0bf5f9 100644 --- a/include/llvm/CodeGen/MachineDominators.h +++ b/include/llvm/CodeGen/MachineDominators.h @@ -15,11 +15,11 @@ #ifndef LLVM_CODEGEN_MACHINEDOMINATORS_H #define LLVM_CODEGEN_MACHINEDOMINATORS_H -#include "llvm/Analysis/DominatorInternals.h" -#include "llvm/Analysis/Dominators.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Support/GenericDomTree.h" +#include "llvm/Support/GenericDomTreeConstruction.h" namespace llvm { @@ -48,7 +48,7 @@ public: DominatorTreeBase<MachineBasicBlock>& getBase() { return *DT; } - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; /// getRoots - Return the root blocks of the current CFG. This may include /// multiple blocks if we are computing post dominators. For forward @@ -66,7 +66,7 @@ public: return DT->getRootNode(); } - virtual bool runOnMachineFunction(MachineFunction &F); + bool runOnMachineFunction(MachineFunction &F) override; inline bool dominates(const MachineDomTreeNode* A, const MachineDomTreeNode* B) const { @@ -166,9 +166,9 @@ public: return DT->isReachableFromEntry(A); } - virtual void releaseMemory(); + void releaseMemory() override; - virtual void print(raw_ostream &OS, const Module*) const; + void print(raw_ostream &OS, const Module*) const override; }; //===------------------------------------- diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index 022634df87..1dedd74492 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -101,11 +101,6 @@ class MachineFrameInfo { // cannot alias any other memory objects. bool isSpillSlot; - // MayNeedSP - If true the stack object triggered the creation of the stack - // protector. We should allocate this object right after the stack - // protector. - bool MayNeedSP; - /// Alloca - If this stack object is originated from an Alloca instruction /// this value saves the original IR allocation. Can be NULL. const AllocaInst *Alloca; @@ -115,9 +110,9 @@ class MachineFrameInfo { bool PreAllocated; StackObject(uint64_t Sz, unsigned Al, int64_t SP, bool IM, - bool isSS, bool NSP, const AllocaInst *Val) + bool isSS, const AllocaInst *Val) : SPOffset(SP), Size(Sz), Alignment(Al), isImmutable(IM), - isSpillSlot(isSS), MayNeedSP(NSP), Alloca(Val), PreAllocated(false) {} + isSpillSlot(isSS), Alloca(Val), PreAllocated(false) {} }; const TargetMachine &TM; @@ -145,6 +140,14 @@ class MachineFrameInfo { /// to builtin \@llvm.returnaddress. bool ReturnAddressTaken; + /// HasStackMap - This boolean keeps track of whether there is a call + /// to builtin \@llvm.experimental.stackmap. + bool HasStackMap; + + /// HasPatchPoint - This boolean keeps track of whether there is a call + /// to builtin \@llvm.experimental.patchpoint. + bool HasPatchPoint; + /// StackSize - The prolog/epilog code inserter calculates the final stack /// offsets for all of the fixed size objects, updating the Objects list /// above. It then updates StackSize to contain the number of bytes that need @@ -223,6 +226,10 @@ class MachineFrameInfo { /// Whether the "realign-stack" option is on. bool RealignOption; + /// True if the function includes inline assembly that adjusts the stack + /// pointer. + bool HasInlineAsmWithSPAdjust; + const TargetFrameLowering *getFrameLowering() const; public: explicit MachineFrameInfo(const TargetMachine &TM, bool RealignOpt) @@ -231,6 +238,8 @@ public: HasVarSizedObjects = false; FrameAddressTaken = false; ReturnAddressTaken = false; + HasStackMap = false; + HasPatchPoint = false; AdjustsStack = false; HasCalls = false; StackProtectorIdx = -1; @@ -240,6 +249,7 @@ public: LocalFrameSize = 0; LocalFrameMaxAlign = 0; UseLocalStackAllocationBlock = false; + HasInlineAsmWithSPAdjust = false; } /// hasStackObjects - Return true if there are any stack objects in this @@ -276,6 +286,18 @@ public: bool isReturnAddressTaken() const { return ReturnAddressTaken; } void setReturnAddressIsTaken(bool s) { ReturnAddressTaken = s; } + /// hasStackMap - This method may be called any time after instruction + /// selection is complete to determine if there is a call to builtin + /// \@llvm.experimental.stackmap. + bool hasStackMap() const { return HasStackMap; } + void setHasStackMap(bool s = true) { HasStackMap = s; } + + /// hasPatchPoint - This method may be called any time after instruction + /// selection is complete to determine if there is a call to builtin + /// \@llvm.experimental.patchpoint. + bool hasPatchPoint() const { return HasPatchPoint; } + void setHasPatchPoint(bool s = true) { HasPatchPoint = s; } + /// getObjectIndexBegin - Return the minimum frame object index. /// int getObjectIndexBegin() const { return -NumFixedObjects; } @@ -380,14 +402,6 @@ public: return Objects[ObjectIdx+NumFixedObjects].Alloca; } - /// NeedsStackProtector - Returns true if the object may need stack - /// protectors. - bool MayNeedStackProtector(int ObjectIdx) const { - assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && - "Invalid Object Idx!"); - return Objects[ObjectIdx+NumFixedObjects].MayNeedSP; - } - /// getObjectOffset - Return the assigned stack offset of the specified object /// from the incoming stack pointer. /// @@ -451,6 +465,10 @@ public: bool hasCalls() const { return HasCalls; } void setHasCalls(bool V) { HasCalls = V; } + /// Returns true if the function contains any stack-adjusting inline assembly. + bool hasInlineAsmWithSPAdjust() const { return HasInlineAsmWithSPAdjust; } + void setHasInlineAsmWithSPAdjust(bool B) { HasInlineAsmWithSPAdjust = B; } + /// getMaxCallFrameSize - Return the maximum size of a call frame that must be /// allocated for an outgoing function call. This is only available if /// CallFrameSetup/Destroy pseudo instructions are used by the target, and @@ -501,7 +519,7 @@ public: /// a nonnegative identifier to represent it. /// int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, - bool MayNeedSP = false, const AllocaInst *Alloca = 0); + const AllocaInst *Alloca = 0); /// CreateSpillStackObject - Create a new statically sized stack object that /// represents a spill slot, returning a nonnegative identifier to represent @@ -521,7 +539,7 @@ public: /// variable sized object is created, whether or not the index returned is /// actually used. /// - int CreateVariableSizedObject(unsigned Alignment); + int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca); /// getCalleeSavedInfo - Returns a reference to call saved info vector for the /// current function. diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index c886e256e0..652d63dbbf 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -20,9 +20,9 @@ #include "llvm/ADT/ilist.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" -#include "llvm/Support/DebugLoc.h" #include "llvm/Support/Recycler.h" namespace llvm { @@ -131,8 +131,8 @@ class MachineFunction { /// about the control flow of such functions. bool ExposesReturnsTwice; - /// True if the function includes MS-style inline assembly. - bool HasMSInlineAsm; + /// True if the function includes any inline assembly. + bool HasInlineAsm; MachineFunction(const MachineFunction &) LLVM_DELETED_FUNCTION; void operator=(const MachineFunction&) LLVM_DELETED_FUNCTION; @@ -218,15 +218,14 @@ public: ExposesReturnsTwice = B; } - /// Returns true if the function contains any MS-style inline assembly. - bool hasMSInlineAsm() const { - return HasMSInlineAsm; + /// Returns true if the function contains any inline assembly. + bool hasInlineAsm() const { + return HasInlineAsm; } - /// Set a flag that indicates that the function contains MS-style inline - /// assembly. - void setHasMSInlineAsm(bool B) { - HasMSInlineAsm = B; + /// Set a flag that indicates that the function contains inline assembly. + void setHasInlineAsm(bool B) { + HasInlineAsm = B; } /// getInfo - Keep track of various per-function pieces of information for @@ -427,6 +426,15 @@ public: OperandRecycler.deallocate(Cap, Array); } + /// \brief Allocate and initialize a register mask with @p NumRegister bits. + uint32_t *allocateRegisterMask(unsigned NumRegister) { + unsigned Size = (NumRegister + 31) / 32; + uint32_t *Mask = Allocator.Allocate<uint32_t>(Size); + for (unsigned i = 0; i != Size; ++i) + Mask[i] = 0; + return Mask; + } + /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand /// pointers. This array is owned by the MachineFunction. MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num); diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h index 112f07ea50..36f1c66270 100644 --- a/include/llvm/CodeGen/MachineFunctionAnalysis.h +++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h @@ -34,16 +34,16 @@ public: ~MachineFunctionAnalysis(); MachineFunction &getMF() const { return *MF; } - - virtual const char* getPassName() const { + + const char* getPassName() const override { return "Machine Function Analysis"; } private: - virtual bool doInitialization(Module &M); - virtual bool runOnFunction(Function &F); - virtual void releaseMemory(); - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + bool doInitialization(Module &M) override; + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void getAnalysisUsage(AnalysisUsage &AU) const override; }; } // End llvm namespace diff --git a/include/llvm/CodeGen/MachineFunctionPass.h b/include/llvm/CodeGen/MachineFunctionPass.h index 04881e52ca..50a1f6e962 100644 --- a/include/llvm/CodeGen/MachineFunctionPass.h +++ b/include/llvm/CodeGen/MachineFunctionPass.h @@ -44,14 +44,14 @@ protected: /// For MachineFunctionPasses, calling AU.preservesCFG() indicates that /// the pass does not modify the MachineBasicBlock CFG. /// - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; private: /// createPrinterPass - Get a machine function printer pass. - virtual Pass *createPrinterPass(raw_ostream &O, - const std::string &Banner) const; + Pass *createPrinterPass(raw_ostream &O, + const std::string &Banner) const override; - virtual bool runOnFunction(Function &F); + bool runOnFunction(Function &F) override; }; } // End llvm namespace diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index cccab81efb..f5dc75eb5a 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -22,13 +22,13 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/ArrayRecycler.h" -#include "llvm/Support/DebugLoc.h" #include "llvm/Target/TargetOpcodes.h" -#include <vector> namespace llvm { @@ -287,11 +287,25 @@ public: const_mop_iterator operands_begin() const { return Operands; } const_mop_iterator operands_end() const { return Operands + NumOperands; } + inline iterator_range<mop_iterator> operands() { + return iterator_range<mop_iterator>(operands_begin(), operands_end()); + } + inline iterator_range<const_mop_iterator> operands() const { + return iterator_range<const_mop_iterator>(operands_begin(), operands_end()); + } + /// Access to memory operands of the instruction mmo_iterator memoperands_begin() const { return MemRefs; } mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; } bool memoperands_empty() const { return NumMemRefs == 0; } + inline iterator_range<mmo_iterator> memoperands() { + return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end()); + } + inline iterator_range<mmo_iterator> memoperands() const { + return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end()); + } + /// hasOneMemOperand - Return true if this instruction has exactly one /// MachineMemOperand. bool hasOneMemOperand() const { @@ -623,19 +637,19 @@ public: /// bundle remain bundled. void eraseFromBundle(); + bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } + bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } + /// isLabel - Returns true if the MachineInstr represents a label. /// - bool isLabel() const { - return getOpcode() == TargetOpcode::PROLOG_LABEL || - getOpcode() == TargetOpcode::EH_LABEL || - getOpcode() == TargetOpcode::GC_LABEL; + bool isLabel() const { return isEHLabel() || isGCLabel(); } + bool isCFIInstruction() const { + return getOpcode() == TargetOpcode::CFI_INSTRUCTION; } - bool isPrologLabel() const { - return getOpcode() == TargetOpcode::PROLOG_LABEL; - } - bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } - bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } + // True if the instruction represents a position in the function. + bool isPosition() const { return isLabel() || isCFIInstruction(); } + bool isDebugValue() const { return getOpcode() == TargetOpcode::DBG_VALUE; } /// A DBG_VALUE is indirect iff the first operand is a register and /// the second operand is an immediate. @@ -701,7 +715,7 @@ public: // Pseudo-instructions that don't produce any real output. case TargetOpcode::IMPLICIT_DEF: case TargetOpcode::KILL: - case TargetOpcode::PROLOG_LABEL: + case TargetOpcode::CFI_INSTRUCTION: case TargetOpcode::EH_LABEL: case TargetOpcode::GC_LABEL: case TargetOpcode::DBG_VALUE: @@ -830,6 +844,37 @@ public: const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; + /// \brief Applies the constraints (def/use) implied by this MI on \p Reg to + /// the given \p CurRC. + /// If \p ExploreBundle is set and MI is part of a bundle, all the + /// instructions inside the bundle will be taken into account. In other words, + /// this method accumulates all the constrains of the operand of this MI and + /// the related bundle if MI is a bundle or inside a bundle. + /// + /// Returns the register class that statisfies both \p CurRC and the + /// constraints set by MI. Returns NULL if such a register class does not + /// exist. + /// + /// \pre CurRC must not be NULL. + const TargetRegisterClass *getRegClassConstraintEffectForVReg( + unsigned Reg, const TargetRegisterClass *CurRC, + const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, + bool ExploreBundle = false) const; + + /// \brief Applies the constraints (def/use) implied by the \p OpIdx operand + /// to the given \p CurRC. + /// + /// Returns the register class that statisfies both \p CurRC and the + /// constraints set by \p OpIdx MI. Returns NULL if such a register class + /// does not exist. + /// + /// \pre CurRC must not be NULL. + /// \pre The operand at \p OpIdx must be a register. + const TargetRegisterClass * + getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, + const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI) const; + /// tieOperands - Add a tie between the register operands at DefIdx and /// UseIdx. The tie will cause the register allocator to ensure that the two /// operands are assigned the same physical register. @@ -1038,6 +1083,13 @@ private: /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a /// bundle. bool hasPropertyInBundle(unsigned Mask, QueryType Type) const; + + /// \brief Implements the logic of getRegClassConstraintEffectForVReg for the + /// this MI and the given operand index \p OpIdx. + /// If the related operand does not constrained Reg, this returns CurRC. + const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( + unsigned OpIdx, unsigned Reg, const TargetRegisterClass *CurRC, + const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; }; /// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index df01371a47..d7eb7060c9 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -172,7 +172,12 @@ public: MI->addOperand(*MF, MachineOperand::CreateMetadata(MD)); return *this; } - + + const MachineInstrBuilder &addCFIIndex(unsigned CFIIndex) const { + MI->addOperand(*MF, MachineOperand::CreateCFIIndex(CFIIndex)); + return *this; + } + const MachineInstrBuilder &addSym(MCSymbol *Sym) const { MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym)); return *this; diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h index b058ecb4c2..4fbd46b6fb 100644 --- a/include/llvm/CodeGen/MachineLoopInfo.h +++ b/include/llvm/CodeGen/MachineLoopInfo.h @@ -31,6 +31,7 @@ #define LLVM_CODEGEN_MACHINELOOPINFO_H #include "llvm/Analysis/LoopInfo.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { @@ -119,11 +120,11 @@ public: /// runOnFunction - Calculate the natural loop information. /// - virtual bool runOnMachineFunction(MachineFunction &F); + bool runOnMachineFunction(MachineFunction &F) override; - virtual void releaseMemory() { LI.releaseMemory(); } + void releaseMemory() override { LI.releaseMemory(); } - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; /// removeLoop - This removes the specified top-level loop from this loop info /// object. The loop is not deleted, as it will presumably be inserted into diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index 00a55b57f3..f01b8ebca1 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -134,6 +134,8 @@ public: /// number. int64_t getOffset() const { return PtrInfo.Offset; } + unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); } + /// getSize - Return the size in bytes of the memory reference. uint64_t getSize() const { return Size; } diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 460c08c8ca..28f4544137 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -35,14 +35,14 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MachineLocation.h" #include "llvm/Pass.h" #include "llvm/Support/DataTypes.h" -#include "llvm/Support/DebugLoc.h" #include "llvm/Support/Dwarf.h" -#include "llvm/Support/ValueHandle.h" namespace llvm { @@ -168,10 +168,13 @@ class MachineModuleInfo : public ImmutablePass { public: static char ID; // Pass identification, replacement for typeid - typedef std::pair<unsigned, DebugLoc> UnsignedDebugLocPair; - typedef SmallVector<std::pair<TrackingVH<MDNode>, UnsignedDebugLocPair>, 4> - VariableDbgInfoMapTy; - VariableDbgInfoMapTy VariableDbgInfo; + struct VariableDbgInfo { + TrackingVH<MDNode> Var; + unsigned Slot; + DebugLoc Loc; + }; + typedef SmallVector<VariableDbgInfo, 4> VariableDbgInfoMapTy; + VariableDbgInfoMapTy VariableDbgInfos; MachineModuleInfo(); // DUMMY CONSTRUCTOR, DO NOT CALL. // Real constructor. @@ -180,8 +183,8 @@ public: ~MachineModuleInfo(); // Initialization and Finalization - virtual bool doInitialization(Module &); - virtual bool doFinalization(Module &); + bool doInitialization(Module &) override; + bool doFinalization(Module &) override; /// EndFunction - Discard function meta information. /// @@ -238,8 +241,10 @@ public: return FrameInstructions; } - void addFrameInst(const MCCFIInstruction &Inst) { + unsigned LLVM_ATTRIBUTE_UNUSED_RESULT + addFrameInst(const MCCFIInstruction &Inst) { FrameInstructions.push_back(Inst); + return FrameInstructions.size() - 1; } /// getCompactUnwindEncoding - Returns the compact unwind encoding for a @@ -399,10 +404,11 @@ public: /// setVariableDbgInfo - Collect information used to emit debugging /// information of a variable. void setVariableDbgInfo(MDNode *N, unsigned Slot, DebugLoc Loc) { - VariableDbgInfo.push_back(std::make_pair(N, std::make_pair(Slot, Loc))); + VariableDbgInfo Info = { N, Slot, Loc }; + VariableDbgInfos.push_back(std::move(Info)); } - VariableDbgInfoMapTy &getVariableDbgInfo() { return VariableDbgInfo; } + VariableDbgInfoMapTy &getVariableDbgInfo() { return VariableDbgInfos; } }; // End class MachineModuleInfo diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h index 40f3580bfd..57bdb4c7fa 100644 --- a/include/llvm/CodeGen/MachineOperand.h +++ b/include/llvm/CodeGen/MachineOperand.h @@ -43,21 +43,23 @@ class MCSymbol; class MachineOperand { public: enum MachineOperandType { - MO_Register, ///< Register operand. - MO_Immediate, ///< Immediate operand - MO_CImmediate, ///< Immediate >64bit operand - MO_FPImmediate, ///< Floating-point immediate operand - MO_MachineBasicBlock, ///< MachineBasicBlock reference - MO_FrameIndex, ///< Abstract Stack Frame Index - MO_ConstantPoolIndex, ///< Address of indexed Constant in Constant Pool - MO_TargetIndex, ///< Target-dependent index+offset operand. - MO_JumpTableIndex, ///< Address of indexed Jump Table for switch - MO_ExternalSymbol, ///< Name of external global symbol - MO_GlobalAddress, ///< Address of a global value - MO_BlockAddress, ///< Address of a basic block - MO_RegisterMask, ///< Mask of preserved registers. - MO_Metadata, ///< Metadata reference (for debug info) - MO_MCSymbol ///< MCSymbol reference (for debug/eh info) + MO_Register, ///< Register operand. + MO_Immediate, ///< Immediate operand + MO_CImmediate, ///< Immediate >64bit operand + MO_FPImmediate, ///< Floating-point immediate operand + MO_MachineBasicBlock, ///< MachineBasicBlock reference + MO_FrameIndex, ///< Abstract Stack Frame Index + MO_ConstantPoolIndex, ///< Address of indexed Constant in Constant Pool + MO_TargetIndex, ///< Target-dependent index+offset operand. + MO_JumpTableIndex, ///< Address of indexed Jump Table for switch + MO_ExternalSymbol, ///< Name of external global symbol + MO_GlobalAddress, ///< Address of a global value + MO_BlockAddress, ///< Address of a basic block + MO_RegisterMask, ///< Mask of preserved registers. + MO_RegisterLiveOut, ///< Mask of live-out registers. + MO_Metadata, ///< Metadata reference (for debug info) + MO_MCSymbol, ///< MCSymbol reference (for debug/eh info) + MO_CFIIndex ///< MCCFIInstruction index. }; private: @@ -149,13 +151,14 @@ private: /// Contents union - This contains the payload for the various operand types. union { - MachineBasicBlock *MBB; // For MO_MachineBasicBlock. - const ConstantFP *CFP; // For MO_FPImmediate. - const ConstantInt *CI; // For MO_CImmediate. Integers > 64bit. - int64_t ImmVal; // For MO_Immediate. - const uint32_t *RegMask; // For MO_RegisterMask. - const MDNode *MD; // For MO_Metadata. - MCSymbol *Sym; // For MO_MCSymbol + MachineBasicBlock *MBB; // For MO_MachineBasicBlock. + const ConstantFP *CFP; // For MO_FPImmediate. + const ConstantInt *CI; // For MO_CImmediate. Integers > 64bit. + int64_t ImmVal; // For MO_Immediate. + const uint32_t *RegMask; // For MO_RegisterMask and MO_RegisterLiveOut. + const MDNode *MD; // For MO_Metadata. + MCSymbol *Sym; // For MO_MCSymbol. + unsigned CFIIndex; // For MO_CFI. struct { // For MO_Register. // Register number is in SmallContents.RegNo. @@ -246,10 +249,12 @@ public: bool isBlockAddress() const { return OpKind == MO_BlockAddress; } /// isRegMask - Tests if this is a MO_RegisterMask operand. bool isRegMask() const { return OpKind == MO_RegisterMask; } + /// isRegLiveOut - Tests if this is a MO_RegisterLiveOut operand. + bool isRegLiveOut() const { return OpKind == MO_RegisterLiveOut; } /// isMetadata - Tests if this is a MO_Metadata operand. bool isMetadata() const { return OpKind == MO_Metadata; } bool isMCSymbol() const { return OpKind == MO_MCSymbol; } - + bool isCFIIndex() const { return OpKind == MO_CFIIndex; } //===--------------------------------------------------------------------===// // Accessors for Register Operands @@ -440,6 +445,11 @@ public: return Contents.Sym; } + unsigned getCFIIndex() const { + assert(isCFIIndex() && "Wrong MachineOperand accessor"); + return Contents.CFIIndex; + } + /// getOffset - Return the offset from the symbol in this operand. This always /// returns 0 for ExternalSymbol operands. int64_t getOffset() const { @@ -476,6 +486,12 @@ public: return Contents.RegMask; } + /// getRegLiveOut - Returns a bit mask of live-out registers. + const uint32_t *getRegLiveOut() const { + assert(isRegLiveOut() && "Wrong MachineOperand accessor"); + return Contents.RegMask; + } + const MDNode *getMetadata() const { assert(isMetadata() && "Wrong MachineOperand accessor"); return Contents.MD; @@ -659,6 +675,12 @@ public: Op.Contents.RegMask = Mask; return Op; } + static MachineOperand CreateRegLiveOut(const uint32_t *Mask) { + assert(Mask && "Missing live-out register mask"); + MachineOperand Op(MachineOperand::MO_RegisterLiveOut); + Op.Contents.RegMask = Mask; + return Op; + } static MachineOperand CreateMetadata(const MDNode *Meta) { MachineOperand Op(MachineOperand::MO_Metadata); Op.Contents.MD = Meta; @@ -671,6 +693,12 @@ public: return Op; } + static MachineOperand CreateCFIIndex(unsigned CFIIndex) { + MachineOperand Op(MachineOperand::MO_CFIIndex); + Op.Contents.CFIIndex = CFIIndex; + return Op; + } + friend class MachineInstr; friend class MachineRegisterInfo; private: diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h index 90ee7f4bb9..cd212abfa2 100644 --- a/include/llvm/CodeGen/MachinePassRegistry.h +++ b/include/llvm/CodeGen/MachinePassRegistry.h @@ -142,12 +142,10 @@ public: // Implement the MachinePassRegistryListener callbacks. // - virtual void NotifyAdd(const char *N, - MachinePassCtor C, - const char *D) { + void NotifyAdd(const char *N, MachinePassCtor C, const char *D) override { this->addLiteralOption(N, (typename RegistryClass::FunctionPassCtor)C, D); } - virtual void NotifyRemove(const char *N) { + void NotifyRemove(const char *N) override { this->removeLiteralOption(N); } }; diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h index ca09aef826..a6f9f3df76 100644 --- a/include/llvm/CodeGen/MachinePostDominators.h +++ b/include/llvm/CodeGen/MachinePostDominators.h @@ -15,7 +15,6 @@ #ifndef LLVM_CODEGEN_MACHINEPOSTDOMINATORS_H #define LLVM_CODEGEN_MACHINEPOSTDOMINATORS_H -#include "llvm/Analysis/Dominators.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -78,9 +77,9 @@ public: return DT->findNearestCommonDominator(A, B); } - virtual bool runOnMachineFunction(MachineFunction &MF); - virtual void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void print(llvm::raw_ostream &OS, const Module *M = 0) const; + bool runOnMachineFunction(MachineFunction &MF) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + void print(llvm::raw_ostream &OS, const Module *M = 0) const override; }; } //end of namespace llvm diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index 58ca907316..228513076e 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -16,6 +16,7 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -200,32 +201,118 @@ public: /// reg_begin/reg_end - Provide iteration support to walk over all definitions /// and uses of a register within the MachineFunction that corresponds to this /// MachineRegisterInfo object. - template<bool Uses, bool Defs, bool SkipDebug> + template<bool Uses, bool Defs, bool SkipDebug, + bool ByOperand, bool ByInstr, bool ByBundle> class defusechain_iterator; + template<bool Uses, bool Defs, bool SkipDebug, + bool ByOperand, bool ByInstr, bool ByBundle> + class defusechain_instr_iterator; // Make it a friend so it can access getNextOperandForReg(). - template<bool, bool, bool> friend class defusechain_iterator; + template<bool, bool, bool, bool, bool, bool> + friend class defusechain_iterator; + template<bool, bool, bool, bool, bool, bool> + friend class defusechain_instr_iterator; + + /// reg_iterator/reg_begin/reg_end - Walk all defs and uses of the specified /// register. - typedef defusechain_iterator<true,true,false> reg_iterator; + typedef defusechain_iterator<true,true,false,true,false,false> + reg_iterator; reg_iterator reg_begin(unsigned RegNo) const { return reg_iterator(getRegUseDefListHead(RegNo)); } static reg_iterator reg_end() { return reg_iterator(0); } + inline iterator_range<reg_iterator> reg_operands(unsigned Reg) const { + return iterator_range<reg_iterator>(reg_begin(Reg), reg_end()); + } + + /// reg_instr_iterator/reg_instr_begin/reg_instr_end - Walk all defs and uses + /// of the specified register, stepping by MachineInstr. + typedef defusechain_instr_iterator<true,true,false,false,true,false> + reg_instr_iterator; + reg_instr_iterator reg_instr_begin(unsigned RegNo) const { + return reg_instr_iterator(getRegUseDefListHead(RegNo)); + } + static reg_instr_iterator reg_instr_end() { return reg_instr_iterator(0); } + + inline iterator_range<reg_instr_iterator> + reg_instructions(unsigned Reg) const { + return iterator_range<reg_instr_iterator>(reg_instr_begin(Reg), + reg_instr_end()); + } + + /// reg_bundle_iterator/reg_bundle_begin/reg_bundle_end - Walk all defs and uses + /// of the specified register, stepping by bundle. + typedef defusechain_instr_iterator<true,true,false,false,false,true> + reg_bundle_iterator; + reg_bundle_iterator reg_bundle_begin(unsigned RegNo) const { + return reg_bundle_iterator(getRegUseDefListHead(RegNo)); + } + static reg_bundle_iterator reg_bundle_end() { return reg_bundle_iterator(0); } + + inline iterator_range<reg_bundle_iterator> reg_bundles(unsigned Reg) const { + return iterator_range<reg_bundle_iterator>(reg_bundle_begin(Reg), + reg_bundle_end()); + } + /// reg_empty - Return true if there are no instructions using or defining the /// specified register (it may be live-in). bool reg_empty(unsigned RegNo) const { return reg_begin(RegNo) == reg_end(); } /// reg_nodbg_iterator/reg_nodbg_begin/reg_nodbg_end - Walk all defs and uses /// of the specified register, skipping those marked as Debug. - typedef defusechain_iterator<true,true,true> reg_nodbg_iterator; + typedef defusechain_iterator<true,true,true,true,false,false> + reg_nodbg_iterator; reg_nodbg_iterator reg_nodbg_begin(unsigned RegNo) const { return reg_nodbg_iterator(getRegUseDefListHead(RegNo)); } static reg_nodbg_iterator reg_nodbg_end() { return reg_nodbg_iterator(0); } + inline iterator_range<reg_nodbg_iterator> + reg_nodbg_operands(unsigned Reg) const { + return iterator_range<reg_nodbg_iterator>(reg_nodbg_begin(Reg), + reg_nodbg_end()); + } + + /// reg_instr_nodbg_iterator/reg_instr_nodbg_begin/reg_instr_nodbg_end - Walk + /// all defs and uses of the specified register, stepping by MachineInstr, + /// skipping those marked as Debug. + typedef defusechain_instr_iterator<true,true,true,false,true,false> + reg_instr_nodbg_iterator; + reg_instr_nodbg_iterator reg_instr_nodbg_begin(unsigned RegNo) const { + return reg_instr_nodbg_iterator(getRegUseDefListHead(RegNo)); + } + static reg_instr_nodbg_iterator reg_instr_nodbg_end() { + return reg_instr_nodbg_iterator(0); + } + + inline iterator_range<reg_instr_nodbg_iterator> + reg_nodbg_instructions(unsigned Reg) const { + return iterator_range<reg_instr_nodbg_iterator>(reg_instr_nodbg_begin(Reg), + reg_instr_nodbg_end()); + } + + /// reg_bundle_nodbg_iterator/reg_bundle_nodbg_begin/reg_bundle_nodbg_end - Walk + /// all defs and uses of the specified register, stepping by bundle, + /// skipping those marked as Debug. + typedef defusechain_instr_iterator<true,true,true,false,false,true> + reg_bundle_nodbg_iterator; + reg_bundle_nodbg_iterator reg_bundle_nodbg_begin(unsigned RegNo) const { + return reg_bundle_nodbg_iterator(getRegUseDefListHead(RegNo)); + } + static reg_bundle_nodbg_iterator reg_bundle_nodbg_end() { + return reg_bundle_nodbg_iterator(0); + } + + inline iterator_range<reg_bundle_nodbg_iterator> + reg_nodbg_bundles(unsigned Reg) const { + return iterator_range<reg_bundle_nodbg_iterator>(reg_bundle_nodbg_begin(Reg), + reg_bundle_nodbg_end()); + } + /// reg_nodbg_empty - Return true if the only instructions using or defining /// Reg are Debug instructions. bool reg_nodbg_empty(unsigned RegNo) const { @@ -233,12 +320,46 @@ public: } /// def_iterator/def_begin/def_end - Walk all defs of the specified register. - typedef defusechain_iterator<false,true,false> def_iterator; + typedef defusechain_iterator<false,true,false,true,false,false> + def_iterator; def_iterator def_begin(unsigned RegNo) const { return def_iterator(getRegUseDefListHead(RegNo)); } static def_iterator def_end() { return def_iterator(0); } + inline iterator_range<def_iterator> def_operands(unsigned Reg) const { + return iterator_range<def_iterator>(def_begin(Reg), def_end()); + } + + /// def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the + /// specified register, stepping by MachineInst. + typedef defusechain_instr_iterator<false,true,false,false,true,false> + def_instr_iterator; + def_instr_iterator def_instr_begin(unsigned RegNo) const { + return def_instr_iterator(getRegUseDefListHead(RegNo)); + } + static def_instr_iterator def_instr_end() { return def_instr_iterator(0); } + + inline iterator_range<def_instr_iterator> + def_instructions(unsigned Reg) const { + return iterator_range<def_instr_iterator>(def_instr_begin(Reg), + def_instr_end()); + } + + /// def_bundle_iterator/def_bundle_begin/def_bundle_end - Walk all defs of the + /// specified register, stepping by bundle. + typedef defusechain_instr_iterator<false,true,false,false,false,true> + def_bundle_iterator; + def_bundle_iterator def_bundle_begin(unsigned RegNo) const { + return def_bundle_iterator(getRegUseDefListHead(RegNo)); + } + static def_bundle_iterator def_bundle_end() { return def_bundle_iterator(0); } + + inline iterator_range<def_bundle_iterator> def_bundles(unsigned Reg) const { + return iterator_range<def_bundle_iterator>(def_bundle_begin(Reg), + def_bundle_end()); + } + /// def_empty - Return true if there are no instructions defining the /// specified register (it may be live-in). bool def_empty(unsigned RegNo) const { return def_begin(RegNo) == def_end(); } @@ -253,12 +374,46 @@ public: } /// use_iterator/use_begin/use_end - Walk all uses of the specified register. - typedef defusechain_iterator<true,false,false> use_iterator; + typedef defusechain_iterator<true,false,false,true,false,false> + use_iterator; use_iterator use_begin(unsigned RegNo) const { return use_iterator(getRegUseDefListHead(RegNo)); } static use_iterator use_end() { return use_iterator(0); } + inline iterator_range<use_iterator> use_operands(unsigned Reg) const { + return iterator_range<use_iterator>(use_begin(Reg), use_end()); + } + + /// use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the + /// specified register, stepping by MachineInstr. + typedef defusechain_instr_iterator<true,false,false,false,true,false> + use_instr_iterator; + use_instr_iterator use_instr_begin(unsigned RegNo) const { + return use_instr_iterator(getRegUseDefListHead(RegNo)); + } + static use_instr_iterator use_instr_end() { return use_instr_iterator(0); } + + inline iterator_range<use_instr_iterator> + use_instructions(unsigned Reg) const { + return iterator_range<use_instr_iterator>(use_instr_begin(Reg), + use_instr_end()); + } + + /// use_bundle_iterator/use_bundle_begin/use_bundle_end - Walk all uses of the + /// specified register, stepping by bundle. + typedef defusechain_instr_iterator<true,false,false,false,false,true> + use_bundle_iterator; + use_bundle_iterator use_bundle_begin(unsigned RegNo) const { + return use_bundle_iterator(getRegUseDefListHead(RegNo)); + } + static use_bundle_iterator use_bundle_end() { return use_bundle_iterator(0); } + + inline iterator_range<use_bundle_iterator> use_bundles(unsigned Reg) const { + return iterator_range<use_bundle_iterator>(use_bundle_begin(Reg), + use_bundle_end()); + } + /// use_empty - Return true if there are no instructions using the specified /// register. bool use_empty(unsigned RegNo) const { return use_begin(RegNo) == use_end(); } @@ -274,12 +429,55 @@ public: /// use_nodbg_iterator/use_nodbg_begin/use_nodbg_end - Walk all uses of the /// specified register, skipping those marked as Debug. - typedef defusechain_iterator<true,false,true> use_nodbg_iterator; + typedef defusechain_iterator<true,false,true,true,false,false> + use_nodbg_iterator; use_nodbg_iterator use_nodbg_begin(unsigned RegNo) const { return use_nodbg_iterator(getRegUseDefListHead(RegNo)); } static use_nodbg_iterator use_nodbg_end() { return use_nodbg_iterator(0); } + inline iterator_range<use_nodbg_iterator> + use_nodbg_operands(unsigned Reg) const { + return iterator_range<use_nodbg_iterator>(use_nodbg_begin(Reg), + use_nodbg_end()); + } + + /// use_instr_nodbg_iterator/use_instr_nodbg_begin/use_instr_nodbg_end - Walk + /// all uses of the specified register, stepping by MachineInstr, skipping + /// those marked as Debug. + typedef defusechain_instr_iterator<true,false,true,false,true,false> + use_instr_nodbg_iterator; + use_instr_nodbg_iterator use_instr_nodbg_begin(unsigned RegNo) const { + return use_instr_nodbg_iterator(getRegUseDefListHead(RegNo)); + } + static use_instr_nodbg_iterator use_instr_nodbg_end() { + return use_instr_nodbg_iterator(0); + } + + inline iterator_range<use_instr_nodbg_iterator> + use_nodbg_instructions(unsigned Reg) const { + return iterator_range<use_instr_nodbg_iterator>(use_instr_nodbg_begin(Reg), + use_instr_nodbg_end()); + } + + /// use_bundle_nodbg_iterator/use_bundle_nodbg_begin/use_bundle_nodbg_end - Walk + /// all uses of the specified register, stepping by bundle, skipping + /// those marked as Debug. + typedef defusechain_instr_iterator<true,false,true,false,false,true> + use_bundle_nodbg_iterator; + use_bundle_nodbg_iterator use_bundle_nodbg_begin(unsigned RegNo) const { + return use_bundle_nodbg_iterator(getRegUseDefListHead(RegNo)); + } + static use_bundle_nodbg_iterator use_bundle_nodbg_end() { + return use_bundle_nodbg_iterator(0); + } + + inline iterator_range<use_bundle_nodbg_iterator> + use_nodbg_bundles(unsigned Reg) const { + return iterator_range<use_bundle_nodbg_iterator>(use_bundle_nodbg_begin(Reg), + use_bundle_nodbg_end()); + } + /// use_nodbg_empty - Return true if there are no non-Debug instructions /// using the specified register. bool use_nodbg_empty(unsigned RegNo) const { @@ -401,6 +599,10 @@ public: return Hint.first ? 0 : Hint.second; } + /// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the + /// specified register as undefined which causes the DBG_VALUE to be + /// deleted during LiveDebugVariables analysis. + void markUsesInDebugValueAsUndef(unsigned Reg) const; //===--------------------------------------------------------------------===// // Physical Register Use Info @@ -552,7 +754,8 @@ public: /// returns defs. If neither are true then you are silly and it always /// returns end(). If SkipDebug is true it skips uses marked Debug /// when incrementing. - template<bool ReturnUses, bool ReturnDefs, bool SkipDebug> + template<bool ReturnUses, bool ReturnDefs, bool SkipDebug, + bool ByOperand, bool ByInstr, bool ByBundle> class defusechain_iterator : public std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t> { MachineOperand *Op; @@ -563,10 +766,30 @@ public: if ((!ReturnUses && op->isUse()) || (!ReturnDefs && op->isDef()) || (SkipDebug && op->isDebug())) - ++*this; + advance(); } } friend class MachineRegisterInfo; + + void advance() { + assert(Op && "Cannot increment end iterator!"); + Op = getNextOperandForReg(Op); + + // All defs come before the uses, so stop def_iterator early. + if (!ReturnUses) { + if (Op) { + if (Op->isUse()) + Op = 0; + else + assert(!Op->isDebug() && "Can't have debug defs"); + } + } else { + // If this is an operand we don't care about, skip it. + while (Op && ((!ReturnDefs && Op->isDef()) || + (SkipDebug && Op->isDebug()))) + Op = getNextOperandForReg(Op); + } + } public: typedef std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t>::reference reference; @@ -589,6 +812,70 @@ public: // Iterator traversal: forward iteration only defusechain_iterator &operator++() { // Preincrement assert(Op && "Cannot increment end iterator!"); + if (ByOperand) + advance(); + else if (ByInstr) { + MachineInstr *P = Op->getParent(); + do { + advance(); + } while (Op && Op->getParent() == P); + } else if (ByBundle) { + MachineInstr *P = getBundleStart(Op->getParent()); + do { + advance(); + } while (Op && getBundleStart(Op->getParent()) == P); + } + + return *this; + } + defusechain_iterator operator++(int) { // Postincrement + defusechain_iterator tmp = *this; ++*this; return tmp; + } + + /// getOperandNo - Return the operand # of this MachineOperand in its + /// MachineInstr. + unsigned getOperandNo() const { + assert(Op && "Cannot dereference end iterator!"); + return Op - &Op->getParent()->getOperand(0); + } + + // Retrieve a reference to the current operand. + MachineOperand &operator*() const { + assert(Op && "Cannot dereference end iterator!"); + return *Op; + } + + MachineOperand *operator->() const { + assert(Op && "Cannot dereference end iterator!"); + return Op; + } + }; + + /// defusechain_iterator - This class provides iterator support for machine + /// operands in the function that use or define a specific register. If + /// ReturnUses is true it returns uses of registers, if ReturnDefs is true it + /// returns defs. If neither are true then you are silly and it always + /// returns end(). If SkipDebug is true it skips uses marked Debug + /// when incrementing. + template<bool ReturnUses, bool ReturnDefs, bool SkipDebug, + bool ByOperand, bool ByInstr, bool ByBundle> + class defusechain_instr_iterator + : public std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t> { + MachineOperand *Op; + explicit defusechain_instr_iterator(MachineOperand *op) : Op(op) { + // If the first node isn't one we're interested in, advance to one that + // we are interested in. + if (op) { + if ((!ReturnUses && op->isUse()) || + (!ReturnDefs && op->isDef()) || + (SkipDebug && op->isDebug())) + advance(); + } + } + friend class MachineRegisterInfo; + + void advance() { + assert(Op && "Cannot increment end iterator!"); Op = getNextOperandForReg(Op); // All defs come before the uses, so stop def_iterator early. @@ -605,52 +892,59 @@ public: (SkipDebug && Op->isDebug()))) Op = getNextOperandForReg(Op); } + } + public: + typedef std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::reference reference; + typedef std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::pointer pointer; - return *this; + defusechain_instr_iterator(const defusechain_instr_iterator &I) : Op(I.Op){} + defusechain_instr_iterator() : Op(0) {} + + bool operator==(const defusechain_instr_iterator &x) const { + return Op == x.Op; } - defusechain_iterator operator++(int) { // Postincrement - defusechain_iterator tmp = *this; ++*this; return tmp; + bool operator!=(const defusechain_instr_iterator &x) const { + return !operator==(x); } - /// skipInstruction - move forward until reaching a different instruction. - /// Return the skipped instruction that is no longer pointed to, or NULL if - /// already pointing to end(). - MachineInstr *skipInstruction() { - if (!Op) return 0; - MachineInstr *MI = Op->getParent(); - do ++*this; - while (Op && Op->getParent() == MI); - return MI; - } + /// atEnd - return true if this iterator is equal to reg_end() on the value. + bool atEnd() const { return Op == 0; } - MachineInstr *skipBundle() { - if (!Op) return 0; - MachineInstr *MI = getBundleStart(Op->getParent()); - do ++*this; - while (Op && getBundleStart(Op->getParent()) == MI); - return MI; - } + // Iterator traversal: forward iteration only + defusechain_instr_iterator &operator++() { // Preincrement + assert(Op && "Cannot increment end iterator!"); + if (ByOperand) + advance(); + else if (ByInstr) { + MachineInstr *P = Op->getParent(); + do { + advance(); + } while (Op && Op->getParent() == P); + } else if (ByBundle) { + MachineInstr *P = getBundleStart(Op->getParent()); + do { + advance(); + } while (Op && getBundleStart(Op->getParent()) == P); + } - MachineOperand &getOperand() const { - assert(Op && "Cannot dereference end iterator!"); - return *Op; + return *this; } - - /// getOperandNo - Return the operand # of this MachineOperand in its - /// MachineInstr. - unsigned getOperandNo() const { - assert(Op && "Cannot dereference end iterator!"); - return Op - &Op->getParent()->getOperand(0); + defusechain_instr_iterator operator++(int) { // Postincrement + defusechain_instr_iterator tmp = *this; ++*this; return tmp; } // Retrieve a reference to the current operand. MachineInstr &operator*() const { assert(Op && "Cannot dereference end iterator!"); + if (ByBundle) return *(getBundleStart(Op->getParent())); return *Op->getParent(); } MachineInstr *operator->() const { assert(Op && "Cannot dereference end iterator!"); + if (ByBundle) return getBundleStart(Op->getParent()); return Op->getParent(); } }; diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h index edf93d13bd..8fc367e258 100644 --- a/include/llvm/CodeGen/MachineSSAUpdater.h +++ b/include/llvm/CodeGen/MachineSSAUpdater.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_MACHINESSAUPDATER_H #define LLVM_CODEGEN_MACHINESSAUPDATER_H +#include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" namespace llvm { @@ -26,7 +27,6 @@ namespace llvm { class TargetRegisterClass; template<typename T> class SmallVectorImpl; template<typename T> class SSAUpdaterTraits; - class BumpPtrAllocator; /// MachineSSAUpdater - This class updates SSA form for a set of virtual /// registers defined in multiple blocks. This is used when code duplication @@ -105,7 +105,6 @@ public: void RewriteUse(MachineOperand &U); private: - void ReplaceRegWith(unsigned OldReg, unsigned NewReg); unsigned GetValueAtEndOfBlockInternal(MachineBasicBlock *BB); void operator=(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION; diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 7782895334..c54300ca4f 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -23,7 +23,7 @@ // return new CustomMachineScheduler(C); // } // -// The default scheduler, ScheduleDAGMI, builds the DAG and drives list +// The default scheduler, ScheduleDAGMILive, builds the DAG and drives list // scheduling while updating the instruction stream, register pressure, and live // intervals. Most targets don't need to override the DAG builder and list // schedulier, but subtargets that require custom scheduling heuristics may @@ -93,6 +93,7 @@ class MachineLoopInfo; class RegisterClassInfo; class ScheduleDAGInstrs; class SchedDFSResult; +class ScheduleHazardRecognizer; /// MachineSchedContext provides enough context from the MachineScheduler pass /// for the target to instantiate a scheduler. @@ -154,8 +155,8 @@ struct MachineSchedPolicy { bool OnlyTopDown; bool OnlyBottomUp; - MachineSchedPolicy(): - ShouldTrackPressure(false), OnlyTopDown(false), OnlyBottomUp(false) {} + MachineSchedPolicy(): ShouldTrackPressure(false), OnlyTopDown(false), + OnlyBottomUp(false) {} }; /// MachineSchedStrategy - Interface to the scheduling algorithm used by @@ -204,63 +205,6 @@ public: virtual void releaseBottomNode(SUnit *SU) = 0; }; -/// ReadyQueue encapsulates vector of "ready" SUnits with basic convenience -/// methods for pushing and removing nodes. ReadyQueue's are uniquely identified -/// by an ID. SUnit::NodeQueueId is a mask of the ReadyQueues the SUnit is in. -/// -/// This is a convenience class that may be used by implementations of -/// MachineSchedStrategy. -class ReadyQueue { - unsigned ID; - std::string Name; - std::vector<SUnit*> Queue; - -public: - ReadyQueue(unsigned id, const Twine &name): ID(id), Name(name.str()) {} - - unsigned getID() const { return ID; } - - StringRef getName() const { return Name; } - - // SU is in this queue if it's NodeQueueID is a superset of this ID. - bool isInQueue(SUnit *SU) const { return (SU->NodeQueueId & ID); } - - bool empty() const { return Queue.empty(); } - - void clear() { Queue.clear(); } - - unsigned size() const { return Queue.size(); } - - typedef std::vector<SUnit*>::iterator iterator; - - iterator begin() { return Queue.begin(); } - - iterator end() { return Queue.end(); } - - ArrayRef<SUnit*> elements() { return Queue; } - - iterator find(SUnit *SU) { - return std::find(Queue.begin(), Queue.end(), SU); - } - - void push(SUnit *SU) { - Queue.push_back(SU); - SU->NodeQueueId |= ID; - } - - iterator remove(iterator I) { - (*I)->NodeQueueId &= ~ID; - *I = Queue.back(); - unsigned idx = I - Queue.begin(); - Queue.pop_back(); - return Queue.begin() + idx; - } - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - void dump(); -#endif -}; - /// Mutate the DAG as a postpass after normal DAG building. class ScheduleDAGMutation { virtual void anchor(); @@ -270,19 +214,15 @@ public: virtual void apply(ScheduleDAGMI *DAG) = 0; }; -/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that schedules -/// machine instructions while updating LiveIntervals and tracking regpressure. +/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply +/// schedules machine instructions according to the given MachineSchedStrategy +/// without much extra book-keeping. This is the common functionality between +/// PreRA and PostRA MachineScheduler. class ScheduleDAGMI : public ScheduleDAGInstrs { protected: AliasAnalysis *AA; - RegisterClassInfo *RegClassInfo; MachineSchedStrategy *SchedImpl; - /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees - /// will be empty. - SchedDFSResult *DFSResult; - BitVector ScheduledTrees; - /// Topo - A topological ordering for SUnits which permits fast IsReachable /// and similar queries. ScheduleDAGTopologicalSort Topo; @@ -290,32 +230,11 @@ protected: /// Ordered list of DAG postprocessing steps. std::vector<ScheduleDAGMutation*> Mutations; - MachineBasicBlock::iterator LiveRegionEnd; - - // Map each SU to its summary of pressure changes. This array is updated for - // liveness during bottom-up scheduling. Top-down scheduling may proceed but - // has no affect on the pressure diffs. - PressureDiffs SUPressureDiffs; - - /// Register pressure in this region computed by initRegPressure. - bool ShouldTrackPressure; - IntervalPressure RegPressure; - RegPressureTracker RPTracker; - - /// List of pressure sets that exceed the target's pressure limit before - /// scheduling, listed in increasing set ID order. Each pressure set is paired - /// with its max pressure in the currently scheduled regions. - std::vector<PressureChange> RegionCriticalPSets; - /// The top of the unscheduled zone. MachineBasicBlock::iterator CurrentTop; - IntervalPressure TopPressure; - RegPressureTracker TopRPTracker; /// The bottom of the unscheduled zone. MachineBasicBlock::iterator CurrentBottom; - IntervalPressure BotPressure; - RegPressureTracker BotRPTracker; /// Record the next node in a scheduled cluster. const SUnit *NextClusterPred; @@ -326,15 +245,12 @@ protected: /// scheduler at the point determined by misched-cutoff. unsigned NumInstrsScheduled; #endif - public: - ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S): - ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, /*IsPostRA=*/false, C->LIS), - AA(C->AA), RegClassInfo(C->RegClassInfo), SchedImpl(S), DFSResult(0), - Topo(SUnits, &ExitSU), ShouldTrackPressure(false), - RPTracker(RegPressure), CurrentTop(), TopRPTracker(TopPressure), - CurrentBottom(), BotRPTracker(BotPressure), - NextClusterPred(NULL), NextClusterSucc(NULL) { + ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S, bool IsPostRA): + ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, IsPostRA, + /*RemoveKillFlags=*/IsPostRA, C->LIS), + AA(C->AA), SchedImpl(S), Topo(SUnits, &ExitSU), CurrentTop(), + CurrentBottom(), NextClusterPred(NULL), NextClusterSucc(NULL) { #ifndef NDEBUG NumInstrsScheduled = 0; #endif @@ -342,8 +258,8 @@ public: virtual ~ScheduleDAGMI(); - /// \brief Return true if register pressure tracking is enabled. - bool isTrackingPressure() const { return ShouldTrackPressure; } + /// Return true if this DAG supports VReg liveness and RegPressure. + virtual bool hasVRegLiveness() const { return false; } /// Add a postprocessing step to the DAG builder. /// Mutations are applied in the order that they are added after normal DAG @@ -374,16 +290,105 @@ public: void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, - unsigned regioninstrs) LLVM_OVERRIDE; + unsigned regioninstrs) override; /// Implement ScheduleDAGInstrs interface for scheduling a sequence of /// reorderable instructions. - virtual void schedule(); + void schedule() override; /// Change the position of an instruction within the basic block and update /// live ranges and region boundary iterators. void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos); + const SUnit *getNextClusterPred() const { return NextClusterPred; } + + const SUnit *getNextClusterSucc() const { return NextClusterSucc; } + + void viewGraph(const Twine &Name, const Twine &Title) override; + void viewGraph() override; + +protected: + // Top-Level entry points for the schedule() driver... + + /// Apply each ScheduleDAGMutation step in order. This allows different + /// instances of ScheduleDAGMI to perform custom DAG postprocessing. + void postprocessDAG(); + + /// Release ExitSU predecessors and setup scheduler queues. + void initQueues(ArrayRef<SUnit*> TopRoots, ArrayRef<SUnit*> BotRoots); + + /// Update scheduler DAG and queues after scheduling an instruction. + void updateQueues(SUnit *SU, bool IsTopNode); + + /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues. + void placeDebugValues(); + + /// \brief dump the scheduled Sequence. + void dumpSchedule() const; + + // Lesser helpers... + bool checkSchedLimit(); + + void findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots, + SmallVectorImpl<SUnit*> &BotRoots); + + void releaseSucc(SUnit *SU, SDep *SuccEdge); + void releaseSuccessors(SUnit *SU); + void releasePred(SUnit *SU, SDep *PredEdge); + void releasePredecessors(SUnit *SU); +}; + +/// ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules +/// machine instructions while updating LiveIntervals and tracking regpressure. +class ScheduleDAGMILive : public ScheduleDAGMI { +protected: + RegisterClassInfo *RegClassInfo; + + /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees + /// will be empty. + SchedDFSResult *DFSResult; + BitVector ScheduledTrees; + + MachineBasicBlock::iterator LiveRegionEnd; + + // Map each SU to its summary of pressure changes. This array is updated for + // liveness during bottom-up scheduling. Top-down scheduling may proceed but + // has no affect on the pressure diffs. + PressureDiffs SUPressureDiffs; + + /// Register pressure in this region computed by initRegPressure. + bool ShouldTrackPressure; + IntervalPressure RegPressure; + RegPressureTracker RPTracker; + + /// List of pressure sets that exceed the target's pressure limit before + /// scheduling, listed in increasing set ID order. Each pressure set is paired + /// with its max pressure in the currently scheduled regions. + std::vector<PressureChange> RegionCriticalPSets; + + /// The top of the unscheduled zone. + IntervalPressure TopPressure; + RegPressureTracker TopRPTracker; + + /// The bottom of the unscheduled zone. + IntervalPressure BotPressure; + RegPressureTracker BotRPTracker; + +public: + ScheduleDAGMILive(MachineSchedContext *C, MachineSchedStrategy *S): + ScheduleDAGMI(C, S, /*IsPostRA=*/false), RegClassInfo(C->RegClassInfo), + DFSResult(0), ShouldTrackPressure(false), RPTracker(RegPressure), + TopRPTracker(TopPressure), BotRPTracker(BotPressure) + {} + + virtual ~ScheduleDAGMILive(); + + /// Return true if this DAG supports VReg liveness and RegPressure. + bool hasVRegLiveness() const override { return true; } + + /// \brief Return true if register pressure tracking is enabled. + bool isTrackingPressure() const { return ShouldTrackPressure; } + /// Get current register pressure for the top scheduled instructions. const IntervalPressure &getTopPressure() const { return TopPressure; } const RegPressureTracker &getTopRPTracker() const { return TopRPTracker; } @@ -403,10 +408,6 @@ public: return SUPressureDiffs[SU->NodeNum]; } - const SUnit *getNextClusterPred() const { return NextClusterPred; } - - const SUnit *getNextClusterSucc() const { return NextClusterSucc; } - /// Compute a DFSResult after DAG building is complete, and before any /// queue comparisons. void computeDFSResult(); @@ -416,12 +417,21 @@ public: BitVector &getScheduledTrees() { return ScheduledTrees; } + /// Implement the ScheduleDAGInstrs interface for handling the next scheduling + /// region. This covers all instructions in a block, while schedule() may only + /// cover a subset. + void enterRegion(MachineBasicBlock *bb, + MachineBasicBlock::iterator begin, + MachineBasicBlock::iterator end, + unsigned regioninstrs) override; + + /// Implement ScheduleDAGInstrs interface for scheduling a sequence of + /// reorderable instructions. + void schedule() override; + /// Compute the cyclic critical path through the DAG. unsigned computeCyclicCriticalPath(); - void viewGraph(const Twine &Name, const Twine &Title) LLVM_OVERRIDE; - void viewGraph() LLVM_OVERRIDE; - protected: // Top-Level entry points for the schedule() driver... @@ -431,25 +441,9 @@ protected: /// bottom of the DAG region without covereing any unscheduled instruction. void buildDAGWithRegPressure(); - /// Apply each ScheduleDAGMutation step in order. This allows different - /// instances of ScheduleDAGMI to perform custom DAG postprocessing. - void postprocessDAG(); - - /// Release ExitSU predecessors and setup scheduler queues. - void initQueues(ArrayRef<SUnit*> TopRoots, ArrayRef<SUnit*> BotRoots); - /// Move an instruction and update register pressure. void scheduleMI(SUnit *SU, bool IsTopNode); - /// Update scheduler DAG and queues after scheduling an instruction. - void updateQueues(SUnit *SU, bool IsTopNode); - - /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues. - void placeDebugValues(); - - /// \brief dump the scheduled Sequence. - void dumpSchedule() const; - // Lesser helpers... void initRegPressure(); @@ -458,16 +452,286 @@ protected: void updateScheduledPressure(const SUnit *SU, const std::vector<unsigned> &NewMaxPressure); +}; - bool checkSchedLimit(); +//===----------------------------------------------------------------------===// +/// +/// Helpers for implementing custom MachineSchedStrategy classes. These take +/// care of the book-keeping associated with list scheduling heuristics. +/// +//===----------------------------------------------------------------------===// - void findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots, - SmallVectorImpl<SUnit*> &BotRoots); +/// ReadyQueue encapsulates vector of "ready" SUnits with basic convenience +/// methods for pushing and removing nodes. ReadyQueue's are uniquely identified +/// by an ID. SUnit::NodeQueueId is a mask of the ReadyQueues the SUnit is in. +/// +/// This is a convenience class that may be used by implementations of +/// MachineSchedStrategy. +class ReadyQueue { + unsigned ID; + std::string Name; + std::vector<SUnit*> Queue; - void releaseSucc(SUnit *SU, SDep *SuccEdge); - void releaseSuccessors(SUnit *SU); - void releasePred(SUnit *SU, SDep *PredEdge); - void releasePredecessors(SUnit *SU); +public: + ReadyQueue(unsigned id, const Twine &name): ID(id), Name(name.str()) {} + + unsigned getID() const { return ID; } + + StringRef getName() const { return Name; } + + // SU is in this queue if it's NodeQueueID is a superset of this ID. + bool isInQueue(SUnit *SU) const { return (SU->NodeQueueId & ID); } + + bool empty() const { return Queue.empty(); } + + void clear() { Queue.clear(); } + + unsigned size() const { return Queue.size(); } + + typedef std::vector<SUnit*>::iterator iterator; + + iterator begin() { return Queue.begin(); } + + iterator end() { return Queue.end(); } + + ArrayRef<SUnit*> elements() { return Queue; } + + iterator find(SUnit *SU) { + return std::find(Queue.begin(), Queue.end(), SU); + } + + void push(SUnit *SU) { + Queue.push_back(SU); + SU->NodeQueueId |= ID; + } + + iterator remove(iterator I) { + (*I)->NodeQueueId &= ~ID; + *I = Queue.back(); + unsigned idx = I - Queue.begin(); + Queue.pop_back(); + return Queue.begin() + idx; + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + void dump(); +#endif +}; + +/// Summarize the unscheduled region. +struct SchedRemainder { + // Critical path through the DAG in expected latency. + unsigned CriticalPath; + unsigned CyclicCritPath; + + // Scaled count of micro-ops left to schedule. + unsigned RemIssueCount; + + bool IsAcyclicLatencyLimited; + + // Unscheduled resources + SmallVector<unsigned, 16> RemainingCounts; + + void reset() { + CriticalPath = 0; + CyclicCritPath = 0; + RemIssueCount = 0; + IsAcyclicLatencyLimited = false; + RemainingCounts.clear(); + } + + SchedRemainder() { reset(); } + + void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel); +}; + +/// Each Scheduling boundary is associated with ready queues. It tracks the +/// current cycle in the direction of movement, and maintains the state +/// of "hazards" and other interlocks at the current cycle. +class SchedBoundary { +public: + /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both) + enum { + TopQID = 1, + BotQID = 2, + LogMaxQID = 2 + }; + + ScheduleDAGMI *DAG; + const TargetSchedModel *SchedModel; + SchedRemainder *Rem; + + ReadyQueue Available; + ReadyQueue Pending; + + ScheduleHazardRecognizer *HazardRec; + +private: + /// True if the pending Q should be checked/updated before scheduling another + /// instruction. + bool CheckPending; + + // For heuristics, keep a list of the nodes that immediately depend on the + // most recently scheduled node. + SmallPtrSet<const SUnit*, 8> NextSUs; + + /// Number of cycles it takes to issue the instructions scheduled in this + /// zone. It is defined as: scheduled-micro-ops / issue-width + stalls. + /// See getStalls(). + unsigned CurrCycle; + + /// Micro-ops issued in the current cycle + unsigned CurrMOps; + + /// MinReadyCycle - Cycle of the soonest available instruction. + unsigned MinReadyCycle; + + // The expected latency of the critical path in this scheduled zone. + unsigned ExpectedLatency; + + // The latency of dependence chains leading into this zone. + // For each node scheduled bottom-up: DLat = max DLat, N.Depth. + // For each cycle scheduled: DLat -= 1. + unsigned DependentLatency; + + /// Count the scheduled (issued) micro-ops that can be retired by + /// time=CurrCycle assuming the first scheduled instr is retired at time=0. + unsigned RetiredMOps; + + // Count scheduled resources that have been executed. Resources are + // considered executed if they become ready in the time that it takes to + // saturate any resource including the one in question. Counts are scaled + // for direct comparison with other resources. Counts can be compared with + // MOps * getMicroOpFactor and Latency * getLatencyFactor. + SmallVector<unsigned, 16> ExecutedResCounts; + + /// Cache the max count for a single resource. + unsigned MaxExecutedResCount; + + // Cache the critical resources ID in this scheduled zone. + unsigned ZoneCritResIdx; + + // Is the scheduled region resource limited vs. latency limited. + bool IsResourceLimited; + + // Record the highest cycle at which each resource has been reserved by a + // scheduled instruction. + SmallVector<unsigned, 16> ReservedCycles; + +#ifndef NDEBUG + // Remember the greatest operand latency as an upper bound on the number of + // times we should retry the pending queue because of a hazard. + unsigned MaxObservedLatency; +#endif + +public: + /// Pending queues extend the ready queues with the same ID and the + /// PendingFlag set. + SchedBoundary(unsigned ID, const Twine &Name): + DAG(0), SchedModel(0), Rem(0), Available(ID, Name+".A"), + Pending(ID << LogMaxQID, Name+".P"), + HazardRec(0) { + reset(); + } + + ~SchedBoundary(); + + void reset(); + + void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, + SchedRemainder *rem); + + bool isTop() const { + return Available.getID() == TopQID; + } + + /// Number of cycles to issue the instructions scheduled in this zone. + unsigned getCurrCycle() const { return CurrCycle; } + + /// Micro-ops issued in the current cycle + unsigned getCurrMOps() const { return CurrMOps; } + + /// Return true if the given SU is used by the most recently scheduled + /// instruction. + bool isNextSU(const SUnit *SU) const { return NextSUs.count(SU); } + + // The latency of dependence chains leading into this zone. + unsigned getDependentLatency() const { return DependentLatency; } + + /// Get the number of latency cycles "covered" by the scheduled + /// instructions. This is the larger of the critical path within the zone + /// and the number of cycles required to issue the instructions. + unsigned getScheduledLatency() const { + return std::max(ExpectedLatency, CurrCycle); + } + + unsigned getUnscheduledLatency(SUnit *SU) const { + return isTop() ? SU->getHeight() : SU->getDepth(); + } + + unsigned getResourceCount(unsigned ResIdx) const { + return ExecutedResCounts[ResIdx]; + } + + /// Get the scaled count of scheduled micro-ops and resources, including + /// executed resources. + unsigned getCriticalCount() const { + if (!ZoneCritResIdx) + return RetiredMOps * SchedModel->getMicroOpFactor(); + return getResourceCount(ZoneCritResIdx); + } + + /// Get a scaled count for the minimum execution time of the scheduled + /// micro-ops that are ready to execute by getExecutedCount. Notice the + /// feedback loop. + unsigned getExecutedCount() const { + return std::max(CurrCycle * SchedModel->getLatencyFactor(), + MaxExecutedResCount); + } + + unsigned getZoneCritResIdx() const { return ZoneCritResIdx; } + + // Is the scheduled region resource limited vs. latency limited. + bool isResourceLimited() const { return IsResourceLimited; } + + /// Get the difference between the given SUnit's ready time and the current + /// cycle. + unsigned getLatencyStallCycles(SUnit *SU); + + unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles); + + bool checkHazard(SUnit *SU); + + unsigned findMaxLatency(ArrayRef<SUnit*> ReadySUs); + + unsigned getOtherResourceCount(unsigned &OtherCritIdx); + + void releaseNode(SUnit *SU, unsigned ReadyCycle); + + void releaseTopNode(SUnit *SU); + + void releaseBottomNode(SUnit *SU); + + void bumpCycle(unsigned NextCycle); + + void incExecutedResources(unsigned PIdx, unsigned Count); + + unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle); + + void bumpNode(SUnit *SU); + + void releasePending(); + + void removeReady(SUnit *SU); + + /// Call this before applying any other heuristics to the Available queue. + /// Updates the Available/Pending Q's if necessary and returns the single + /// available instruction, or NULL if there are multiple candidates. + SUnit *pickOnlyChoice(); + +#ifndef NDEBUG + void dumpScheduledState(); +#endif }; } // namespace llvm diff --git a/include/llvm/CodeGen/MachineTraceMetrics.h b/include/llvm/CodeGen/MachineTraceMetrics.h index 9794707e35..dc0bc1d6c2 100644 --- a/include/llvm/CodeGen/MachineTraceMetrics.h +++ b/include/llvm/CodeGen/MachineTraceMetrics.h @@ -77,10 +77,10 @@ public: class Trace; static char ID; MachineTraceMetrics(); - void getAnalysisUsage(AnalysisUsage&) const; - bool runOnMachineFunction(MachineFunction&); - void releaseMemory(); - void verifyAnalysis() const; + void getAnalysisUsage(AnalysisUsage&) const override; + bool runOnMachineFunction(MachineFunction&) override; + void releaseMemory() override; + void verifyAnalysis() const override; friend class Ensemble; friend class Trace; diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h new file mode 100644 index 0000000000..84053cac77 --- /dev/null +++ b/include/llvm/CodeGen/MachineValueType.h @@ -0,0 +1,578 @@ +//===- CodeGen/MachineValueType.h - Machine-Level types ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the set of machine-level target independent types which +// legal values in the code generator use. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINEVALUETYPE_H +#define LLVM_CODEGEN_MACHINEVALUETYPE_H + +#include "llvm/Support/ErrorHandling.h" + +namespace llvm { + + class Type; + + /// MVT - Machine Value Type. Every type that is supported natively by some + /// processor targeted by LLVM occurs here. This means that any legal value + /// type can be represented by an MVT. + class MVT { + public: + enum SimpleValueType { + // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are + // considered extended value types. + INVALID_SIMPLE_VALUE_TYPE = -1, + + // If you change this numbering, you must change the values in + // ValueTypes.td as well! + Other = 0, // This is a non-standard value + i1 = 1, // This is a 1 bit integer value + i8 = 2, // This is an 8 bit integer value + i16 = 3, // This is a 16 bit integer value + i32 = 4, // This is a 32 bit integer value + i64 = 5, // This is a 64 bit integer value + i128 = 6, // This is a 128 bit integer value + + FIRST_INTEGER_VALUETYPE = i1, + LAST_INTEGER_VALUETYPE = i128, + + f16 = 7, // This is a 16 bit floating point value + f32 = 8, // This is a 32 bit floating point value + f64 = 9, // This is a 64 bit floating point value + f80 = 10, // This is a 80 bit floating point value + f128 = 11, // This is a 128 bit floating point value + ppcf128 = 12, // This is a PPC 128-bit floating point value + + FIRST_FP_VALUETYPE = f16, + LAST_FP_VALUETYPE = ppcf128, + + v2i1 = 13, // 2 x i1 + v4i1 = 14, // 4 x i1 + v8i1 = 15, // 8 x i1 + v16i1 = 16, // 16 x i1 + v32i1 = 17, // 32 x i1 + v64i1 = 18, // 64 x i1 + + v1i8 = 19, // 1 x i8 + v2i8 = 20, // 2 x i8 + v4i8 = 21, // 4 x i8 + v8i8 = 22, // 8 x i8 + v16i8 = 23, // 16 x i8 + v32i8 = 24, // 32 x i8 + v64i8 = 25, // 64 x i8 + v1i16 = 26, // 1 x i16 + v2i16 = 27, // 2 x i16 + v4i16 = 28, // 4 x i16 + v8i16 = 29, // 8 x i16 + v16i16 = 30, // 16 x i16 + v32i16 = 31, // 32 x i16 + v1i32 = 32, // 1 x i32 + v2i32 = 33, // 2 x i32 + v4i32 = 34, // 4 x i32 + v8i32 = 35, // 8 x i32 + v16i32 = 36, // 16 x i32 + v1i64 = 37, // 1 x i64 + v2i64 = 38, // 2 x i64 + v4i64 = 39, // 4 x i64 + v8i64 = 40, // 8 x i64 + v16i64 = 41, // 16 x i64 + + FIRST_INTEGER_VECTOR_VALUETYPE = v2i1, + LAST_INTEGER_VECTOR_VALUETYPE = v16i64, + + v2f16 = 42, // 2 x f16 + v4f16 = 43, // 4 x f16 + v8f16 = 44, // 8 x f16 + v1f32 = 45, // 1 x f32 + v2f32 = 46, // 2 x f32 + v4f32 = 47, // 4 x f32 + v8f32 = 48, // 8 x f32 + v16f32 = 49, // 16 x f32 + v1f64 = 50, // 1 x f64 + v2f64 = 51, // 2 x f64 + v4f64 = 52, // 4 x f64 + v8f64 = 53, // 8 x f64 + + FIRST_FP_VECTOR_VALUETYPE = v2f16, + LAST_FP_VECTOR_VALUETYPE = v8f64, + + FIRST_VECTOR_VALUETYPE = v2i1, + LAST_VECTOR_VALUETYPE = v8f64, + + x86mmx = 54, // This is an X86 MMX value + + Glue = 55, // This glues nodes together during pre-RA sched + + isVoid = 56, // This has no value + + Untyped = 57, // This value takes a register, but has + // unspecified type. The register class + // will be determined by the opcode. + + LAST_VALUETYPE = 58, // This always remains at the end of the list. + + // This is the current maximum for LAST_VALUETYPE. + // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors + // This value must be a multiple of 32. + MAX_ALLOWED_VALUETYPE = 64, + + // Metadata - This is MDNode or MDString. + Metadata = 250, + + // iPTRAny - An int value the size of the pointer of the current + // target to any address space. This must only be used internal to + // tblgen. Other than for overloading, we treat iPTRAny the same as iPTR. + iPTRAny = 251, + + // vAny - A vector with any length and element size. This is used + // for intrinsics that have overloadings based on vector types. + // This is only for tblgen's consumption! + vAny = 252, + + // fAny - Any floating-point or vector floating-point value. This is used + // for intrinsics that have overloadings based on floating-point types. + // This is only for tblgen's consumption! + fAny = 253, + + // iAny - An integer or vector integer value of any bit width. This is + // used for intrinsics that have overloadings based on integer bit widths. + // This is only for tblgen's consumption! + iAny = 254, + + // iPTR - An int value the size of the pointer of the current + // target. This should only be used internal to tblgen! + iPTR = 255 + }; + + SimpleValueType SimpleTy; + + MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {} + MVT(SimpleValueType SVT) : SimpleTy(SVT) { } + + bool operator>(const MVT& S) const { return SimpleTy > S.SimpleTy; } + bool operator<(const MVT& S) const { return SimpleTy < S.SimpleTy; } + bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; } + bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; } + bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; } + bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; } + + /// isFloatingPoint - Return true if this is a FP, or a vector FP type. + bool isFloatingPoint() const { + return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE && + SimpleTy <= MVT::LAST_FP_VALUETYPE) || + (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE)); + } + + /// isInteger - Return true if this is an integer, or a vector integer type. + bool isInteger() const { + return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE && + SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) || + (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE)); + } + + /// isVector - Return true if this is a vector value type. + bool isVector() const { + return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE && + SimpleTy <= MVT::LAST_VECTOR_VALUETYPE); + } + + /// is16BitVector - Return true if this is a 16-bit vector type. + bool is16BitVector() const { + return (SimpleTy == MVT::v2i8 || SimpleTy == MVT::v1i16 || + SimpleTy == MVT::v16i1); + } + + /// is32BitVector - Return true if this is a 32-bit vector type. + bool is32BitVector() const { + return (SimpleTy == MVT::v4i8 || SimpleTy == MVT::v2i16 || + SimpleTy == MVT::v1i32); + } + + /// is64BitVector - Return true if this is a 64-bit vector type. + bool is64BitVector() const { + return (SimpleTy == MVT::v8i8 || SimpleTy == MVT::v4i16 || + SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 || + SimpleTy == MVT::v1f64 || SimpleTy == MVT::v2f32); + } + + /// is128BitVector - Return true if this is a 128-bit vector type. + bool is128BitVector() const { + return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || + SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || + SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64); + } + + /// is256BitVector - Return true if this is a 256-bit vector type. + bool is256BitVector() const { + return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 || + SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 || + SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64); + } + + /// is512BitVector - Return true if this is a 512-bit vector type. + bool is512BitVector() const { + return (SimpleTy == MVT::v8f64 || SimpleTy == MVT::v16f32 || + SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 || + SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32); + } + + /// is1024BitVector - Return true if this is a 1024-bit vector type. + bool is1024BitVector() const { + return (SimpleTy == MVT::v16i64); + } + + /// isOverloaded - Return true if this is an overloaded type for TableGen. + bool isOverloaded() const { + return (SimpleTy==MVT::iAny || SimpleTy==MVT::fAny || + SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny); + } + + /// isPow2VectorType - Returns true if the given vector is a power of 2. + bool isPow2VectorType() const { + unsigned NElts = getVectorNumElements(); + return !(NElts & (NElts - 1)); + } + + /// getPow2VectorType - Widens the length of the given vector MVT up to + /// the nearest power of 2 and returns that type. + MVT getPow2VectorType() const { + if (isPow2VectorType()) + return *this; + + unsigned NElts = getVectorNumElements(); + unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts); + return MVT::getVectorVT(getVectorElementType(), Pow2NElts); + } + + /// getScalarType - If this is a vector type, return the element type, + /// otherwise return this. + MVT getScalarType() const { + return isVector() ? getVectorElementType() : *this; + } + + MVT getVectorElementType() const { + switch (SimpleTy) { + default: + llvm_unreachable("Not a vector MVT!"); + case v2i1 : + case v4i1 : + case v8i1 : + case v16i1 : + case v32i1 : + case v64i1: return i1; + case v1i8 : + case v2i8 : + case v4i8 : + case v8i8 : + case v16i8: + case v32i8: + case v64i8: return i8; + case v1i16: + case v2i16: + case v4i16: + case v8i16: + case v16i16: + case v32i16: return i16; + case v1i32: + case v2i32: + case v4i32: + case v8i32: + case v16i32: return i32; + case v1i64: + case v2i64: + case v4i64: + case v8i64: + case v16i64: return i64; + case v2f16: + case v4f16: + case v8f16: return f16; + case v1f32: + case v2f32: + case v4f32: + case v8f32: + case v16f32: return f32; + case v1f64: + case v2f64: + case v4f64: + case v8f64: return f64; + } + } + + unsigned getVectorNumElements() const { + switch (SimpleTy) { + default: + llvm_unreachable("Not a vector MVT!"); + case v32i1: + case v32i8: + case v32i16: return 32; + case v64i1: + case v64i8: return 64; + case v16i1: + case v16i8: + case v16i16: + case v16i32: + case v16i64: + case v16f32: return 16; + case v8i1 : + case v8i8 : + case v8i16: + case v8i32: + case v8i64: + case v8f16: + case v8f32: + case v8f64: return 8; + case v4i1: + case v4i8: + case v4i16: + case v4i32: + case v4i64: + case v4f16: + case v4f32: + case v4f64: return 4; + case v2i1: + case v2i8: + case v2i16: + case v2i32: + case v2i64: + case v2f16: + case v2f32: + case v2f64: return 2; + case v1i8: + case v1i16: + case v1i32: + case v1i64: + case v1f32: + case v1f64: return 1; + } + } + + unsigned getSizeInBits() const { + switch (SimpleTy) { + default: + llvm_unreachable("getSizeInBits called on extended MVT."); + case Other: + llvm_unreachable("Value type is non-standard value, Other."); + case iPTR: + llvm_unreachable("Value type size is target-dependent. Ask TLI."); + case iPTRAny: + case iAny: + case fAny: + case vAny: + llvm_unreachable("Value type is overloaded."); + case Metadata: + llvm_unreachable("Value type is metadata."); + case i1 : return 1; + case v2i1: return 2; + case v4i1: return 4; + case i8 : + case v1i8: + case v8i1: return 8; + case i16 : + case f16: + case v16i1: + case v2i8: + case v1i16: return 16; + case f32 : + case i32 : + case v32i1: + case v4i8: + case v2i16: + case v2f16: + case v1f32: + case v1i32: return 32; + case x86mmx: + case f64 : + case i64 : + case v64i1: + case v8i8: + case v4i16: + case v2i32: + case v1i64: + case v4f16: + case v2f32: + case v1f64: return 64; + case f80 : return 80; + case f128: + case ppcf128: + case i128: + case v16i8: + case v8i16: + case v4i32: + case v2i64: + case v8f16: + case v4f32: + case v2f64: return 128; + case v32i8: + case v16i16: + case v8i32: + case v4i64: + case v8f32: + case v4f64: return 256; + case v64i8: + case v32i16: + case v16i32: + case v8i64: + case v16f32: + case v8f64: return 512; + case v16i64:return 1024; + } + } + + unsigned getScalarSizeInBits() const { + return getScalarType().getSizeInBits(); + } + + /// getStoreSize - Return the number of bytes overwritten by a store + /// of the specified value type. + unsigned getStoreSize() const { + return (getSizeInBits() + 7) / 8; + } + + /// getStoreSizeInBits - Return the number of bits overwritten by a store + /// of the specified value type. + unsigned getStoreSizeInBits() const { + return getStoreSize() * 8; + } + + /// Return true if this has more bits than VT. + bool bitsGT(MVT VT) const { + return getSizeInBits() > VT.getSizeInBits(); + } + + /// Return true if this has no less bits than VT. + bool bitsGE(MVT VT) const { + return getSizeInBits() >= VT.getSizeInBits(); + } + + /// Return true if this has less bits than VT. + bool bitsLT(MVT VT) const { + return getSizeInBits() < VT.getSizeInBits(); + } + + /// Return true if this has no more bits than VT. + bool bitsLE(MVT VT) const { + return getSizeInBits() <= VT.getSizeInBits(); + } + + + static MVT getFloatingPointVT(unsigned BitWidth) { + switch (BitWidth) { + default: + llvm_unreachable("Bad bit width!"); + case 16: + return MVT::f16; + case 32: + return MVT::f32; + case 64: + return MVT::f64; + case 80: + return MVT::f80; + case 128: + return MVT::f128; + } + } + + static MVT getIntegerVT(unsigned BitWidth) { + switch (BitWidth) { + default: + return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); + case 1: + return MVT::i1; + case 8: + return MVT::i8; + case 16: + return MVT::i16; + case 32: + return MVT::i32; + case 64: + return MVT::i64; + case 128: + return MVT::i128; + } + } + + static MVT getVectorVT(MVT VT, unsigned NumElements) { + switch (VT.SimpleTy) { + default: + break; + case MVT::i1: + if (NumElements == 2) return MVT::v2i1; + if (NumElements == 4) return MVT::v4i1; + if (NumElements == 8) return MVT::v8i1; + if (NumElements == 16) return MVT::v16i1; + if (NumElements == 32) return MVT::v32i1; + if (NumElements == 64) return MVT::v64i1; + break; + case MVT::i8: + if (NumElements == 1) return MVT::v1i8; + if (NumElements == 2) return MVT::v2i8; + if (NumElements == 4) return MVT::v4i8; + if (NumElements == 8) return MVT::v8i8; + if (NumElements == 16) return MVT::v16i8; + if (NumElements == 32) return MVT::v32i8; + if (NumElements == 64) return MVT::v64i8; + break; + case MVT::i16: + if (NumElements == 1) return MVT::v1i16; + if (NumElements == 2) return MVT::v2i16; + if (NumElements == 4) return MVT::v4i16; + if (NumElements == 8) return MVT::v8i16; + if (NumElements == 16) return MVT::v16i16; + if (NumElements == 32) return MVT::v32i16; + break; + case MVT::i32: + if (NumElements == 1) return MVT::v1i32; + if (NumElements == 2) return MVT::v2i32; + if (NumElements == 4) return MVT::v4i32; + if (NumElements == 8) return MVT::v8i32; + if (NumElements == 16) return MVT::v16i32; + break; + case MVT::i64: + if (NumElements == 1) return MVT::v1i64; + if (NumElements == 2) return MVT::v2i64; + if (NumElements == 4) return MVT::v4i64; + if (NumElements == 8) return MVT::v8i64; + if (NumElements == 16) return MVT::v16i64; + break; + case MVT::f16: + if (NumElements == 2) return MVT::v2f16; + if (NumElements == 4) return MVT::v4f16; + if (NumElements == 8) return MVT::v8f16; + break; + case MVT::f32: + if (NumElements == 1) return MVT::v1f32; + if (NumElements == 2) return MVT::v2f32; + if (NumElements == 4) return MVT::v4f32; + if (NumElements == 8) return MVT::v8f32; + if (NumElements == 16) return MVT::v16f32; + break; + case MVT::f64: + if (NumElements == 1) return MVT::v1f64; + if (NumElements == 2) return MVT::v2f64; + if (NumElements == 4) return MVT::v4f64; + if (NumElements == 8) return MVT::v8f64; + break; + } + return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); + } + + /// Return the value type corresponding to the specified type. This returns + /// all pointers as iPTR. If HandleUnknown is true, unknown types are + /// returned as Other, otherwise they are invalid. + static MVT getVT(Type *Ty, bool HandleUnknown = false); + + }; + +} // End llvm namespace + +#endif diff --git a/include/llvm/CodeGen/PBQP/CostAllocator.h b/include/llvm/CodeGen/PBQP/CostAllocator.h new file mode 100644 index 0000000000..164633453e --- /dev/null +++ b/include/llvm/CodeGen/PBQP/CostAllocator.h @@ -0,0 +1,147 @@ +//===---------- CostAllocator.h - PBQP Cost Allocator -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Defines classes conforming to the PBQP cost value manager concept. +// +// Cost value managers are memory managers for PBQP cost values (vectors and +// matrices). Since PBQP graphs can grow very large (E.g. hundreds of thousands +// of edges on the largest function in SPEC2006). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_COSTALLOCATOR_H +#define LLVM_COSTALLOCATOR_H + +#include <set> +#include <type_traits> + +namespace PBQP { + +template <typename CostT, + typename CostKeyTComparator> +class CostPool { +public: + + class PoolEntry { + public: + template <typename CostKeyT> + PoolEntry(CostPool &pool, CostKeyT cost) + : pool(pool), cost(std::move(cost)), refCount(0) {} + ~PoolEntry() { pool.removeEntry(this); } + void incRef() { ++refCount; } + bool decRef() { --refCount; return (refCount == 0); } + CostT& getCost() { return cost; } + const CostT& getCost() const { return cost; } + private: + CostPool &pool; + CostT cost; + std::size_t refCount; + }; + + class PoolRef { + public: + PoolRef(PoolEntry *entry) : entry(entry) { + this->entry->incRef(); + } + PoolRef(const PoolRef &r) { + entry = r.entry; + entry->incRef(); + } + PoolRef& operator=(const PoolRef &r) { + assert(entry != 0 && "entry should not be null."); + PoolEntry *temp = r.entry; + temp->incRef(); + entry->decRef(); + entry = temp; + return *this; + } + + ~PoolRef() { + if (entry->decRef()) + delete entry; + } + void reset(PoolEntry *entry) { + entry->incRef(); + this->entry->decRef(); + this->entry = entry; + } + CostT& operator*() { return entry->getCost(); } + const CostT& operator*() const { return entry->getCost(); } + CostT* operator->() { return &entry->getCost(); } + const CostT* operator->() const { return &entry->getCost(); } + private: + PoolEntry *entry; + }; + +private: + class EntryComparator { + public: + template <typename CostKeyT> + typename std::enable_if< + !std::is_same<PoolEntry*, + typename std::remove_const<CostKeyT>::type>::value, + bool>::type + operator()(const PoolEntry* a, const CostKeyT &b) { + return compare(a->getCost(), b); + } + bool operator()(const PoolEntry* a, const PoolEntry* b) { + return compare(a->getCost(), b->getCost()); + } + private: + CostKeyTComparator compare; + }; + + typedef std::set<PoolEntry*, EntryComparator> EntrySet; + + EntrySet entrySet; + + void removeEntry(PoolEntry *p) { entrySet.erase(p); } + +public: + + template <typename CostKeyT> + PoolRef getCost(CostKeyT costKey) { + typename EntrySet::iterator itr = + std::lower_bound(entrySet.begin(), entrySet.end(), costKey, + EntryComparator()); + + if (itr != entrySet.end() && costKey == (*itr)->getCost()) + return PoolRef(*itr); + + PoolEntry *p = new PoolEntry(*this, std::move(costKey)); + entrySet.insert(itr, p); + return PoolRef(p); + } +}; + +template <typename VectorT, typename VectorTComparator, + typename MatrixT, typename MatrixTComparator> +class PoolCostAllocator { +private: + typedef CostPool<VectorT, VectorTComparator> VectorCostPool; + typedef CostPool<MatrixT, MatrixTComparator> MatrixCostPool; +public: + typedef VectorT Vector; + typedef MatrixT Matrix; + typedef typename VectorCostPool::PoolRef VectorPtr; + typedef typename MatrixCostPool::PoolRef MatrixPtr; + + template <typename VectorKeyT> + VectorPtr getVector(VectorKeyT v) { return vectorPool.getCost(std::move(v)); } + + template <typename MatrixKeyT> + MatrixPtr getMatrix(MatrixKeyT m) { return matrixPool.getCost(std::move(m)); } +private: + VectorCostPool vectorPool; + MatrixCostPool matrixPool; +}; + +} + +#endif // LLVM_COSTALLOCATOR_H diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h index aca0a91303..07c3337731 100644 --- a/include/llvm/CodeGen/PBQP/Graph.h +++ b/include/llvm/CodeGen/PBQP/Graph.h @@ -15,464 +15,628 @@ #ifndef LLVM_CODEGEN_PBQP_GRAPH_H #define LLVM_CODEGEN_PBQP_GRAPH_H -#include "Math.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/Support/Compiler.h" #include <list> #include <map> #include <set> namespace PBQP { - /// PBQP Graph class. - /// Instances of this class describe PBQP problems. - class Graph { + class GraphBase { public: - typedef unsigned NodeId; typedef unsigned EdgeId; - private: + /// \brief Returns a value representing an invalid (non-existant) node. + static NodeId invalidNodeId() { + return std::numeric_limits<NodeId>::max(); + } - typedef std::set<NodeId> AdjEdgeList; + /// \brief Returns a value representing an invalid (non-existant) edge. + static EdgeId invalidEdgeId() { + return std::numeric_limits<EdgeId>::max(); + } + }; + /// PBQP Graph class. + /// Instances of this class describe PBQP problems. + /// + template <typename SolverT> + class Graph : public GraphBase { + private: + typedef typename SolverT::CostAllocator CostAllocator; public: - - typedef AdjEdgeList::iterator AdjEdgeItr; + typedef typename SolverT::RawVector RawVector; + typedef typename SolverT::RawMatrix RawMatrix; + typedef typename SolverT::Vector Vector; + typedef typename SolverT::Matrix Matrix; + typedef typename CostAllocator::VectorPtr VectorPtr; + typedef typename CostAllocator::MatrixPtr MatrixPtr; + typedef typename SolverT::NodeMetadata NodeMetadata; + typedef typename SolverT::EdgeMetadata EdgeMetadata; private: class NodeEntry { - private: - Vector costs; - AdjEdgeList adjEdges; - void *data; - NodeEntry() : costs(0, 0) {} public: - NodeEntry(const Vector &costs) : costs(costs), data(0) {} - Vector& getCosts() { return costs; } - const Vector& getCosts() const { return costs; } - unsigned getDegree() const { return adjEdges.size(); } - AdjEdgeItr edgesBegin() { return adjEdges.begin(); } - AdjEdgeItr edgesEnd() { return adjEdges.end(); } - AdjEdgeItr addEdge(EdgeId e) { - return adjEdges.insert(adjEdges.end(), e); + typedef std::vector<EdgeId> AdjEdgeList; + typedef AdjEdgeList::size_type AdjEdgeIdx; + typedef AdjEdgeList::const_iterator AdjEdgeItr; + + static AdjEdgeIdx getInvalidAdjEdgeIdx() { + return std::numeric_limits<AdjEdgeIdx>::max(); } - void removeEdge(AdjEdgeItr ae) { - adjEdges.erase(ae); + + NodeEntry(VectorPtr Costs) : Costs(Costs) {} + + AdjEdgeIdx addAdjEdgeId(EdgeId EId) { + AdjEdgeIdx Idx = AdjEdgeIds.size(); + AdjEdgeIds.push_back(EId); + return Idx; + } + + void removeAdjEdgeId(Graph &G, NodeId ThisNId, AdjEdgeIdx Idx) { + // Swap-and-pop for fast removal. + // 1) Update the adj index of the edge currently at back(). + // 2) Move last Edge down to Idx. + // 3) pop_back() + // If Idx == size() - 1 then the setAdjEdgeIdx and swap are + // redundant, but both operations are cheap. + G.getEdge(AdjEdgeIds.back()).setAdjEdgeIdx(ThisNId, Idx); + AdjEdgeIds[Idx] = AdjEdgeIds.back(); + AdjEdgeIds.pop_back(); } - void setData(void *data) { this->data = data; } - void* getData() { return data; } + + const AdjEdgeList& getAdjEdgeIds() const { return AdjEdgeIds; } + + VectorPtr Costs; + NodeMetadata Metadata; + private: + AdjEdgeList AdjEdgeIds; }; class EdgeEntry { - private: - NodeId node1, node2; - Matrix costs; - AdjEdgeItr node1AEItr, node2AEItr; - void *data; - EdgeEntry() : costs(0, 0, 0), data(0) {} public: - EdgeEntry(NodeId node1, NodeId node2, const Matrix &costs) - : node1(node1), node2(node2), costs(costs) {} - NodeId getNode1() const { return node1; } - NodeId getNode2() const { return node2; } - Matrix& getCosts() { return costs; } - const Matrix& getCosts() const { return costs; } - void setNode1AEItr(AdjEdgeItr ae) { node1AEItr = ae; } - AdjEdgeItr getNode1AEItr() { return node1AEItr; } - void setNode2AEItr(AdjEdgeItr ae) { node2AEItr = ae; } - AdjEdgeItr getNode2AEItr() { return node2AEItr; } - void setData(void *data) { this->data = data; } - void *getData() { return data; } + EdgeEntry(NodeId N1Id, NodeId N2Id, MatrixPtr Costs) + : Costs(Costs) { + NIds[0] = N1Id; + NIds[1] = N2Id; + ThisEdgeAdjIdxs[0] = NodeEntry::getInvalidAdjEdgeIdx(); + ThisEdgeAdjIdxs[1] = NodeEntry::getInvalidAdjEdgeIdx(); + } + + void invalidate() { + NIds[0] = NIds[1] = Graph::invalidNodeId(); + ThisEdgeAdjIdxs[0] = ThisEdgeAdjIdxs[1] = + NodeEntry::getInvalidAdjEdgeIdx(); + Costs = nullptr; + } + + void connectToN(Graph &G, EdgeId ThisEdgeId, unsigned NIdx) { + assert(ThisEdgeAdjIdxs[NIdx] == NodeEntry::getInvalidAdjEdgeIdx() && + "Edge already connected to NIds[NIdx]."); + NodeEntry &N = G.getNode(NIds[NIdx]); + ThisEdgeAdjIdxs[NIdx] = N.addAdjEdgeId(ThisEdgeId); + } + + void connectTo(Graph &G, EdgeId ThisEdgeId, NodeId NId) { + if (NId == NIds[0]) + connectToN(G, ThisEdgeId, 0); + else { + assert(NId == NIds[1] && "Edge does not connect NId."); + connectToN(G, ThisEdgeId, 1); + } + } + + void connect(Graph &G, EdgeId ThisEdgeId) { + connectToN(G, ThisEdgeId, 0); + connectToN(G, ThisEdgeId, 1); + } + + void setAdjEdgeIdx(NodeId NId, typename NodeEntry::AdjEdgeIdx NewIdx) { + if (NId == NIds[0]) + ThisEdgeAdjIdxs[0] = NewIdx; + else { + assert(NId == NIds[1] && "Edge not connected to NId"); + ThisEdgeAdjIdxs[1] = NewIdx; + } + } + + void disconnectFromN(Graph &G, unsigned NIdx) { + assert(ThisEdgeAdjIdxs[NIdx] != NodeEntry::getInvalidAdjEdgeIdx() && + "Edge not connected to NIds[NIdx]."); + NodeEntry &N = G.getNode(NIds[NIdx]); + N.removeAdjEdgeId(G, NIds[NIdx], ThisEdgeAdjIdxs[NIdx]); + ThisEdgeAdjIdxs[NIdx] = NodeEntry::getInvalidAdjEdgeIdx(); + } + + void disconnectFrom(Graph &G, NodeId NId) { + if (NId == NIds[0]) + disconnectFromN(G, 0); + else { + assert(NId == NIds[1] && "Edge does not connect NId"); + disconnectFromN(G, 1); + } + } + + NodeId getN1Id() const { return NIds[0]; } + NodeId getN2Id() const { return NIds[1]; } + MatrixPtr Costs; + EdgeMetadata Metadata; + private: + NodeId NIds[2]; + typename NodeEntry::AdjEdgeIdx ThisEdgeAdjIdxs[2]; }; // ----- MEMBERS ----- + CostAllocator CostAlloc; + SolverT *Solver; + typedef std::vector<NodeEntry> NodeVector; typedef std::vector<NodeId> FreeNodeVector; - NodeVector nodes; - FreeNodeVector freeNodes; + NodeVector Nodes; + FreeNodeVector FreeNodeIds; typedef std::vector<EdgeEntry> EdgeVector; typedef std::vector<EdgeId> FreeEdgeVector; - EdgeVector edges; - FreeEdgeVector freeEdges; + EdgeVector Edges; + FreeEdgeVector FreeEdgeIds; // ----- INTERNAL METHODS ----- - NodeEntry& getNode(NodeId nId) { return nodes[nId]; } - const NodeEntry& getNode(NodeId nId) const { return nodes[nId]; } + NodeEntry& getNode(NodeId NId) { return Nodes[NId]; } + const NodeEntry& getNode(NodeId NId) const { return Nodes[NId]; } - EdgeEntry& getEdge(EdgeId eId) { return edges[eId]; } - const EdgeEntry& getEdge(EdgeId eId) const { return edges[eId]; } + EdgeEntry& getEdge(EdgeId EId) { return Edges[EId]; } + const EdgeEntry& getEdge(EdgeId EId) const { return Edges[EId]; } - NodeId addConstructedNode(const NodeEntry &n) { - NodeId nodeId = 0; - if (!freeNodes.empty()) { - nodeId = freeNodes.back(); - freeNodes.pop_back(); - nodes[nodeId] = n; + NodeId addConstructedNode(const NodeEntry &N) { + NodeId NId = 0; + if (!FreeNodeIds.empty()) { + NId = FreeNodeIds.back(); + FreeNodeIds.pop_back(); + Nodes[NId] = std::move(N); } else { - nodeId = nodes.size(); - nodes.push_back(n); + NId = Nodes.size(); + Nodes.push_back(std::move(N)); } - return nodeId; + return NId; } - EdgeId addConstructedEdge(const EdgeEntry &e) { - assert(findEdge(e.getNode1(), e.getNode2()) == invalidEdgeId() && + EdgeId addConstructedEdge(const EdgeEntry &E) { + assert(findEdge(E.getN1Id(), E.getN2Id()) == invalidEdgeId() && "Attempt to add duplicate edge."); - EdgeId edgeId = 0; - if (!freeEdges.empty()) { - edgeId = freeEdges.back(); - freeEdges.pop_back(); - edges[edgeId] = e; + EdgeId EId = 0; + if (!FreeEdgeIds.empty()) { + EId = FreeEdgeIds.back(); + FreeEdgeIds.pop_back(); + Edges[EId] = std::move(E); } else { - edgeId = edges.size(); - edges.push_back(e); + EId = Edges.size(); + Edges.push_back(std::move(E)); } - EdgeEntry &ne = getEdge(edgeId); - NodeEntry &n1 = getNode(ne.getNode1()); - NodeEntry &n2 = getNode(ne.getNode2()); - - // Sanity check on matrix dimensions: - assert((n1.getCosts().getLength() == ne.getCosts().getRows()) && - (n2.getCosts().getLength() == ne.getCosts().getCols()) && - "Edge cost dimensions do not match node costs dimensions."); + EdgeEntry &NE = getEdge(EId); - ne.setNode1AEItr(n1.addEdge(edgeId)); - ne.setNode2AEItr(n2.addEdge(edgeId)); - return edgeId; + // Add the edge to the adjacency sets of its nodes. + NE.connect(*this, EId); + return EId; } - Graph(const Graph &other) {} - void operator=(const Graph &other) {} + Graph(const Graph &Other) {} + void operator=(const Graph &Other) {} public: + typedef typename NodeEntry::AdjEdgeItr AdjEdgeItr; + class NodeItr { public: - NodeItr(NodeId nodeId, const Graph &g) - : nodeId(nodeId), endNodeId(g.nodes.size()), freeNodes(g.freeNodes) { - this->nodeId = findNextInUse(nodeId); // Move to the first in-use nodeId + NodeItr(NodeId CurNId, const Graph &G) + : CurNId(CurNId), EndNId(G.Nodes.size()), FreeNodeIds(G.FreeNodeIds) { + this->CurNId = findNextInUse(CurNId); // Move to first in-use node id } - bool operator==(const NodeItr& n) const { return nodeId == n.nodeId; } - bool operator!=(const NodeItr& n) const { return !(*this == n); } - NodeItr& operator++() { nodeId = findNextInUse(++nodeId); return *this; } - NodeId operator*() const { return nodeId; } + bool operator==(const NodeItr &O) const { return CurNId == O.CurNId; } + bool operator!=(const NodeItr &O) const { return !(*this == O); } + NodeItr& operator++() { CurNId = findNextInUse(++CurNId); return *this; } + NodeId operator*() const { return CurNId; } private: - NodeId findNextInUse(NodeId n) const { - while (n < endNodeId && - std::find(freeNodes.begin(), freeNodes.end(), n) != - freeNodes.end()) { - ++n; + NodeId findNextInUse(NodeId NId) const { + while (NId < EndNId && + std::find(FreeNodeIds.begin(), FreeNodeIds.end(), NId) != + FreeNodeIds.end()) { + ++NId; } - return n; + return NId; } - NodeId nodeId, endNodeId; - const FreeNodeVector& freeNodes; + NodeId CurNId, EndNId; + const FreeNodeVector &FreeNodeIds; }; class EdgeItr { public: - EdgeItr(EdgeId edgeId, const Graph &g) - : edgeId(edgeId), endEdgeId(g.edges.size()), freeEdges(g.freeEdges) { - this->edgeId = findNextInUse(edgeId); // Move to the first in-use edgeId + EdgeItr(EdgeId CurEId, const Graph &G) + : CurEId(CurEId), EndEId(G.Edges.size()), FreeEdgeIds(G.FreeEdgeIds) { + this->CurEId = findNextInUse(CurEId); // Move to first in-use edge id } - bool operator==(const EdgeItr& n) const { return edgeId == n.edgeId; } - bool operator!=(const EdgeItr& n) const { return !(*this == n); } - EdgeItr& operator++() { edgeId = findNextInUse(++edgeId); return *this; } - EdgeId operator*() const { return edgeId; } + bool operator==(const EdgeItr &O) const { return CurEId == O.CurEId; } + bool operator!=(const EdgeItr &O) const { return !(*this == O); } + EdgeItr& operator++() { CurEId = findNextInUse(++CurEId); return *this; } + EdgeId operator*() const { return CurEId; } private: - EdgeId findNextInUse(EdgeId n) const { - while (n < endEdgeId && - std::find(freeEdges.begin(), freeEdges.end(), n) != - freeEdges.end()) { - ++n; + EdgeId findNextInUse(EdgeId EId) const { + while (EId < EndEId && + std::find(FreeEdgeIds.begin(), FreeEdgeIds.end(), EId) != + FreeEdgeIds.end()) { + ++EId; } - return n; + return EId; + } + + EdgeId CurEId, EndEId; + const FreeEdgeVector &FreeEdgeIds; + }; + + class NodeIdSet { + public: + NodeIdSet(const Graph &G) : G(G) { } + NodeItr begin() const { return NodeItr(0, G); } + NodeItr end() const { return NodeItr(G.Nodes.size(), G); } + bool empty() const { return G.Nodes.empty(); } + typename NodeVector::size_type size() const { + return G.Nodes.size() - G.FreeNodeIds.size(); } + private: + const Graph& G; + }; - EdgeId edgeId, endEdgeId; - const FreeEdgeVector& freeEdges; + class EdgeIdSet { + public: + EdgeIdSet(const Graph &G) : G(G) { } + EdgeItr begin() const { return EdgeItr(0, G); } + EdgeItr end() const { return EdgeItr(G.Edges.size(), G); } + bool empty() const { return G.Edges.empty(); } + typename NodeVector::size_type size() const { + return G.Edges.size() - G.FreeEdgeIds.size(); + } + private: + const Graph& G; + }; + + class AdjEdgeIdSet { + public: + AdjEdgeIdSet(const NodeEntry &NE) : NE(NE) { } + typename NodeEntry::AdjEdgeItr begin() const { + return NE.getAdjEdgeIds().begin(); + } + typename NodeEntry::AdjEdgeItr end() const { + return NE.getAdjEdgeIds().end(); + } + bool empty() const { return NE.getAdjEdgeIds().empty(); } + typename NodeEntry::AdjEdgeList::size_type size() const { + return NE.getAdjEdgeIds().size(); + } + private: + const NodeEntry &NE; }; /// \brief Construct an empty PBQP graph. - Graph() {} + Graph() : Solver(nullptr) { } + + /// \brief Lock this graph to the given solver instance in preparation + /// for running the solver. This method will call solver.handleAddNode for + /// each node in the graph, and handleAddEdge for each edge, to give the + /// solver an opportunity to set up any requried metadata. + void setSolver(SolverT &S) { + assert(Solver == nullptr && "Solver already set. Call unsetSolver()."); + Solver = &S; + for (auto NId : nodeIds()) + Solver->handleAddNode(NId); + for (auto EId : edgeIds()) + Solver->handleAddEdge(EId); + } + + /// \brief Release from solver instance. + void unsetSolver() { + assert(Solver != nullptr && "Solver not set."); + Solver = nullptr; + } /// \brief Add a node with the given costs. - /// @param costs Cost vector for the new node. + /// @param Costs Cost vector for the new node. /// @return Node iterator for the added node. - NodeId addNode(const Vector &costs) { - return addConstructedNode(NodeEntry(costs)); + template <typename OtherVectorT> + NodeId addNode(OtherVectorT Costs) { + // Get cost vector from the problem domain + VectorPtr AllocatedCosts = CostAlloc.getVector(std::move(Costs)); + NodeId NId = addConstructedNode(NodeEntry(AllocatedCosts)); + if (Solver) + Solver->handleAddNode(NId); + return NId; } /// \brief Add an edge between the given nodes with the given costs. - /// @param n1Id First node. - /// @param n2Id Second node. + /// @param N1Id First node. + /// @param N2Id Second node. /// @return Edge iterator for the added edge. - EdgeId addEdge(NodeId n1Id, NodeId n2Id, const Matrix &costs) { - assert(getNodeCosts(n1Id).getLength() == costs.getRows() && - getNodeCosts(n2Id).getLength() == costs.getCols() && + template <typename OtherVectorT> + EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs) { + assert(getNodeCosts(N1Id).getLength() == Costs.getRows() && + getNodeCosts(N2Id).getLength() == Costs.getCols() && "Matrix dimensions mismatch."); - return addConstructedEdge(EdgeEntry(n1Id, n2Id, costs)); + // Get cost matrix from the problem domain. + MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs)); + EdgeId EId = addConstructedEdge(EdgeEntry(N1Id, N2Id, AllocatedCosts)); + if (Solver) + Solver->handleAddEdge(EId); + return EId; } + /// \brief Returns true if the graph is empty. + bool empty() const { return NodeIdSet(*this).empty(); } + + NodeIdSet nodeIds() const { return NodeIdSet(*this); } + EdgeIdSet edgeIds() const { return EdgeIdSet(*this); } + + AdjEdgeIdSet adjEdgeIds(NodeId NId) { return AdjEdgeIdSet(getNode(NId)); } + /// \brief Get the number of nodes in the graph. /// @return Number of nodes in the graph. - unsigned getNumNodes() const { return nodes.size() - freeNodes.size(); } + unsigned getNumNodes() const { return NodeIdSet(*this).size(); } /// \brief Get the number of edges in the graph. /// @return Number of edges in the graph. - unsigned getNumEdges() const { return edges.size() - freeEdges.size(); } - - /// \brief Get a node's cost vector. - /// @param nId Node id. - /// @return Node cost vector. - Vector& getNodeCosts(NodeId nId) { return getNode(nId).getCosts(); } + unsigned getNumEdges() const { return EdgeIdSet(*this).size(); } + + /// \brief Set a node's cost vector. + /// @param NId Node to update. + /// @param Costs New costs to set. + template <typename OtherVectorT> + void setNodeCosts(NodeId NId, OtherVectorT Costs) { + VectorPtr AllocatedCosts = CostAlloc.getVector(std::move(Costs)); + if (Solver) + Solver->handleSetNodeCosts(NId, *AllocatedCosts); + getNode(NId).Costs = AllocatedCosts; + } /// \brief Get a node's cost vector (const version). - /// @param nId Node id. + /// @param NId Node id. /// @return Node cost vector. - const Vector& getNodeCosts(NodeId nId) const { - return getNode(nId).getCosts(); + const Vector& getNodeCosts(NodeId NId) const { + return *getNode(NId).Costs; } - /// \brief Set a node's data pointer. - /// @param nId Node id. - /// @param data Pointer to node data. - /// - /// Typically used by a PBQP solver to attach data to aid in solution. - void setNodeData(NodeId nId, void *data) { getNode(nId).setData(data); } - - /// \brief Get the node's data pointer. - /// @param nId Node id. - /// @return Pointer to node data. - void* getNodeData(NodeId nId) { return getNode(nId).getData(); } - - /// \brief Get an edge's cost matrix. - /// @param eId Edge id. - /// @return Edge cost matrix. - Matrix& getEdgeCosts(EdgeId eId) { return getEdge(eId).getCosts(); } - - /// \brief Get an edge's cost matrix (const version). - /// @param eId Edge id. - /// @return Edge cost matrix. - const Matrix& getEdgeCosts(EdgeId eId) const { - return getEdge(eId).getCosts(); + NodeMetadata& getNodeMetadata(NodeId NId) { + return getNode(NId).Metadata; } - /// \brief Set an edge's data pointer. - /// @param eId Edge id. - /// @param data Pointer to edge data. - /// - /// Typically used by a PBQP solver to attach data to aid in solution. - void setEdgeData(EdgeId eId, void *data) { getEdge(eId).setData(data); } - - /// \brief Get an edge's data pointer. - /// @param eId Edge id. - /// @return Pointer to edge data. - void* getEdgeData(EdgeId eId) { return getEdge(eId).getData(); } - - /// \brief Get a node's degree. - /// @param nId Node id. - /// @return The degree of the node. - unsigned getNodeDegree(NodeId nId) const { - return getNode(nId).getDegree(); + const NodeMetadata& getNodeMetadata(NodeId NId) const { + return getNode(NId).Metadata; } - /// \brief Begin iterator for node set. - NodeItr nodesBegin() const { return NodeItr(0, *this); } - - /// \brief End iterator for node set. - NodeItr nodesEnd() const { return NodeItr(nodes.size(), *this); } + typename NodeEntry::AdjEdgeList::size_type getNodeDegree(NodeId NId) const { + return getNode(NId).getAdjEdgeIds().size(); + } - /// \brief Begin iterator for edge set. - EdgeItr edgesBegin() const { return EdgeItr(0, *this); } + /// \brief Set an edge's cost matrix. + /// @param EId Edge id. + /// @param Costs New cost matrix. + template <typename OtherMatrixT> + void setEdgeCosts(EdgeId EId, OtherMatrixT Costs) { + MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs)); + if (Solver) + Solver->handleSetEdgeCosts(EId, *AllocatedCosts); + getEdge(EId).Costs = AllocatedCosts; + } - /// \brief End iterator for edge set. - EdgeItr edgesEnd() const { return EdgeItr(edges.size(), *this); } + /// \brief Get an edge's cost matrix (const version). + /// @param EId Edge id. + /// @return Edge cost matrix. + const Matrix& getEdgeCosts(EdgeId EId) const { return *getEdge(EId).Costs; } - /// \brief Get begin iterator for adjacent edge set. - /// @param nId Node id. - /// @return Begin iterator for the set of edges connected to the given node. - AdjEdgeItr adjEdgesBegin(NodeId nId) { - return getNode(nId).edgesBegin(); + EdgeMetadata& getEdgeMetadata(EdgeId NId) { + return getEdge(NId).Metadata; } - /// \brief Get end iterator for adjacent edge set. - /// @param nId Node id. - /// @return End iterator for the set of edges connected to the given node. - AdjEdgeItr adjEdgesEnd(NodeId nId) { - return getNode(nId).edgesEnd(); + const EdgeMetadata& getEdgeMetadata(EdgeId NId) const { + return getEdge(NId).Metadata; } /// \brief Get the first node connected to this edge. - /// @param eId Edge id. + /// @param EId Edge id. /// @return The first node connected to the given edge. - NodeId getEdgeNode1(EdgeId eId) { - return getEdge(eId).getNode1(); + NodeId getEdgeNode1Id(EdgeId EId) { + return getEdge(EId).getN1Id(); } /// \brief Get the second node connected to this edge. - /// @param eId Edge id. + /// @param EId Edge id. /// @return The second node connected to the given edge. - NodeId getEdgeNode2(EdgeId eId) { - return getEdge(eId).getNode2(); + NodeId getEdgeNode2Id(EdgeId EId) { + return getEdge(EId).getN2Id(); } /// \brief Get the "other" node connected to this edge. - /// @param eId Edge id. - /// @param nId Node id for the "given" node. + /// @param EId Edge id. + /// @param NId Node id for the "given" node. /// @return The iterator for the "other" node connected to this edge. - NodeId getEdgeOtherNode(EdgeId eId, NodeId nId) { - EdgeEntry &e = getEdge(eId); - if (e.getNode1() == nId) { - return e.getNode2(); + NodeId getEdgeOtherNodeId(EdgeId EId, NodeId NId) { + EdgeEntry &E = getEdge(EId); + if (E.getN1Id() == NId) { + return E.getN2Id(); } // else - return e.getNode1(); - } - - EdgeId invalidEdgeId() const { - return std::numeric_limits<EdgeId>::max(); + return E.getN1Id(); } /// \brief Get the edge connecting two nodes. - /// @param n1Id First node id. - /// @param n2Id Second node id. - /// @return An id for edge (n1Id, n2Id) if such an edge exists, + /// @param N1Id First node id. + /// @param N2Id Second node id. + /// @return An id for edge (N1Id, N2Id) if such an edge exists, /// otherwise returns an invalid edge id. - EdgeId findEdge(NodeId n1Id, NodeId n2Id) { - for (AdjEdgeItr aeItr = adjEdgesBegin(n1Id), aeEnd = adjEdgesEnd(n1Id); - aeItr != aeEnd; ++aeItr) { - if ((getEdgeNode1(*aeItr) == n2Id) || - (getEdgeNode2(*aeItr) == n2Id)) { - return *aeItr; + EdgeId findEdge(NodeId N1Id, NodeId N2Id) { + for (auto AEId : adjEdgeIds(N1Id)) { + if ((getEdgeNode1Id(AEId) == N2Id) || + (getEdgeNode2Id(AEId) == N2Id)) { + return AEId; } } return invalidEdgeId(); } /// \brief Remove a node from the graph. - /// @param nId Node id. - void removeNode(NodeId nId) { - NodeEntry &n = getNode(nId); - for (AdjEdgeItr itr = n.edgesBegin(), end = n.edgesEnd(); itr != end; ++itr) { - EdgeId eId = *itr; - removeEdge(eId); + /// @param NId Node id. + void removeNode(NodeId NId) { + if (Solver) + Solver->handleRemoveNode(NId); + NodeEntry &N = getNode(NId); + // TODO: Can this be for-each'd? + for (AdjEdgeItr AEItr = N.adjEdgesBegin(), + AEEnd = N.adjEdgesEnd(); + AEItr != AEEnd;) { + EdgeId EId = *AEItr; + ++AEItr; + removeEdge(EId); } - freeNodes.push_back(nId); + FreeNodeIds.push_back(NId); + } + + /// \brief Disconnect an edge from the given node. + /// + /// Removes the given edge from the adjacency list of the given node. + /// This operation leaves the edge in an 'asymmetric' state: It will no + /// longer appear in an iteration over the given node's (NId's) edges, but + /// will appear in an iteration over the 'other', unnamed node's edges. + /// + /// This does not correspond to any normal graph operation, but exists to + /// support efficient PBQP graph-reduction based solvers. It is used to + /// 'effectively' remove the unnamed node from the graph while the solver + /// is performing the reduction. The solver will later call reconnectNode + /// to restore the edge in the named node's adjacency list. + /// + /// Since the degree of a node is the number of connected edges, + /// disconnecting an edge from a node 'u' will cause the degree of 'u' to + /// drop by 1. + /// + /// A disconnected edge WILL still appear in an iteration over the graph + /// edges. + /// + /// A disconnected edge should not be removed from the graph, it should be + /// reconnected first. + /// + /// A disconnected edge can be reconnected by calling the reconnectEdge + /// method. + void disconnectEdge(EdgeId EId, NodeId NId) { + if (Solver) + Solver->handleDisconnectEdge(EId, NId); + + EdgeEntry &E = getEdge(EId); + E.disconnectFrom(*this, NId); + } + + /// \brief Convenience method to disconnect all neighbours from the given + /// node. + void disconnectAllNeighborsFromNode(NodeId NId) { + for (auto AEId : adjEdgeIds(NId)) + disconnectEdge(AEId, getEdgeOtherNodeId(AEId, NId)); + } + + /// \brief Re-attach an edge to its nodes. + /// + /// Adds an edge that had been previously disconnected back into the + /// adjacency set of the nodes that the edge connects. + void reconnectEdge(EdgeId EId, NodeId NId) { + EdgeEntry &E = getEdge(EId); + E.connectTo(*this, EId, NId); + if (Solver) + Solver->handleReconnectEdge(EId, NId); } /// \brief Remove an edge from the graph. - /// @param eId Edge id. - void removeEdge(EdgeId eId) { - EdgeEntry &e = getEdge(eId); - NodeEntry &n1 = getNode(e.getNode1()); - NodeEntry &n2 = getNode(e.getNode2()); - n1.removeEdge(e.getNode1AEItr()); - n2.removeEdge(e.getNode2AEItr()); - freeEdges.push_back(eId); + /// @param EId Edge id. + void removeEdge(EdgeId EId) { + if (Solver) + Solver->handleRemoveEdge(EId); + EdgeEntry &E = getEdge(EId); + E.disconnect(); + FreeEdgeIds.push_back(EId); + Edges[EId].invalidate(); } /// \brief Remove all nodes and edges from the graph. void clear() { - nodes.clear(); - freeNodes.clear(); - edges.clear(); - freeEdges.clear(); + Nodes.clear(); + FreeNodeIds.clear(); + Edges.clear(); + FreeEdgeIds.clear(); } /// \brief Dump a graph to an output stream. template <typename OStream> - void dump(OStream &os) { - os << getNumNodes() << " " << getNumEdges() << "\n"; - - for (NodeItr nodeItr = nodesBegin(), nodeEnd = nodesEnd(); - nodeItr != nodeEnd; ++nodeItr) { - const Vector& v = getNodeCosts(*nodeItr); - os << "\n" << v.getLength() << "\n"; - assert(v.getLength() != 0 && "Empty vector in graph."); - os << v[0]; - for (unsigned i = 1; i < v.getLength(); ++i) { - os << " " << v[i]; + void dump(OStream &OS) { + OS << nodeIds().size() << " " << edgeIds().size() << "\n"; + + for (auto NId : nodeIds()) { + const Vector& V = getNodeCosts(NId); + OS << "\n" << V.getLength() << "\n"; + assert(V.getLength() != 0 && "Empty vector in graph."); + OS << V[0]; + for (unsigned i = 1; i < V.getLength(); ++i) { + OS << " " << V[i]; } - os << "\n"; + OS << "\n"; } - for (EdgeItr edgeItr = edgesBegin(), edgeEnd = edgesEnd(); - edgeItr != edgeEnd; ++edgeItr) { - NodeId n1 = getEdgeNode1(*edgeItr); - NodeId n2 = getEdgeNode2(*edgeItr); - assert(n1 != n2 && "PBQP graphs shound not have self-edges."); - const Matrix& m = getEdgeCosts(*edgeItr); - os << "\n" << n1 << " " << n2 << "\n" - << m.getRows() << " " << m.getCols() << "\n"; - assert(m.getRows() != 0 && "No rows in matrix."); - assert(m.getCols() != 0 && "No cols in matrix."); - for (unsigned i = 0; i < m.getRows(); ++i) { - os << m[i][0]; - for (unsigned j = 1; j < m.getCols(); ++j) { - os << " " << m[i][j]; + for (auto EId : edgeIds()) { + NodeId N1Id = getEdgeNode1Id(EId); + NodeId N2Id = getEdgeNode2Id(EId); + assert(N1Id != N2Id && "PBQP graphs shound not have self-edges."); + const Matrix& M = getEdgeCosts(EId); + OS << "\n" << N1Id << " " << N2Id << "\n" + << M.getRows() << " " << M.getCols() << "\n"; + assert(M.getRows() != 0 && "No rows in matrix."); + assert(M.getCols() != 0 && "No cols in matrix."); + for (unsigned i = 0; i < M.getRows(); ++i) { + OS << M[i][0]; + for (unsigned j = 1; j < M.getCols(); ++j) { + OS << " " << M[i][j]; } - os << "\n"; + OS << "\n"; } } } /// \brief Print a representation of this graph in DOT format. - /// @param os Output stream to print on. + /// @param OS Output stream to print on. template <typename OStream> - void printDot(OStream &os) { - - os << "graph {\n"; - - for (NodeItr nodeItr = nodesBegin(), nodeEnd = nodesEnd(); - nodeItr != nodeEnd; ++nodeItr) { - - os << " node" << nodeItr << " [ label=\"" - << nodeItr << ": " << getNodeCosts(*nodeItr) << "\" ]\n"; + void printDot(OStream &OS) { + OS << "graph {\n"; + for (auto NId : nodeIds()) { + OS << " node" << NId << " [ label=\"" + << NId << ": " << getNodeCosts(NId) << "\" ]\n"; } - - os << " edge [ len=" << getNumNodes() << " ]\n"; - - for (EdgeItr edgeItr = edgesBegin(), edgeEnd = edgesEnd(); - edgeItr != edgeEnd; ++edgeItr) { - - os << " node" << getEdgeNode1(*edgeItr) - << " -- node" << getEdgeNode2(*edgeItr) + OS << " edge [ len=" << nodeIds().size() << " ]\n"; + for (auto EId : edgeIds()) { + OS << " node" << getEdgeNode1Id(EId) + << " -- node" << getEdgeNode2Id(EId) << " [ label=\""; - - const Matrix &edgeCosts = getEdgeCosts(*edgeItr); - - for (unsigned i = 0; i < edgeCosts.getRows(); ++i) { - os << edgeCosts.getRowAsVector(i) << "\\n"; + const Matrix &EdgeCosts = getEdgeCosts(EId); + for (unsigned i = 0; i < EdgeCosts.getRows(); ++i) { + OS << EdgeCosts.getRowAsVector(i) << "\\n"; } - os << "\" ]\n"; + OS << "\" ]\n"; } - os << "}\n"; + OS << "}\n"; } - }; -// void Graph::copyFrom(const Graph &other) { -// std::map<Graph::ConstNodeItr, Graph::NodeItr, -// NodeItrComparator> nodeMap; - -// for (Graph::ConstNodeItr nItr = other.nodesBegin(), -// nEnd = other.nodesEnd(); -// nItr != nEnd; ++nItr) { -// nodeMap[nItr] = addNode(other.getNodeCosts(nItr)); -// } -// } - } #endif // LLVM_CODEGEN_PBQP_GRAPH_HPP diff --git a/include/llvm/CodeGen/PBQP/HeuristicBase.h b/include/llvm/CodeGen/PBQP/HeuristicBase.h deleted file mode 100644 index 8bcbb9ed1d..0000000000 --- a/include/llvm/CodeGen/PBQP/HeuristicBase.h +++ /dev/null @@ -1,247 +0,0 @@ -//===-- HeuristcBase.h --- Heuristic base class for PBQP --------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_PBQP_HEURISTICBASE_H -#define LLVM_CODEGEN_PBQP_HEURISTICBASE_H - -#include "HeuristicSolver.h" - -namespace PBQP { - - /// \brief Abstract base class for heuristic implementations. - /// - /// This class provides a handy base for heuristic implementations with common - /// solver behaviour implemented for a number of methods. - /// - /// To implement your own heuristic using this class as a base you'll have to - /// implement, as a minimum, the following methods: - /// <ul> - /// <li> void addToHeuristicList(Graph::NodeItr) : Add a node to the - /// heuristic reduction list. - /// <li> void heuristicReduce() : Perform a single heuristic reduction. - /// <li> void preUpdateEdgeCosts(Graph::EdgeItr) : Handle the (imminent) - /// change to the cost matrix on the given edge (by R2). - /// <li> void postUpdateEdgeCostts(Graph::EdgeItr) : Handle the new - /// costs on the given edge. - /// <li> void handleAddEdge(Graph::EdgeItr) : Handle the addition of a new - /// edge into the PBQP graph (by R2). - /// <li> void handleRemoveEdge(Graph::EdgeItr, Graph::NodeItr) : Handle the - /// disconnection of the given edge from the given node. - /// <li> A constructor for your derived class : to pass back a reference to - /// the solver which is using this heuristic. - /// </ul> - /// - /// These methods are implemented in this class for documentation purposes, - /// but will assert if called. - /// - /// Note that this class uses the curiously recursive template idiom to - /// forward calls to the derived class. These methods need not be made - /// virtual, and indeed probably shouldn't for performance reasons. - /// - /// You'll also need to provide NodeData and EdgeData structs in your class. - /// These can be used to attach data relevant to your heuristic to each - /// node/edge in the PBQP graph. - - template <typename HImpl> - class HeuristicBase { - private: - - typedef std::list<Graph::NodeId> OptimalList; - - HeuristicSolverImpl<HImpl> &s; - Graph &g; - OptimalList optimalList; - - // Return a reference to the derived heuristic. - HImpl& impl() { return static_cast<HImpl&>(*this); } - - // Add the given node to the optimal reductions list. Keep an iterator to - // its location for fast removal. - void addToOptimalReductionList(Graph::NodeId nId) { - optimalList.insert(optimalList.end(), nId); - } - - public: - - /// \brief Construct an instance with a reference to the given solver. - /// @param solver The solver which is using this heuristic instance. - HeuristicBase(HeuristicSolverImpl<HImpl> &solver) - : s(solver), g(s.getGraph()) { } - - /// \brief Get the solver which is using this heuristic instance. - /// @return The solver which is using this heuristic instance. - /// - /// You can use this method to get access to the solver in your derived - /// heuristic implementation. - HeuristicSolverImpl<HImpl>& getSolver() { return s; } - - /// \brief Get the graph representing the problem to be solved. - /// @return The graph representing the problem to be solved. - Graph& getGraph() { return g; } - - /// \brief Tell the solver to simplify the graph before the reduction phase. - /// @return Whether or not the solver should run a simplification phase - /// prior to the main setup and reduction. - /// - /// HeuristicBase returns true from this method as it's a sensible default, - /// however you can over-ride it in your derived class if you want different - /// behaviour. - bool solverRunSimplify() const { return true; } - - /// \brief Decide whether a node should be optimally or heuristically - /// reduced. - /// @return Whether or not the given node should be listed for optimal - /// reduction (via R0, R1 or R2). - /// - /// HeuristicBase returns true for any node with degree less than 3. This is - /// sane and sensible for many situations, but not all. You can over-ride - /// this method in your derived class if you want a different selection - /// criteria. Note however that your criteria for selecting optimal nodes - /// should be <i>at least</i> as strong as this. I.e. Nodes of degree 3 or - /// higher should not be selected under any circumstances. - bool shouldOptimallyReduce(Graph::NodeId nId) { - if (g.getNodeDegree(nId) < 3) - return true; - // else - return false; - } - - /// \brief Add the given node to the list of nodes to be optimally reduced. - /// @param nId Node id to be added. - /// - /// You probably don't want to over-ride this, except perhaps to record - /// statistics before calling this implementation. HeuristicBase relies on - /// its behaviour. - void addToOptimalReduceList(Graph::NodeId nId) { - optimalList.push_back(nId); - } - - /// \brief Initialise the heuristic. - /// - /// HeuristicBase iterates over all nodes in the problem and adds them to - /// the appropriate list using addToOptimalReduceList or - /// addToHeuristicReduceList based on the result of shouldOptimallyReduce. - /// - /// This behaviour should be fine for most situations. - void setup() { - for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd(); - nItr != nEnd; ++nItr) { - if (impl().shouldOptimallyReduce(*nItr)) { - addToOptimalReduceList(*nItr); - } else { - impl().addToHeuristicReduceList(*nItr); - } - } - } - - /// \brief Optimally reduce one of the nodes in the optimal reduce list. - /// @return True if a reduction takes place, false if the optimal reduce - /// list is empty. - /// - /// Selects a node from the optimal reduce list and removes it, applying - /// R0, R1 or R2 as appropriate based on the selected node's degree. - bool optimalReduce() { - if (optimalList.empty()) - return false; - - Graph::NodeId nId = optimalList.front(); - optimalList.pop_front(); - - switch (s.getSolverDegree(nId)) { - case 0: s.applyR0(nId); break; - case 1: s.applyR1(nId); break; - case 2: s.applyR2(nId); break; - default: llvm_unreachable( - "Optimal reductions of degree > 2 nodes is invalid."); - } - - return true; - } - - /// \brief Perform the PBQP reduction process. - /// - /// Reduces the problem to the empty graph by repeated application of the - /// reduction rules R0, R1, R2 and RN. - /// R0, R1 or R2 are always applied if possible before RN is used. - void reduce() { - bool finished = false; - - while (!finished) { - if (!optimalReduce()) { - if (impl().heuristicReduce()) { - getSolver().recordRN(); - } else { - finished = true; - } - } - } - } - - /// \brief Add a node to the heuristic reduce list. - /// @param nId Node id to add to the heuristic reduce list. - void addToHeuristicList(Graph::NodeId nId) { - llvm_unreachable("Must be implemented in derived class."); - } - - /// \brief Heuristically reduce one of the nodes in the heuristic - /// reduce list. - /// @return True if a reduction takes place, false if the heuristic reduce - /// list is empty. - bool heuristicReduce() { - llvm_unreachable("Must be implemented in derived class."); - return false; - } - - /// \brief Prepare a change in the costs on the given edge. - /// @param eId Edge id. - void preUpdateEdgeCosts(Graph::EdgeId eId) { - llvm_unreachable("Must be implemented in derived class."); - } - - /// \brief Handle the change in the costs on the given edge. - /// @param eId Edge id. - void postUpdateEdgeCostts(Graph::EdgeId eId) { - llvm_unreachable("Must be implemented in derived class."); - } - - /// \brief Handle the addition of a new edge into the PBQP graph. - /// @param eId Edge id for the added edge. - void handleAddEdge(Graph::EdgeId eId) { - llvm_unreachable("Must be implemented in derived class."); - } - - /// \brief Handle disconnection of an edge from a node. - /// @param eId Edge id for edge being disconnected. - /// @param nId Node id for the node being disconnected from. - /// - /// Edges are frequently removed due to the removal of a node. This - /// method allows for the effect to be computed only for the remaining - /// node in the graph. - void handleRemoveEdge(Graph::EdgeId eId, Graph::NodeId nId) { - llvm_unreachable("Must be implemented in derived class."); - } - - /// \brief Clean up any structures used by HeuristicBase. - /// - /// At present this just performs a sanity check: that the optimal reduce - /// list is empty now that reduction has completed. - /// - /// If your derived class has more complex structures which need tearing - /// down you should over-ride this method but include a call back to this - /// implementation. - void cleanup() { - assert(optimalList.empty() && "Nodes left over in optimal reduce list?"); - } - - }; - -} - - -#endif // LLVM_CODEGEN_PBQP_HEURISTICBASE_H diff --git a/include/llvm/CodeGen/PBQP/HeuristicSolver.h b/include/llvm/CodeGen/PBQP/HeuristicSolver.h deleted file mode 100644 index e26ca02fff..0000000000 --- a/include/llvm/CodeGen/PBQP/HeuristicSolver.h +++ /dev/null @@ -1,618 +0,0 @@ -//===-- HeuristicSolver.h - Heuristic PBQP Solver --------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Heuristic PBQP solver. This solver is able to perform optimal reductions for -// nodes of degree 0, 1 or 2. For nodes of degree >2 a plugable heuristic is -// used to select a node for reduction. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H -#define LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H - -#include "Graph.h" -#include "Solution.h" -#include <limits> -#include <vector> - -namespace PBQP { - - /// \brief Heuristic PBQP solver implementation. - /// - /// This class should usually be created (and destroyed) indirectly via a call - /// to HeuristicSolver<HImpl>::solve(Graph&). - /// See the comments for HeuristicSolver. - /// - /// HeuristicSolverImpl provides the R0, R1 and R2 reduction rules, - /// backpropagation phase, and maintains the internal copy of the graph on - /// which the reduction is carried out (the original being kept to facilitate - /// backpropagation). - template <typename HImpl> - class HeuristicSolverImpl { - private: - - typedef typename HImpl::NodeData HeuristicNodeData; - typedef typename HImpl::EdgeData HeuristicEdgeData; - - typedef std::list<Graph::EdgeId> SolverEdges; - - public: - - /// \brief Iterator type for edges in the solver graph. - typedef SolverEdges::iterator SolverEdgeItr; - - private: - - class NodeData { - public: - NodeData() : solverDegree(0) {} - - HeuristicNodeData& getHeuristicData() { return hData; } - - SolverEdgeItr addSolverEdge(Graph::EdgeId eId) { - ++solverDegree; - return solverEdges.insert(solverEdges.end(), eId); - } - - void removeSolverEdge(SolverEdgeItr seItr) { - --solverDegree; - solverEdges.erase(seItr); - } - - SolverEdgeItr solverEdgesBegin() { return solverEdges.begin(); } - SolverEdgeItr solverEdgesEnd() { return solverEdges.end(); } - unsigned getSolverDegree() const { return solverDegree; } - void clearSolverEdges() { - solverDegree = 0; - solverEdges.clear(); - } - - private: - HeuristicNodeData hData; - unsigned solverDegree; - SolverEdges solverEdges; - }; - - class EdgeData { - public: - HeuristicEdgeData& getHeuristicData() { return hData; } - - void setN1SolverEdgeItr(SolverEdgeItr n1SolverEdgeItr) { - this->n1SolverEdgeItr = n1SolverEdgeItr; - } - - SolverEdgeItr getN1SolverEdgeItr() { return n1SolverEdgeItr; } - - void setN2SolverEdgeItr(SolverEdgeItr n2SolverEdgeItr){ - this->n2SolverEdgeItr = n2SolverEdgeItr; - } - - SolverEdgeItr getN2SolverEdgeItr() { return n2SolverEdgeItr; } - - private: - - HeuristicEdgeData hData; - SolverEdgeItr n1SolverEdgeItr, n2SolverEdgeItr; - }; - - Graph &g; - HImpl h; - Solution s; - std::vector<Graph::NodeId> stack; - - typedef std::list<NodeData> NodeDataList; - NodeDataList nodeDataList; - - typedef std::list<EdgeData> EdgeDataList; - EdgeDataList edgeDataList; - - public: - - /// \brief Construct a heuristic solver implementation to solve the given - /// graph. - /// @param g The graph representing the problem instance to be solved. - HeuristicSolverImpl(Graph &g) : g(g), h(*this) {} - - /// \brief Get the graph being solved by this solver. - /// @return The graph representing the problem instance being solved by this - /// solver. - Graph& getGraph() { return g; } - - /// \brief Get the heuristic data attached to the given node. - /// @param nId Node id. - /// @return The heuristic data attached to the given node. - HeuristicNodeData& getHeuristicNodeData(Graph::NodeId nId) { - return getSolverNodeData(nId).getHeuristicData(); - } - - /// \brief Get the heuristic data attached to the given edge. - /// @param eId Edge id. - /// @return The heuristic data attached to the given node. - HeuristicEdgeData& getHeuristicEdgeData(Graph::EdgeId eId) { - return getSolverEdgeData(eId).getHeuristicData(); - } - - /// \brief Begin iterator for the set of edges adjacent to the given node in - /// the solver graph. - /// @param nId Node id. - /// @return Begin iterator for the set of edges adjacent to the given node - /// in the solver graph. - SolverEdgeItr solverEdgesBegin(Graph::NodeId nId) { - return getSolverNodeData(nId).solverEdgesBegin(); - } - - /// \brief End iterator for the set of edges adjacent to the given node in - /// the solver graph. - /// @param nId Node id. - /// @return End iterator for the set of edges adjacent to the given node in - /// the solver graph. - SolverEdgeItr solverEdgesEnd(Graph::NodeId nId) { - return getSolverNodeData(nId).solverEdgesEnd(); - } - - /// \brief Remove a node from the solver graph. - /// @param eId Edge id for edge to be removed. - /// - /// Does <i>not</i> notify the heuristic of the removal. That should be - /// done manually if necessary. - void removeSolverEdge(Graph::EdgeId eId) { - EdgeData &eData = getSolverEdgeData(eId); - NodeData &n1Data = getSolverNodeData(g.getEdgeNode1(eId)), - &n2Data = getSolverNodeData(g.getEdgeNode2(eId)); - - n1Data.removeSolverEdge(eData.getN1SolverEdgeItr()); - n2Data.removeSolverEdge(eData.getN2SolverEdgeItr()); - } - - /// \brief Compute a solution to the PBQP problem instance with which this - /// heuristic solver was constructed. - /// @return A solution to the PBQP problem. - /// - /// Performs the full PBQP heuristic solver algorithm, including setup, - /// calls to the heuristic (which will call back to the reduction rules in - /// this class), and cleanup. - Solution computeSolution() { - setup(); - h.setup(); - h.reduce(); - backpropagate(); - h.cleanup(); - cleanup(); - return s; - } - - /// \brief Add to the end of the stack. - /// @param nId Node id to add to the reduction stack. - void pushToStack(Graph::NodeId nId) { - getSolverNodeData(nId).clearSolverEdges(); - stack.push_back(nId); - } - - /// \brief Returns the solver degree of the given node. - /// @param nId Node id for which degree is requested. - /// @return Node degree in the <i>solver</i> graph (not the original graph). - unsigned getSolverDegree(Graph::NodeId nId) { - return getSolverNodeData(nId).getSolverDegree(); - } - - /// \brief Set the solution of the given node. - /// @param nId Node id to set solution for. - /// @param selection Selection for node. - void setSolution(const Graph::NodeId &nId, unsigned selection) { - s.setSelection(nId, selection); - - for (Graph::AdjEdgeItr aeItr = g.adjEdgesBegin(nId), - aeEnd = g.adjEdgesEnd(nId); - aeItr != aeEnd; ++aeItr) { - Graph::EdgeId eId(*aeItr); - Graph::NodeId anId(g.getEdgeOtherNode(eId, nId)); - getSolverNodeData(anId).addSolverEdge(eId); - } - } - - /// \brief Apply rule R0. - /// @param nId Node id for node to apply R0 to. - /// - /// Node will be automatically pushed to the solver stack. - void applyR0(Graph::NodeId nId) { - assert(getSolverNodeData(nId).getSolverDegree() == 0 && - "R0 applied to node with degree != 0."); - - // Nothing to do. Just push the node onto the reduction stack. - pushToStack(nId); - - s.recordR0(); - } - - /// \brief Apply rule R1. - /// @param xnId Node id for node to apply R1 to. - /// - /// Node will be automatically pushed to the solver stack. - void applyR1(Graph::NodeId xnId) { - NodeData &nd = getSolverNodeData(xnId); - assert(nd.getSolverDegree() == 1 && - "R1 applied to node with degree != 1."); - - Graph::EdgeId eId = *nd.solverEdgesBegin(); - - const Matrix &eCosts = g.getEdgeCosts(eId); - const Vector &xCosts = g.getNodeCosts(xnId); - - // Duplicate a little to avoid transposing matrices. - if (xnId == g.getEdgeNode1(eId)) { - Graph::NodeId ynId = g.getEdgeNode2(eId); - Vector &yCosts = g.getNodeCosts(ynId); - for (unsigned j = 0; j < yCosts.getLength(); ++j) { - PBQPNum min = eCosts[0][j] + xCosts[0]; - for (unsigned i = 1; i < xCosts.getLength(); ++i) { - PBQPNum c = eCosts[i][j] + xCosts[i]; - if (c < min) - min = c; - } - yCosts[j] += min; - } - h.handleRemoveEdge(eId, ynId); - } else { - Graph::NodeId ynId = g.getEdgeNode1(eId); - Vector &yCosts = g.getNodeCosts(ynId); - for (unsigned i = 0; i < yCosts.getLength(); ++i) { - PBQPNum min = eCosts[i][0] + xCosts[0]; - for (unsigned j = 1; j < xCosts.getLength(); ++j) { - PBQPNum c = eCosts[i][j] + xCosts[j]; - if (c < min) - min = c; - } - yCosts[i] += min; - } - h.handleRemoveEdge(eId, ynId); - } - removeSolverEdge(eId); - assert(nd.getSolverDegree() == 0 && - "Degree 1 with edge removed should be 0."); - pushToStack(xnId); - s.recordR1(); - } - - /// \brief Apply rule R2. - /// @param xnId Node id for node to apply R2 to. - /// - /// Node will be automatically pushed to the solver stack. - void applyR2(Graph::NodeId xnId) { - assert(getSolverNodeData(xnId).getSolverDegree() == 2 && - "R2 applied to node with degree != 2."); - - NodeData &nd = getSolverNodeData(xnId); - const Vector &xCosts = g.getNodeCosts(xnId); - - SolverEdgeItr aeItr = nd.solverEdgesBegin(); - Graph::EdgeId yxeId = *aeItr, - zxeId = *(++aeItr); - - Graph::NodeId ynId = g.getEdgeOtherNode(yxeId, xnId), - znId = g.getEdgeOtherNode(zxeId, xnId); - - bool flipEdge1 = (g.getEdgeNode1(yxeId) == xnId), - flipEdge2 = (g.getEdgeNode1(zxeId) == xnId); - - const Matrix *yxeCosts = flipEdge1 ? - new Matrix(g.getEdgeCosts(yxeId).transpose()) : - &g.getEdgeCosts(yxeId); - - const Matrix *zxeCosts = flipEdge2 ? - new Matrix(g.getEdgeCosts(zxeId).transpose()) : - &g.getEdgeCosts(zxeId); - - unsigned xLen = xCosts.getLength(), - yLen = yxeCosts->getRows(), - zLen = zxeCosts->getRows(); - - Matrix delta(yLen, zLen); - - for (unsigned i = 0; i < yLen; ++i) { - for (unsigned j = 0; j < zLen; ++j) { - PBQPNum min = (*yxeCosts)[i][0] + (*zxeCosts)[j][0] + xCosts[0]; - for (unsigned k = 1; k < xLen; ++k) { - PBQPNum c = (*yxeCosts)[i][k] + (*zxeCosts)[j][k] + xCosts[k]; - if (c < min) { - min = c; - } - } - delta[i][j] = min; - } - } - - if (flipEdge1) - delete yxeCosts; - - if (flipEdge2) - delete zxeCosts; - - Graph::EdgeId yzeId = g.findEdge(ynId, znId); - bool addedEdge = false; - - if (yzeId == g.invalidEdgeId()) { - yzeId = g.addEdge(ynId, znId, delta); - addedEdge = true; - } else { - Matrix &yzeCosts = g.getEdgeCosts(yzeId); - h.preUpdateEdgeCosts(yzeId); - if (ynId == g.getEdgeNode1(yzeId)) { - yzeCosts += delta; - } else { - yzeCosts += delta.transpose(); - } - } - - bool nullCostEdge = tryNormaliseEdgeMatrix(yzeId); - - if (!addedEdge) { - // If we modified the edge costs let the heuristic know. - h.postUpdateEdgeCosts(yzeId); - } - - if (nullCostEdge) { - // If this edge ended up null remove it. - if (!addedEdge) { - // We didn't just add it, so we need to notify the heuristic - // and remove it from the solver. - h.handleRemoveEdge(yzeId, ynId); - h.handleRemoveEdge(yzeId, znId); - removeSolverEdge(yzeId); - } - g.removeEdge(yzeId); - } else if (addedEdge) { - // If the edge was added, and non-null, finish setting it up, add it to - // the solver & notify heuristic. - edgeDataList.push_back(EdgeData()); - g.setEdgeData(yzeId, &edgeDataList.back()); - addSolverEdge(yzeId); - h.handleAddEdge(yzeId); - } - - h.handleRemoveEdge(yxeId, ynId); - removeSolverEdge(yxeId); - h.handleRemoveEdge(zxeId, znId); - removeSolverEdge(zxeId); - - pushToStack(xnId); - s.recordR2(); - } - - /// \brief Record an application of the RN rule. - /// - /// For use by the HeuristicBase. - void recordRN() { s.recordRN(); } - - private: - - NodeData& getSolverNodeData(Graph::NodeId nId) { - return *static_cast<NodeData*>(g.getNodeData(nId)); - } - - EdgeData& getSolverEdgeData(Graph::EdgeId eId) { - return *static_cast<EdgeData*>(g.getEdgeData(eId)); - } - - void addSolverEdge(Graph::EdgeId eId) { - EdgeData &eData = getSolverEdgeData(eId); - NodeData &n1Data = getSolverNodeData(g.getEdgeNode1(eId)), - &n2Data = getSolverNodeData(g.getEdgeNode2(eId)); - - eData.setN1SolverEdgeItr(n1Data.addSolverEdge(eId)); - eData.setN2SolverEdgeItr(n2Data.addSolverEdge(eId)); - } - - void setup() { - if (h.solverRunSimplify()) { - simplify(); - } - - // Create node data objects. - for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd(); - nItr != nEnd; ++nItr) { - nodeDataList.push_back(NodeData()); - g.setNodeData(*nItr, &nodeDataList.back()); - } - - // Create edge data objects. - for (Graph::EdgeItr eItr = g.edgesBegin(), eEnd = g.edgesEnd(); - eItr != eEnd; ++eItr) { - edgeDataList.push_back(EdgeData()); - g.setEdgeData(*eItr, &edgeDataList.back()); - addSolverEdge(*eItr); - } - } - - void simplify() { - disconnectTrivialNodes(); - eliminateIndependentEdges(); - } - - // Eliminate trivial nodes. - void disconnectTrivialNodes() { - unsigned numDisconnected = 0; - - for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd(); - nItr != nEnd; ++nItr) { - - Graph::NodeId nId = *nItr; - - if (g.getNodeCosts(nId).getLength() == 1) { - - std::vector<Graph::EdgeId> edgesToRemove; - - for (Graph::AdjEdgeItr aeItr = g.adjEdgesBegin(nId), - aeEnd = g.adjEdgesEnd(nId); - aeItr != aeEnd; ++aeItr) { - - Graph::EdgeId eId = *aeItr; - - if (g.getEdgeNode1(eId) == nId) { - Graph::NodeId otherNodeId = g.getEdgeNode2(eId); - g.getNodeCosts(otherNodeId) += - g.getEdgeCosts(eId).getRowAsVector(0); - } - else { - Graph::NodeId otherNodeId = g.getEdgeNode1(eId); - g.getNodeCosts(otherNodeId) += - g.getEdgeCosts(eId).getColAsVector(0); - } - - edgesToRemove.push_back(eId); - } - - if (!edgesToRemove.empty()) - ++numDisconnected; - - while (!edgesToRemove.empty()) { - g.removeEdge(edgesToRemove.back()); - edgesToRemove.pop_back(); - } - } - } - } - - void eliminateIndependentEdges() { - std::vector<Graph::EdgeId> edgesToProcess; - unsigned numEliminated = 0; - - for (Graph::EdgeItr eItr = g.edgesBegin(), eEnd = g.edgesEnd(); - eItr != eEnd; ++eItr) { - edgesToProcess.push_back(*eItr); - } - - while (!edgesToProcess.empty()) { - if (tryToEliminateEdge(edgesToProcess.back())) - ++numEliminated; - edgesToProcess.pop_back(); - } - } - - bool tryToEliminateEdge(Graph::EdgeId eId) { - if (tryNormaliseEdgeMatrix(eId)) { - g.removeEdge(eId); - return true; - } - return false; - } - - bool tryNormaliseEdgeMatrix(Graph::EdgeId &eId) { - - const PBQPNum infinity = std::numeric_limits<PBQPNum>::infinity(); - - Matrix &edgeCosts = g.getEdgeCosts(eId); - Vector &uCosts = g.getNodeCosts(g.getEdgeNode1(eId)), - &vCosts = g.getNodeCosts(g.getEdgeNode2(eId)); - - for (unsigned r = 0; r < edgeCosts.getRows(); ++r) { - PBQPNum rowMin = infinity; - - for (unsigned c = 0; c < edgeCosts.getCols(); ++c) { - if (vCosts[c] != infinity && edgeCosts[r][c] < rowMin) - rowMin = edgeCosts[r][c]; - } - - uCosts[r] += rowMin; - - if (rowMin != infinity) { - edgeCosts.subFromRow(r, rowMin); - } - else { - edgeCosts.setRow(r, 0); - } - } - - for (unsigned c = 0; c < edgeCosts.getCols(); ++c) { - PBQPNum colMin = infinity; - - for (unsigned r = 0; r < edgeCosts.getRows(); ++r) { - if (uCosts[r] != infinity && edgeCosts[r][c] < colMin) - colMin = edgeCosts[r][c]; - } - - vCosts[c] += colMin; - - if (colMin != infinity) { - edgeCosts.subFromCol(c, colMin); - } - else { - edgeCosts.setCol(c, 0); - } - } - - return edgeCosts.isZero(); - } - - void backpropagate() { - while (!stack.empty()) { - computeSolution(stack.back()); - stack.pop_back(); - } - } - - void computeSolution(Graph::NodeId nId) { - - NodeData &nodeData = getSolverNodeData(nId); - - Vector v(g.getNodeCosts(nId)); - - // Solve based on existing solved edges. - for (SolverEdgeItr solvedEdgeItr = nodeData.solverEdgesBegin(), - solvedEdgeEnd = nodeData.solverEdgesEnd(); - solvedEdgeItr != solvedEdgeEnd; ++solvedEdgeItr) { - - Graph::EdgeId eId(*solvedEdgeItr); - Matrix &edgeCosts = g.getEdgeCosts(eId); - - if (nId == g.getEdgeNode1(eId)) { - Graph::NodeId adjNode(g.getEdgeNode2(eId)); - unsigned adjSolution = s.getSelection(adjNode); - v += edgeCosts.getColAsVector(adjSolution); - } - else { - Graph::NodeId adjNode(g.getEdgeNode1(eId)); - unsigned adjSolution = s.getSelection(adjNode); - v += edgeCosts.getRowAsVector(adjSolution); - } - - } - - setSolution(nId, v.minIndex()); - } - - void cleanup() { - h.cleanup(); - nodeDataList.clear(); - edgeDataList.clear(); - } - }; - - /// \brief PBQP heuristic solver class. - /// - /// Given a PBQP Graph g representing a PBQP problem, you can find a solution - /// by calling - /// <tt>Solution s = HeuristicSolver<H>::solve(g);</tt> - /// - /// The choice of heuristic for the H parameter will affect both the solver - /// speed and solution quality. The heuristic should be chosen based on the - /// nature of the problem being solved. - /// Currently the only solver included with LLVM is the Briggs heuristic for - /// register allocation. - template <typename HImpl> - class HeuristicSolver { - public: - static Solution solve(Graph &g) { - HeuristicSolverImpl<HImpl> hs(g); - return hs.computeSolution(); - } - }; - -} - -#endif // LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H diff --git a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h b/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h deleted file mode 100644 index c355c2c2f8..0000000000 --- a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h +++ /dev/null @@ -1,468 +0,0 @@ -//===-- Briggs.h --- Briggs Heuristic for PBQP ------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This class implements the Briggs test for "allocability" of nodes in a -// PBQP graph representing a register allocation problem. Nodes which can be -// proven allocable (by a safe and relatively accurate test) are removed from -// the PBQP graph first. If no provably allocable node is present in the graph -// then the node with the minimal spill-cost to degree ratio is removed. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H -#define LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H - -#include "../HeuristicBase.h" -#include "../HeuristicSolver.h" -#include <limits> - -namespace PBQP { - namespace Heuristics { - - /// \brief PBQP Heuristic which applies an allocability test based on - /// Briggs. - /// - /// This heuristic assumes that the elements of cost vectors in the PBQP - /// problem represent storage options, with the first being the spill - /// option and subsequent elements representing legal registers for the - /// corresponding node. Edge cost matrices are likewise assumed to represent - /// register constraints. - /// If one or more nodes can be proven allocable by this heuristic (by - /// inspection of their constraint matrices) then the allocable node of - /// highest degree is selected for the next reduction and pushed to the - /// solver stack. If no nodes can be proven allocable then the node with - /// the lowest estimated spill cost is selected and push to the solver stack - /// instead. - /// - /// This implementation is built on top of HeuristicBase. - class Briggs : public HeuristicBase<Briggs> { - private: - - class LinkDegreeComparator { - public: - LinkDegreeComparator(HeuristicSolverImpl<Briggs> &s) : s(&s) {} - bool operator()(Graph::NodeId n1Id, Graph::NodeId n2Id) const { - if (s->getSolverDegree(n1Id) > s->getSolverDegree(n2Id)) - return true; - return false; - } - private: - HeuristicSolverImpl<Briggs> *s; - }; - - class SpillCostComparator { - public: - SpillCostComparator(HeuristicSolverImpl<Briggs> &s) - : s(&s), g(&s.getGraph()) {} - bool operator()(Graph::NodeId n1Id, Graph::NodeId n2Id) const { - const PBQP::Vector &cv1 = g->getNodeCosts(n1Id); - const PBQP::Vector &cv2 = g->getNodeCosts(n2Id); - - PBQPNum cost1 = cv1[0] / s->getSolverDegree(n1Id); - PBQPNum cost2 = cv2[0] / s->getSolverDegree(n2Id); - - if (cost1 < cost2) - return true; - return false; - } - - private: - HeuristicSolverImpl<Briggs> *s; - Graph *g; - }; - - typedef std::list<Graph::NodeId> RNAllocableList; - typedef RNAllocableList::iterator RNAllocableListItr; - - typedef std::list<Graph::NodeId> RNUnallocableList; - typedef RNUnallocableList::iterator RNUnallocableListItr; - - public: - - struct NodeData { - typedef std::vector<unsigned> UnsafeDegreesArray; - bool isHeuristic, isAllocable, isInitialized; - unsigned numDenied, numSafe; - UnsafeDegreesArray unsafeDegrees; - RNAllocableListItr rnaItr; - RNUnallocableListItr rnuItr; - - NodeData() - : isHeuristic(false), isAllocable(false), isInitialized(false), - numDenied(0), numSafe(0) { } - }; - - struct EdgeData { - typedef std::vector<unsigned> UnsafeArray; - unsigned worst, reverseWorst; - UnsafeArray unsafe, reverseUnsafe; - bool isUpToDate; - - EdgeData() : worst(0), reverseWorst(0), isUpToDate(false) {} - }; - - /// \brief Construct an instance of the Briggs heuristic. - /// @param solver A reference to the solver which is using this heuristic. - Briggs(HeuristicSolverImpl<Briggs> &solver) : - HeuristicBase<Briggs>(solver) {} - - /// \brief Determine whether a node should be reduced using optimal - /// reduction. - /// @param nId Node id to be considered. - /// @return True if the given node should be optimally reduced, false - /// otherwise. - /// - /// Selects nodes of degree 0, 1 or 2 for optimal reduction, with one - /// exception. Nodes whose spill cost (element 0 of their cost vector) is - /// infinite are checked for allocability first. Allocable nodes may be - /// optimally reduced, but nodes whose allocability cannot be proven are - /// selected for heuristic reduction instead. - bool shouldOptimallyReduce(Graph::NodeId nId) { - if (getSolver().getSolverDegree(nId) < 3) { - return true; - } - // else - return false; - } - - /// \brief Add a node to the heuristic reduce list. - /// @param nId Node id to add to the heuristic reduce list. - void addToHeuristicReduceList(Graph::NodeId nId) { - NodeData &nd = getHeuristicNodeData(nId); - initializeNode(nId); - nd.isHeuristic = true; - if (nd.isAllocable) { - nd.rnaItr = rnAllocableList.insert(rnAllocableList.end(), nId); - } else { - nd.rnuItr = rnUnallocableList.insert(rnUnallocableList.end(), nId); - } - } - - /// \brief Heuristically reduce one of the nodes in the heuristic - /// reduce list. - /// @return True if a reduction takes place, false if the heuristic reduce - /// list is empty. - /// - /// If the list of allocable nodes is non-empty a node is selected - /// from it and pushed to the stack. Otherwise if the non-allocable list - /// is non-empty a node is selected from it and pushed to the stack. - /// If both lists are empty the method simply returns false with no action - /// taken. - bool heuristicReduce() { - if (!rnAllocableList.empty()) { - RNAllocableListItr rnaItr = - min_element(rnAllocableList.begin(), rnAllocableList.end(), - LinkDegreeComparator(getSolver())); - Graph::NodeId nId = *rnaItr; - rnAllocableList.erase(rnaItr); - handleRemoveNode(nId); - getSolver().pushToStack(nId); - return true; - } else if (!rnUnallocableList.empty()) { - RNUnallocableListItr rnuItr = - min_element(rnUnallocableList.begin(), rnUnallocableList.end(), - SpillCostComparator(getSolver())); - Graph::NodeId nId = *rnuItr; - rnUnallocableList.erase(rnuItr); - handleRemoveNode(nId); - getSolver().pushToStack(nId); - return true; - } - // else - return false; - } - - /// \brief Prepare a change in the costs on the given edge. - /// @param eId Edge id. - void preUpdateEdgeCosts(Graph::EdgeId eId) { - Graph &g = getGraph(); - Graph::NodeId n1Id = g.getEdgeNode1(eId), - n2Id = g.getEdgeNode2(eId); - NodeData &n1 = getHeuristicNodeData(n1Id), - &n2 = getHeuristicNodeData(n2Id); - - if (n1.isHeuristic) - subtractEdgeContributions(eId, getGraph().getEdgeNode1(eId)); - if (n2.isHeuristic) - subtractEdgeContributions(eId, getGraph().getEdgeNode2(eId)); - - EdgeData &ed = getHeuristicEdgeData(eId); - ed.isUpToDate = false; - } - - /// \brief Handle the change in the costs on the given edge. - /// @param eId Edge id. - void postUpdateEdgeCosts(Graph::EdgeId eId) { - // This is effectively the same as adding a new edge now, since - // we've factored out the costs of the old one. - handleAddEdge(eId); - } - - /// \brief Handle the addition of a new edge into the PBQP graph. - /// @param eId Edge id for the added edge. - /// - /// Updates allocability of any nodes connected by this edge which are - /// being managed by the heuristic. If allocability changes they are - /// moved to the appropriate list. - void handleAddEdge(Graph::EdgeId eId) { - Graph &g = getGraph(); - Graph::NodeId n1Id = g.getEdgeNode1(eId), - n2Id = g.getEdgeNode2(eId); - NodeData &n1 = getHeuristicNodeData(n1Id), - &n2 = getHeuristicNodeData(n2Id); - - // If neither node is managed by the heuristic there's nothing to be - // done. - if (!n1.isHeuristic && !n2.isHeuristic) - return; - - // Ok - we need to update at least one node. - computeEdgeContributions(eId); - - // Update node 1 if it's managed by the heuristic. - if (n1.isHeuristic) { - bool n1WasAllocable = n1.isAllocable; - addEdgeContributions(eId, n1Id); - updateAllocability(n1Id); - if (n1WasAllocable && !n1.isAllocable) { - rnAllocableList.erase(n1.rnaItr); - n1.rnuItr = - rnUnallocableList.insert(rnUnallocableList.end(), n1Id); - } - } - - // Likewise for node 2. - if (n2.isHeuristic) { - bool n2WasAllocable = n2.isAllocable; - addEdgeContributions(eId, n2Id); - updateAllocability(n2Id); - if (n2WasAllocable && !n2.isAllocable) { - rnAllocableList.erase(n2.rnaItr); - n2.rnuItr = - rnUnallocableList.insert(rnUnallocableList.end(), n2Id); - } - } - } - - /// \brief Handle disconnection of an edge from a node. - /// @param eId Edge id for edge being disconnected. - /// @param nId Node id for the node being disconnected from. - /// - /// Updates allocability of the given node and, if appropriate, moves the - /// node to a new list. - void handleRemoveEdge(Graph::EdgeId eId, Graph::NodeId nId) { - NodeData &nd =getHeuristicNodeData(nId); - - // If the node is not managed by the heuristic there's nothing to be - // done. - if (!nd.isHeuristic) - return; - - EdgeData &ed = getHeuristicEdgeData(eId); - (void)ed; - assert(ed.isUpToDate && "Edge data is not up to date."); - - // Update node. - bool ndWasAllocable = nd.isAllocable; - subtractEdgeContributions(eId, nId); - updateAllocability(nId); - - // If the node has gone optimal... - if (shouldOptimallyReduce(nId)) { - nd.isHeuristic = false; - addToOptimalReduceList(nId); - if (ndWasAllocable) { - rnAllocableList.erase(nd.rnaItr); - } else { - rnUnallocableList.erase(nd.rnuItr); - } - } else { - // Node didn't go optimal, but we might have to move it - // from "unallocable" to "allocable". - if (!ndWasAllocable && nd.isAllocable) { - rnUnallocableList.erase(nd.rnuItr); - nd.rnaItr = rnAllocableList.insert(rnAllocableList.end(), nId); - } - } - } - - private: - - NodeData& getHeuristicNodeData(Graph::NodeId nId) { - return getSolver().getHeuristicNodeData(nId); - } - - EdgeData& getHeuristicEdgeData(Graph::EdgeId eId) { - return getSolver().getHeuristicEdgeData(eId); - } - - // Work out what this edge will contribute to the allocability of the - // nodes connected to it. - void computeEdgeContributions(Graph::EdgeId eId) { - EdgeData &ed = getHeuristicEdgeData(eId); - - if (ed.isUpToDate) - return; // Edge data is already up to date. - - Matrix &eCosts = getGraph().getEdgeCosts(eId); - - unsigned numRegs = eCosts.getRows() - 1, - numReverseRegs = eCosts.getCols() - 1; - - std::vector<unsigned> rowInfCounts(numRegs, 0), - colInfCounts(numReverseRegs, 0); - - ed.worst = 0; - ed.reverseWorst = 0; - ed.unsafe.clear(); - ed.unsafe.resize(numRegs, 0); - ed.reverseUnsafe.clear(); - ed.reverseUnsafe.resize(numReverseRegs, 0); - - for (unsigned i = 0; i < numRegs; ++i) { - for (unsigned j = 0; j < numReverseRegs; ++j) { - if (eCosts[i + 1][j + 1] == - std::numeric_limits<PBQPNum>::infinity()) { - ed.unsafe[i] = 1; - ed.reverseUnsafe[j] = 1; - ++rowInfCounts[i]; - ++colInfCounts[j]; - - if (colInfCounts[j] > ed.worst) { - ed.worst = colInfCounts[j]; - } - - if (rowInfCounts[i] > ed.reverseWorst) { - ed.reverseWorst = rowInfCounts[i]; - } - } - } - } - - ed.isUpToDate = true; - } - - // Add the contributions of the given edge to the given node's - // numDenied and safe members. No action is taken other than to update - // these member values. Once updated these numbers can be used by clients - // to update the node's allocability. - void addEdgeContributions(Graph::EdgeId eId, Graph::NodeId nId) { - EdgeData &ed = getHeuristicEdgeData(eId); - - assert(ed.isUpToDate && "Using out-of-date edge numbers."); - - NodeData &nd = getHeuristicNodeData(nId); - unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1; - - bool nIsNode1 = nId == getGraph().getEdgeNode1(eId); - EdgeData::UnsafeArray &unsafe = - nIsNode1 ? ed.unsafe : ed.reverseUnsafe; - nd.numDenied += nIsNode1 ? ed.worst : ed.reverseWorst; - - for (unsigned r = 0; r < numRegs; ++r) { - if (unsafe[r]) { - if (nd.unsafeDegrees[r]==0) { - --nd.numSafe; - } - ++nd.unsafeDegrees[r]; - } - } - } - - // Subtract the contributions of the given edge to the given node's - // numDenied and safe members. No action is taken other than to update - // these member values. Once updated these numbers can be used by clients - // to update the node's allocability. - void subtractEdgeContributions(Graph::EdgeId eId, Graph::NodeId nId) { - EdgeData &ed = getHeuristicEdgeData(eId); - - assert(ed.isUpToDate && "Using out-of-date edge numbers."); - - NodeData &nd = getHeuristicNodeData(nId); - unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1; - - bool nIsNode1 = nId == getGraph().getEdgeNode1(eId); - EdgeData::UnsafeArray &unsafe = - nIsNode1 ? ed.unsafe : ed.reverseUnsafe; - nd.numDenied -= nIsNode1 ? ed.worst : ed.reverseWorst; - - for (unsigned r = 0; r < numRegs; ++r) { - if (unsafe[r]) { - if (nd.unsafeDegrees[r] == 1) { - ++nd.numSafe; - } - --nd.unsafeDegrees[r]; - } - } - } - - void updateAllocability(Graph::NodeId nId) { - NodeData &nd = getHeuristicNodeData(nId); - unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1; - nd.isAllocable = nd.numDenied < numRegs || nd.numSafe > 0; - } - - void initializeNode(Graph::NodeId nId) { - NodeData &nd = getHeuristicNodeData(nId); - - if (nd.isInitialized) - return; // Node data is already up to date. - - unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1; - - nd.numDenied = 0; - const Vector& nCosts = getGraph().getNodeCosts(nId); - for (unsigned i = 1; i < nCosts.getLength(); ++i) { - if (nCosts[i] == std::numeric_limits<PBQPNum>::infinity()) - ++nd.numDenied; - } - - nd.numSafe = numRegs; - nd.unsafeDegrees.resize(numRegs, 0); - - typedef HeuristicSolverImpl<Briggs>::SolverEdgeItr SolverEdgeItr; - - for (SolverEdgeItr aeItr = getSolver().solverEdgesBegin(nId), - aeEnd = getSolver().solverEdgesEnd(nId); - aeItr != aeEnd; ++aeItr) { - - Graph::EdgeId eId = *aeItr; - computeEdgeContributions(eId); - addEdgeContributions(eId, nId); - } - - updateAllocability(nId); - nd.isInitialized = true; - } - - void handleRemoveNode(Graph::NodeId xnId) { - typedef HeuristicSolverImpl<Briggs>::SolverEdgeItr SolverEdgeItr; - std::vector<Graph::EdgeId> edgesToRemove; - for (SolverEdgeItr aeItr = getSolver().solverEdgesBegin(xnId), - aeEnd = getSolver().solverEdgesEnd(xnId); - aeItr != aeEnd; ++aeItr) { - Graph::NodeId ynId = getGraph().getEdgeOtherNode(*aeItr, xnId); - handleRemoveEdge(*aeItr, ynId); - edgesToRemove.push_back(*aeItr); - } - while (!edgesToRemove.empty()) { - getSolver().removeSolverEdge(edgesToRemove.back()); - edgesToRemove.pop_back(); - } - } - - RNAllocableList rnAllocableList; - RNUnallocableList rnUnallocableList; - }; - - } -} - - -#endif // LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h index 08f8b981ae..69a9d83cc0 100644 --- a/include/llvm/CodeGen/PBQP/Math.h +++ b/include/llvm/CodeGen/PBQP/Math.h @@ -20,268 +20,418 @@ typedef float PBQPNum; /// \brief PBQP Vector class. class Vector { - public: - - /// \brief Construct a PBQP vector of the given size. - explicit Vector(unsigned length) : - length(length), data(new PBQPNum[length]) { - } - - /// \brief Construct a PBQP vector with initializer. - Vector(unsigned length, PBQPNum initVal) : - length(length), data(new PBQPNum[length]) { - std::fill(data, data + length, initVal); - } - - /// \brief Copy construct a PBQP vector. - Vector(const Vector &v) : - length(v.length), data(new PBQPNum[length]) { - std::copy(v.data, v.data + length, data); - } - - /// \brief Destroy this vector, return its memory. - ~Vector() { delete[] data; } - - /// \brief Assignment operator. - Vector& operator=(const Vector &v) { - delete[] data; - length = v.length; - data = new PBQPNum[length]; - std::copy(v.data, v.data + length, data); - return *this; - } - - /// \brief Return the length of the vector - unsigned getLength() const { - return length; - } - - /// \brief Element access. - PBQPNum& operator[](unsigned index) { - assert(index < length && "Vector element access out of bounds."); - return data[index]; - } - - /// \brief Const element access. - const PBQPNum& operator[](unsigned index) const { - assert(index < length && "Vector element access out of bounds."); - return data[index]; - } - - /// \brief Add another vector to this one. - Vector& operator+=(const Vector &v) { - assert(length == v.length && "Vector length mismatch."); - std::transform(data, data + length, v.data, data, std::plus<PBQPNum>()); - return *this; - } - - /// \brief Subtract another vector from this one. - Vector& operator-=(const Vector &v) { - assert(length == v.length && "Vector length mismatch."); - std::transform(data, data + length, v.data, data, std::minus<PBQPNum>()); - return *this; - } - - /// \brief Returns the index of the minimum value in this vector - unsigned minIndex() const { - return std::min_element(data, data + length) - data; - } - - private: - unsigned length; - PBQPNum *data; + friend class VectorComparator; +public: + + /// \brief Construct a PBQP vector of the given size. + explicit Vector(unsigned Length) + : Length(Length), Data(new PBQPNum[Length]) { + // llvm::dbgs() << "Constructing PBQP::Vector " + // << this << " (length " << Length << ")\n"; + } + + /// \brief Construct a PBQP vector with initializer. + Vector(unsigned Length, PBQPNum InitVal) + : Length(Length), Data(new PBQPNum[Length]) { + // llvm::dbgs() << "Constructing PBQP::Vector " + // << this << " (length " << Length << ", fill " + // << InitVal << ")\n"; + std::fill(Data, Data + Length, InitVal); + } + + /// \brief Copy construct a PBQP vector. + Vector(const Vector &V) + : Length(V.Length), Data(new PBQPNum[Length]) { + // llvm::dbgs() << "Copy-constructing PBQP::Vector " << this + // << " from PBQP::Vector " << &V << "\n"; + std::copy(V.Data, V.Data + Length, Data); + } + + /// \brief Move construct a PBQP vector. + Vector(Vector &&V) + : Length(V.Length), Data(V.Data) { + V.Length = 0; + V.Data = nullptr; + } + + /// \brief Destroy this vector, return its memory. + ~Vector() { + // llvm::dbgs() << "Deleting PBQP::Vector " << this << "\n"; + delete[] Data; + } + + /// \brief Copy-assignment operator. + Vector& operator=(const Vector &V) { + // llvm::dbgs() << "Assigning to PBQP::Vector " << this + // << " from PBQP::Vector " << &V << "\n"; + delete[] Data; + Length = V.Length; + Data = new PBQPNum[Length]; + std::copy(V.Data, V.Data + Length, Data); + return *this; + } + + /// \brief Move-assignment operator. + Vector& operator=(Vector &&V) { + delete[] Data; + Length = V.Length; + Data = V.Data; + V.Length = 0; + V.Data = nullptr; + return *this; + } + + /// \brief Comparison operator. + bool operator==(const Vector &V) const { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + if (Length != V.Length) + return false; + return std::equal(Data, Data + Length, V.Data); + } + + /// \brief Return the length of the vector + unsigned getLength() const { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + return Length; + } + + /// \brief Element access. + PBQPNum& operator[](unsigned Index) { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + assert(Index < Length && "Vector element access out of bounds."); + return Data[Index]; + } + + /// \brief Const element access. + const PBQPNum& operator[](unsigned Index) const { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + assert(Index < Length && "Vector element access out of bounds."); + return Data[Index]; + } + + /// \brief Add another vector to this one. + Vector& operator+=(const Vector &V) { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + assert(Length == V.Length && "Vector length mismatch."); + std::transform(Data, Data + Length, V.Data, Data, std::plus<PBQPNum>()); + return *this; + } + + /// \brief Subtract another vector from this one. + Vector& operator-=(const Vector &V) { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + assert(Length == V.Length && "Vector length mismatch."); + std::transform(Data, Data + Length, V.Data, Data, std::minus<PBQPNum>()); + return *this; + } + + /// \brief Returns the index of the minimum value in this vector + unsigned minIndex() const { + assert(Length != 0 && Data != nullptr && "Invalid vector"); + return std::min_element(Data, Data + Length) - Data; + } + +private: + unsigned Length; + PBQPNum *Data; +}; + +class VectorComparator { +public: + bool operator()(const Vector &A, const Vector &B) { + if (A.Length < B.Length) + return true; + if (B.Length < A.Length) + return false; + char *AData = reinterpret_cast<char*>(A.Data); + char *BData = reinterpret_cast<char*>(B.Data); + return std::lexicographical_compare(AData, + AData + A.Length * sizeof(PBQPNum), + BData, + BData + A.Length * sizeof(PBQPNum)); + } }; /// \brief Output a textual representation of the given vector on the given /// output stream. template <typename OStream> -OStream& operator<<(OStream &os, const Vector &v) { - assert((v.getLength() != 0) && "Zero-length vector badness."); +OStream& operator<<(OStream &OS, const Vector &V) { + assert((V.getLength() != 0) && "Zero-length vector badness."); - os << "[ " << v[0]; - for (unsigned i = 1; i < v.getLength(); ++i) { - os << ", " << v[i]; - } - os << " ]"; + OS << "[ " << V[0]; + for (unsigned i = 1; i < V.getLength(); ++i) + OS << ", " << V[i]; + OS << " ]"; - return os; -} + return OS; +} /// \brief PBQP Matrix class class Matrix { - public: - - /// \brief Construct a PBQP Matrix with the given dimensions. - Matrix(unsigned rows, unsigned cols) : - rows(rows), cols(cols), data(new PBQPNum[rows * cols]) { - } - - /// \brief Construct a PBQP Matrix with the given dimensions and initial - /// value. - Matrix(unsigned rows, unsigned cols, PBQPNum initVal) : - rows(rows), cols(cols), data(new PBQPNum[rows * cols]) { - std::fill(data, data + (rows * cols), initVal); - } - - /// \brief Copy construct a PBQP matrix. - Matrix(const Matrix &m) : - rows(m.rows), cols(m.cols), data(new PBQPNum[rows * cols]) { - std::copy(m.data, m.data + (rows * cols), data); - } - - /// \brief Destroy this matrix, return its memory. - ~Matrix() { delete[] data; } - - /// \brief Assignment operator. - Matrix& operator=(const Matrix &m) { - delete[] data; - rows = m.rows; cols = m.cols; - data = new PBQPNum[rows * cols]; - std::copy(m.data, m.data + (rows * cols), data); - return *this; - } - - /// \brief Return the number of rows in this matrix. - unsigned getRows() const { return rows; } - - /// \brief Return the number of cols in this matrix. - unsigned getCols() const { return cols; } - - /// \brief Matrix element access. - PBQPNum* operator[](unsigned r) { - assert(r < rows && "Row out of bounds."); - return data + (r * cols); - } - - /// \brief Matrix element access. - const PBQPNum* operator[](unsigned r) const { - assert(r < rows && "Row out of bounds."); - return data + (r * cols); - } - - /// \brief Returns the given row as a vector. - Vector getRowAsVector(unsigned r) const { - Vector v(cols); - for (unsigned c = 0; c < cols; ++c) - v[c] = (*this)[r][c]; - return v; - } - - /// \brief Returns the given column as a vector. - Vector getColAsVector(unsigned c) const { - Vector v(rows); - for (unsigned r = 0; r < rows; ++r) - v[r] = (*this)[r][c]; - return v; - } - - /// \brief Reset the matrix to the given value. - Matrix& reset(PBQPNum val = 0) { - std::fill(data, data + (rows * cols), val); - return *this; - } - - /// \brief Set a single row of this matrix to the given value. - Matrix& setRow(unsigned r, PBQPNum val) { - assert(r < rows && "Row out of bounds."); - std::fill(data + (r * cols), data + ((r + 1) * cols), val); - return *this; - } - - /// \brief Set a single column of this matrix to the given value. - Matrix& setCol(unsigned c, PBQPNum val) { - assert(c < cols && "Column out of bounds."); - for (unsigned r = 0; r < rows; ++r) - (*this)[r][c] = val; - return *this; - } - - /// \brief Matrix transpose. - Matrix transpose() const { - Matrix m(cols, rows); - for (unsigned r = 0; r < rows; ++r) - for (unsigned c = 0; c < cols; ++c) - m[c][r] = (*this)[r][c]; - return m; - } - - /// \brief Returns the diagonal of the matrix as a vector. - /// - /// Matrix must be square. - Vector diagonalize() const { - assert(rows == cols && "Attempt to diagonalize non-square matrix."); - - Vector v(rows); - for (unsigned r = 0; r < rows; ++r) - v[r] = (*this)[r][r]; - return v; - } - - /// \brief Add the given matrix to this one. - Matrix& operator+=(const Matrix &m) { - assert(rows == m.rows && cols == m.cols && - "Matrix dimensions mismatch."); - std::transform(data, data + (rows * cols), m.data, data, - std::plus<PBQPNum>()); - return *this; - } - - /// \brief Returns the minimum of the given row - PBQPNum getRowMin(unsigned r) const { - assert(r < rows && "Row out of bounds"); - return *std::min_element(data + (r * cols), data + ((r + 1) * cols)); - } - - /// \brief Returns the minimum of the given column - PBQPNum getColMin(unsigned c) const { - PBQPNum minElem = (*this)[0][c]; - for (unsigned r = 1; r < rows; ++r) - if ((*this)[r][c] < minElem) minElem = (*this)[r][c]; - return minElem; - } - - /// \brief Subtracts the given scalar from the elements of the given row. - Matrix& subFromRow(unsigned r, PBQPNum val) { - assert(r < rows && "Row out of bounds"); - std::transform(data + (r * cols), data + ((r + 1) * cols), - data + (r * cols), - std::bind2nd(std::minus<PBQPNum>(), val)); - return *this; - } - - /// \brief Subtracts the given scalar from the elements of the given column. - Matrix& subFromCol(unsigned c, PBQPNum val) { - for (unsigned r = 0; r < rows; ++r) - (*this)[r][c] -= val; - return *this; - } - - /// \brief Returns true if this is a zero matrix. - bool isZero() const { - return find_if(data, data + (rows * cols), - std::bind2nd(std::not_equal_to<PBQPNum>(), 0)) == - data + (rows * cols); - } - - private: - unsigned rows, cols; - PBQPNum *data; +private: + friend class MatrixComparator; +public: + + /// \brief Construct a PBQP Matrix with the given dimensions. + Matrix(unsigned Rows, unsigned Cols) : + Rows(Rows), Cols(Cols), Data(new PBQPNum[Rows * Cols]) { + } + + /// \brief Construct a PBQP Matrix with the given dimensions and initial + /// value. + Matrix(unsigned Rows, unsigned Cols, PBQPNum InitVal) + : Rows(Rows), Cols(Cols), Data(new PBQPNum[Rows * Cols]) { + std::fill(Data, Data + (Rows * Cols), InitVal); + } + + /// \brief Copy construct a PBQP matrix. + Matrix(const Matrix &M) + : Rows(M.Rows), Cols(M.Cols), Data(new PBQPNum[Rows * Cols]) { + std::copy(M.Data, M.Data + (Rows * Cols), Data); + } + + /// \brief Move construct a PBQP matrix. + Matrix(Matrix &&M) + : Rows(M.Rows), Cols(M.Cols), Data(M.Data) { + M.Rows = M.Cols = 0; + M.Data = nullptr; + } + + /// \brief Destroy this matrix, return its memory. + ~Matrix() { delete[] Data; } + + /// \brief Copy-assignment operator. + Matrix& operator=(const Matrix &M) { + delete[] Data; + Rows = M.Rows; Cols = M.Cols; + Data = new PBQPNum[Rows * Cols]; + std::copy(M.Data, M.Data + (Rows * Cols), Data); + return *this; + } + + /// \brief Move-assignment operator. + Matrix& operator=(Matrix &&M) { + delete[] Data; + Rows = M.Rows; + Cols = M.Cols; + Data = M.Data; + M.Rows = M.Cols = 0; + M.Data = nullptr; + return *this; + } + + /// \brief Comparison operator. + bool operator==(const Matrix &M) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + if (Rows != M.Rows || Cols != M.Cols) + return false; + return std::equal(Data, Data + (Rows * Cols), M.Data); + } + + /// \brief Return the number of rows in this matrix. + unsigned getRows() const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + return Rows; + } + + /// \brief Return the number of cols in this matrix. + unsigned getCols() const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + return Cols; + } + + /// \brief Matrix element access. + PBQPNum* operator[](unsigned R) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(R < Rows && "Row out of bounds."); + return Data + (R * Cols); + } + + /// \brief Matrix element access. + const PBQPNum* operator[](unsigned R) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(R < Rows && "Row out of bounds."); + return Data + (R * Cols); + } + + /// \brief Returns the given row as a vector. + Vector getRowAsVector(unsigned R) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + Vector V(Cols); + for (unsigned C = 0; C < Cols; ++C) + V[C] = (*this)[R][C]; + return V; + } + + /// \brief Returns the given column as a vector. + Vector getColAsVector(unsigned C) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + Vector V(Rows); + for (unsigned R = 0; R < Rows; ++R) + V[R] = (*this)[R][C]; + return V; + } + + /// \brief Reset the matrix to the given value. + Matrix& reset(PBQPNum Val = 0) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + std::fill(Data, Data + (Rows * Cols), Val); + return *this; + } + + /// \brief Set a single row of this matrix to the given value. + Matrix& setRow(unsigned R, PBQPNum Val) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(R < Rows && "Row out of bounds."); + std::fill(Data + (R * Cols), Data + ((R + 1) * Cols), Val); + return *this; + } + + /// \brief Set a single column of this matrix to the given value. + Matrix& setCol(unsigned C, PBQPNum Val) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(C < Cols && "Column out of bounds."); + for (unsigned R = 0; R < Rows; ++R) + (*this)[R][C] = Val; + return *this; + } + + /// \brief Matrix transpose. + Matrix transpose() const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + Matrix M(Cols, Rows); + for (unsigned r = 0; r < Rows; ++r) + for (unsigned c = 0; c < Cols; ++c) + M[c][r] = (*this)[r][c]; + return M; + } + + /// \brief Returns the diagonal of the matrix as a vector. + /// + /// Matrix must be square. + Vector diagonalize() const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(Rows == Cols && "Attempt to diagonalize non-square matrix."); + Vector V(Rows); + for (unsigned r = 0; r < Rows; ++r) + V[r] = (*this)[r][r]; + return V; + } + + /// \brief Add the given matrix to this one. + Matrix& operator+=(const Matrix &M) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(Rows == M.Rows && Cols == M.Cols && + "Matrix dimensions mismatch."); + std::transform(Data, Data + (Rows * Cols), M.Data, Data, + std::plus<PBQPNum>()); + return *this; + } + + Matrix operator+(const Matrix &M) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + Matrix Tmp(*this); + Tmp += M; + return Tmp; + } + + /// \brief Returns the minimum of the given row + PBQPNum getRowMin(unsigned R) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(R < Rows && "Row out of bounds"); + return *std::min_element(Data + (R * Cols), Data + ((R + 1) * Cols)); + } + + /// \brief Returns the minimum of the given column + PBQPNum getColMin(unsigned C) const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + PBQPNum MinElem = (*this)[0][C]; + for (unsigned R = 1; R < Rows; ++R) + if ((*this)[R][C] < MinElem) + MinElem = (*this)[R][C]; + return MinElem; + } + + /// \brief Subtracts the given scalar from the elements of the given row. + Matrix& subFromRow(unsigned R, PBQPNum Val) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + assert(R < Rows && "Row out of bounds"); + std::transform(Data + (R * Cols), Data + ((R + 1) * Cols), + Data + (R * Cols), + std::bind2nd(std::minus<PBQPNum>(), Val)); + return *this; + } + + /// \brief Subtracts the given scalar from the elements of the given column. + Matrix& subFromCol(unsigned C, PBQPNum Val) { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + for (unsigned R = 0; R < Rows; ++R) + (*this)[R][C] -= Val; + return *this; + } + + /// \brief Returns true if this is a zero matrix. + bool isZero() const { + assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix"); + return find_if(Data, Data + (Rows * Cols), + std::bind2nd(std::not_equal_to<PBQPNum>(), 0)) == + Data + (Rows * Cols); + } + +private: + unsigned Rows, Cols; + PBQPNum *Data; +}; + +class MatrixComparator { +public: + bool operator()(const Matrix &A, const Matrix &B) { + if (A.Rows < B.Rows) + return true; + if (B.Rows < A.Rows) + return false; + if (A.Cols < B.Cols) + return true; + if (B.Cols < A.Cols) + return false; + char *AData = reinterpret_cast<char*>(A.Data); + char *BData = reinterpret_cast<char*>(B.Data); + return std::lexicographical_compare( + AData, AData + (A.Rows * A.Cols * sizeof(PBQPNum)), + BData, BData + (A.Rows * A.Cols * sizeof(PBQPNum))); + } }; /// \brief Output a textual representation of the given matrix on the given /// output stream. template <typename OStream> -OStream& operator<<(OStream &os, const Matrix &m) { - - assert((m.getRows() != 0) && "Zero-row matrix badness."); +OStream& operator<<(OStream &OS, const Matrix &M) { + assert((M.getRows() != 0) && "Zero-row matrix badness."); + for (unsigned i = 0; i < M.getRows(); ++i) + OS << M.getRowAsVector(i); + return OS; +} - for (unsigned i = 0; i < m.getRows(); ++i) { - os << m.getRowAsVector(i); - } +template <typename Metadata> +class MDVector : public Vector { +public: + MDVector(const Vector &v) : Vector(v), md(*this) { } + MDVector(Vector &&v) : Vector(std::move(v)), md(*this) { } + const Metadata& getMetadata() const { return md; } +private: + Metadata md; +}; - return os; -} +template <typename Metadata> +class MDMatrix : public Matrix { +public: + MDMatrix(const Matrix &m) : Matrix(m), md(*this) { } + MDMatrix(Matrix &&m) : Matrix(std::move(m)), md(*this) { } + const Metadata& getMetadata() const { return md; } +private: + Metadata md; +}; } diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h new file mode 100644 index 0000000000..a55a06033c --- /dev/null +++ b/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -0,0 +1,191 @@ +//===----------- ReductionRules.h - Reduction Rules -------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Reduction Rules. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REDUCTIONRULES_H +#define LLVM_REDUCTIONRULES_H + +#include "Graph.h" +#include "Math.h" +#include "Solution.h" + +namespace PBQP { + + /// \brief Reduce a node of degree one. + /// + /// Propagate costs from the given node, which must be of degree one, to its + /// neighbor. Notify the problem domain. + template <typename GraphT> + void applyR1(GraphT &G, typename GraphT::NodeId NId) { + typedef typename GraphT::NodeId NodeId; + typedef typename GraphT::EdgeId EdgeId; + typedef typename GraphT::Vector Vector; + typedef typename GraphT::Matrix Matrix; + typedef typename GraphT::RawVector RawVector; + + assert(G.getNodeDegree(NId) == 1 && + "R1 applied to node with degree != 1."); + + EdgeId EId = *G.adjEdgeIds(NId).begin(); + NodeId MId = G.getEdgeOtherNodeId(EId, NId); + + const Matrix &ECosts = G.getEdgeCosts(EId); + const Vector &XCosts = G.getNodeCosts(NId); + RawVector YCosts = G.getNodeCosts(MId); + + // Duplicate a little to avoid transposing matrices. + if (NId == G.getEdgeNode1Id(EId)) { + for (unsigned j = 0; j < YCosts.getLength(); ++j) { + PBQPNum Min = ECosts[0][j] + XCosts[0]; + for (unsigned i = 1; i < XCosts.getLength(); ++i) { + PBQPNum C = ECosts[i][j] + XCosts[i]; + if (C < Min) + Min = C; + } + YCosts[j] += Min; + } + } else { + for (unsigned i = 0; i < YCosts.getLength(); ++i) { + PBQPNum Min = ECosts[i][0] + XCosts[0]; + for (unsigned j = 1; j < XCosts.getLength(); ++j) { + PBQPNum C = ECosts[i][j] + XCosts[j]; + if (C < Min) + Min = C; + } + YCosts[i] += Min; + } + } + G.setNodeCosts(MId, YCosts); + G.disconnectEdge(EId, MId); + } + + template <typename GraphT> + void applyR2(GraphT &G, typename GraphT::NodeId NId) { + typedef typename GraphT::NodeId NodeId; + typedef typename GraphT::EdgeId EdgeId; + typedef typename GraphT::Vector Vector; + typedef typename GraphT::Matrix Matrix; + typedef typename GraphT::RawMatrix RawMatrix; + + assert(G.getNodeDegree(NId) == 2 && + "R2 applied to node with degree != 2."); + + const Vector &XCosts = G.getNodeCosts(NId); + + typename GraphT::AdjEdgeItr AEItr = G.adjEdgeIds(NId).begin(); + EdgeId YXEId = *AEItr, + ZXEId = *(++AEItr); + + NodeId YNId = G.getEdgeOtherNodeId(YXEId, NId), + ZNId = G.getEdgeOtherNodeId(ZXEId, NId); + + bool FlipEdge1 = (G.getEdgeNode1Id(YXEId) == NId), + FlipEdge2 = (G.getEdgeNode1Id(ZXEId) == NId); + + const Matrix *YXECosts = FlipEdge1 ? + new Matrix(G.getEdgeCosts(YXEId).transpose()) : + &G.getEdgeCosts(YXEId); + + const Matrix *ZXECosts = FlipEdge2 ? + new Matrix(G.getEdgeCosts(ZXEId).transpose()) : + &G.getEdgeCosts(ZXEId); + + unsigned XLen = XCosts.getLength(), + YLen = YXECosts->getRows(), + ZLen = ZXECosts->getRows(); + + RawMatrix Delta(YLen, ZLen); + + for (unsigned i = 0; i < YLen; ++i) { + for (unsigned j = 0; j < ZLen; ++j) { + PBQPNum Min = (*YXECosts)[i][0] + (*ZXECosts)[j][0] + XCosts[0]; + for (unsigned k = 1; k < XLen; ++k) { + PBQPNum C = (*YXECosts)[i][k] + (*ZXECosts)[j][k] + XCosts[k]; + if (C < Min) { + Min = C; + } + } + Delta[i][j] = Min; + } + } + + if (FlipEdge1) + delete YXECosts; + + if (FlipEdge2) + delete ZXECosts; + + EdgeId YZEId = G.findEdge(YNId, ZNId); + + if (YZEId == G.invalidEdgeId()) { + YZEId = G.addEdge(YNId, ZNId, Delta); + } else { + const Matrix &YZECosts = G.getEdgeCosts(YZEId); + if (YNId == G.getEdgeNode1Id(YZEId)) { + G.setEdgeCosts(YZEId, Delta + YZECosts); + } else { + G.setEdgeCosts(YZEId, Delta.transpose() + YZECosts); + } + } + + G.disconnectEdge(YXEId, YNId); + G.disconnectEdge(ZXEId, ZNId); + + // TODO: Try to normalize newly added/modified edge. + } + + + // \brief Find a solution to a fully reduced graph by backpropagation. + // + // Given a graph and a reduction order, pop each node from the reduction + // order and greedily compute a minimum solution based on the node costs, and + // the dependent costs due to previously solved nodes. + // + // Note - This does not return the graph to its original (pre-reduction) + // state: the existing solvers destructively alter the node and edge + // costs. Given that, the backpropagate function doesn't attempt to + // replace the edges either, but leaves the graph in its reduced + // state. + template <typename GraphT, typename StackT> + Solution backpropagate(GraphT& G, StackT stack) { + typedef GraphBase::NodeId NodeId; + typedef typename GraphT::Matrix Matrix; + typedef typename GraphT::RawVector RawVector; + + Solution s; + + while (!stack.empty()) { + NodeId NId = stack.back(); + stack.pop_back(); + + RawVector v = G.getNodeCosts(NId); + + for (auto EId : G.adjEdgeIds(NId)) { + const Matrix& edgeCosts = G.getEdgeCosts(EId); + if (NId == G.getEdgeNode1Id(EId)) { + NodeId mId = G.getEdgeNode2Id(EId); + v += edgeCosts.getColAsVector(s.getSelection(mId)); + } else { + NodeId mId = G.getEdgeNode1Id(EId); + v += edgeCosts.getRowAsVector(s.getSelection(mId)); + } + } + + s.setSelection(NId, v.minIndex()); + } + + return s; + } + +} + +#endif // LLVM_REDUCTIONRULES_H diff --git a/include/llvm/CodeGen/PBQP/RegAllocSolver.h b/include/llvm/CodeGen/PBQP/RegAllocSolver.h new file mode 100644 index 0000000000..79ff6b4ba9 --- /dev/null +++ b/include/llvm/CodeGen/PBQP/RegAllocSolver.h @@ -0,0 +1,359 @@ +//===-- RegAllocSolver.h - Heuristic PBQP Solver for reg alloc --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Heuristic PBQP solver for register allocation problems. This solver uses a +// graph reduction approach. Nodes of degree 0, 1 and 2 are eliminated with +// optimality-preserving rules (see ReductionRules.h). When no low-degree (<3) +// nodes are present, a heuristic derived from Brigg's graph coloring approach +// is used. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H +#define LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H + +#include "CostAllocator.h" +#include "Graph.h" +#include "ReductionRules.h" +#include "Solution.h" +#include "llvm/Support/ErrorHandling.h" +#include <limits> +#include <vector> + +namespace PBQP { + + namespace RegAlloc { + + /// \brief Metadata to speed allocatability test. + /// + /// Keeps track of the number of infinities in each row and column. + class MatrixMetadata { + private: + MatrixMetadata(const MatrixMetadata&); + void operator=(const MatrixMetadata&); + public: + MatrixMetadata(const PBQP::Matrix& M) + : WorstRow(0), WorstCol(0), + UnsafeRows(new bool[M.getRows() - 1]()), + UnsafeCols(new bool[M.getCols() - 1]()) { + + unsigned* ColCounts = new unsigned[M.getCols() - 1](); + + for (unsigned i = 1; i < M.getRows(); ++i) { + unsigned RowCount = 0; + for (unsigned j = 1; j < M.getCols(); ++j) { + if (M[i][j] == std::numeric_limits<PBQP::PBQPNum>::infinity()) { + ++RowCount; + ++ColCounts[j - 1]; + UnsafeRows[i - 1] = true; + UnsafeCols[j - 1] = true; + } + } + WorstRow = std::max(WorstRow, RowCount); + } + unsigned WorstColCountForCurRow = + *std::max_element(ColCounts, ColCounts + M.getCols() - 1); + WorstCol = std::max(WorstCol, WorstColCountForCurRow); + delete[] ColCounts; + } + + ~MatrixMetadata() { + delete[] UnsafeRows; + delete[] UnsafeCols; + } + + unsigned getWorstRow() const { return WorstRow; } + unsigned getWorstCol() const { return WorstCol; } + const bool* getUnsafeRows() const { return UnsafeRows; } + const bool* getUnsafeCols() const { return UnsafeCols; } + + private: + unsigned WorstRow, WorstCol; + bool* UnsafeRows; + bool* UnsafeCols; + }; + + class NodeMetadata { + public: + typedef enum { Unprocessed, + OptimallyReducible, + ConservativelyAllocatable, + NotProvablyAllocatable } ReductionState; + + NodeMetadata() : RS(Unprocessed), DeniedOpts(0), OptUnsafeEdges(0) {} + ~NodeMetadata() { delete[] OptUnsafeEdges; } + + void setup(const Vector& Costs) { + NumOpts = Costs.getLength() - 1; + OptUnsafeEdges = new unsigned[NumOpts](); + } + + ReductionState getReductionState() const { return RS; } + void setReductionState(ReductionState RS) { this->RS = RS; } + + void handleAddEdge(const MatrixMetadata& MD, bool Transpose) { + DeniedOpts += Transpose ? MD.getWorstCol() : MD.getWorstRow(); + const bool* UnsafeOpts = + Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); + for (unsigned i = 0; i < NumOpts; ++i) + OptUnsafeEdges[i] += UnsafeOpts[i]; + } + + void handleRemoveEdge(const MatrixMetadata& MD, bool Transpose) { + DeniedOpts -= Transpose ? MD.getWorstCol() : MD.getWorstRow(); + const bool* UnsafeOpts = + Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows(); + for (unsigned i = 0; i < NumOpts; ++i) + OptUnsafeEdges[i] -= UnsafeOpts[i]; + } + + bool isConservativelyAllocatable() const { + return (DeniedOpts < NumOpts) || + (std::find(OptUnsafeEdges, OptUnsafeEdges + NumOpts, 0) != + OptUnsafeEdges + NumOpts); + } + + private: + ReductionState RS; + unsigned NumOpts; + unsigned DeniedOpts; + unsigned* OptUnsafeEdges; + }; + + class RegAllocSolverImpl { + private: + typedef PBQP::MDMatrix<MatrixMetadata> RAMatrix; + public: + typedef PBQP::Vector RawVector; + typedef PBQP::Matrix RawMatrix; + typedef PBQP::Vector Vector; + typedef RAMatrix Matrix; + typedef PBQP::PoolCostAllocator< + Vector, PBQP::VectorComparator, + Matrix, PBQP::MatrixComparator> CostAllocator; + + typedef PBQP::GraphBase::NodeId NodeId; + typedef PBQP::GraphBase::EdgeId EdgeId; + + typedef RegAlloc::NodeMetadata NodeMetadata; + + struct EdgeMetadata { }; + + typedef PBQP::Graph<RegAllocSolverImpl> Graph; + + RegAllocSolverImpl(Graph &G) : G(G) {} + + Solution solve() { + G.setSolver(*this); + Solution S; + setup(); + S = backpropagate(G, reduce()); + G.unsetSolver(); + return S; + } + + void handleAddNode(NodeId NId) { + G.getNodeMetadata(NId).setup(G.getNodeCosts(NId)); + } + void handleRemoveNode(NodeId NId) {} + void handleSetNodeCosts(NodeId NId, const Vector& newCosts) {} + + void handleAddEdge(EdgeId EId) { + handleReconnectEdge(EId, G.getEdgeNode1Id(EId)); + handleReconnectEdge(EId, G.getEdgeNode2Id(EId)); + } + + void handleRemoveEdge(EdgeId EId) { + handleDisconnectEdge(EId, G.getEdgeNode1Id(EId)); + handleDisconnectEdge(EId, G.getEdgeNode2Id(EId)); + } + + void handleDisconnectEdge(EdgeId EId, NodeId NId) { + NodeMetadata& NMd = G.getNodeMetadata(NId); + const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); + NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId)); + if (G.getNodeDegree(NId) == 3) { + // This node is becoming optimally reducible. + moveToOptimallyReducibleNodes(NId); + } else if (NMd.getReductionState() == + NodeMetadata::NotProvablyAllocatable && + NMd.isConservativelyAllocatable()) { + // This node just became conservatively allocatable. + moveToConservativelyAllocatableNodes(NId); + } + } + + void handleReconnectEdge(EdgeId EId, NodeId NId) { + NodeMetadata& NMd = G.getNodeMetadata(NId); + const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata(); + NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId)); + } + + void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) { + handleRemoveEdge(EId); + + NodeId N1Id = G.getEdgeNode1Id(EId); + NodeId N2Id = G.getEdgeNode2Id(EId); + NodeMetadata& N1Md = G.getNodeMetadata(N1Id); + NodeMetadata& N2Md = G.getNodeMetadata(N2Id); + const MatrixMetadata& MMd = NewCosts.getMetadata(); + N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId)); + N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId)); + } + + private: + + void removeFromCurrentSet(NodeId NId) { + switch (G.getNodeMetadata(NId).getReductionState()) { + case NodeMetadata::Unprocessed: break; + case NodeMetadata::OptimallyReducible: + assert(OptimallyReducibleNodes.find(NId) != + OptimallyReducibleNodes.end() && + "Node not in optimally reducible set."); + OptimallyReducibleNodes.erase(NId); + break; + case NodeMetadata::ConservativelyAllocatable: + assert(ConservativelyAllocatableNodes.find(NId) != + ConservativelyAllocatableNodes.end() && + "Node not in conservatively allocatable set."); + ConservativelyAllocatableNodes.erase(NId); + break; + case NodeMetadata::NotProvablyAllocatable: + assert(NotProvablyAllocatableNodes.find(NId) != + NotProvablyAllocatableNodes.end() && + "Node not in not-provably-allocatable set."); + NotProvablyAllocatableNodes.erase(NId); + break; + } + } + + void moveToOptimallyReducibleNodes(NodeId NId) { + removeFromCurrentSet(NId); + OptimallyReducibleNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::OptimallyReducible); + } + + void moveToConservativelyAllocatableNodes(NodeId NId) { + removeFromCurrentSet(NId); + ConservativelyAllocatableNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::ConservativelyAllocatable); + } + + void moveToNotProvablyAllocatableNodes(NodeId NId) { + removeFromCurrentSet(NId); + NotProvablyAllocatableNodes.insert(NId); + G.getNodeMetadata(NId).setReductionState( + NodeMetadata::NotProvablyAllocatable); + } + + void setup() { + // Set up worklists. + for (auto NId : G.nodeIds()) { + if (G.getNodeDegree(NId) < 3) + moveToOptimallyReducibleNodes(NId); + else if (G.getNodeMetadata(NId).isConservativelyAllocatable()) + moveToConservativelyAllocatableNodes(NId); + else + moveToNotProvablyAllocatableNodes(NId); + } + } + + // Compute a reduction order for the graph by iteratively applying PBQP + // reduction rules. Locally optimal rules are applied whenever possible (R0, + // R1, R2). If no locally-optimal rules apply then any conservatively + // allocatable node is reduced. Finally, if no conservatively allocatable + // node exists then the node with the lowest spill-cost:degree ratio is + // selected. + std::vector<GraphBase::NodeId> reduce() { + assert(!G.empty() && "Cannot reduce empty graph."); + + typedef GraphBase::NodeId NodeId; + std::vector<NodeId> NodeStack; + + // Consume worklists. + while (true) { + if (!OptimallyReducibleNodes.empty()) { + NodeSet::iterator NItr = OptimallyReducibleNodes.begin(); + NodeId NId = *NItr; + OptimallyReducibleNodes.erase(NItr); + NodeStack.push_back(NId); + switch (G.getNodeDegree(NId)) { + case 0: + break; + case 1: + applyR1(G, NId); + break; + case 2: + applyR2(G, NId); + break; + default: llvm_unreachable("Not an optimally reducible node."); + } + } else if (!ConservativelyAllocatableNodes.empty()) { + // Conservatively allocatable nodes will never spill. For now just + // take the first node in the set and push it on the stack. When we + // start optimizing more heavily for register preferencing, it may + // would be better to push nodes with lower 'expected' or worst-case + // register costs first (since early nodes are the most + // constrained). + NodeSet::iterator NItr = ConservativelyAllocatableNodes.begin(); + NodeId NId = *NItr; + ConservativelyAllocatableNodes.erase(NItr); + NodeStack.push_back(NId); + G.disconnectAllNeighborsFromNode(NId); + + } else if (!NotProvablyAllocatableNodes.empty()) { + NodeSet::iterator NItr = + std::min_element(NotProvablyAllocatableNodes.begin(), + NotProvablyAllocatableNodes.end(), + SpillCostComparator(G)); + NodeId NId = *NItr; + NotProvablyAllocatableNodes.erase(NItr); + NodeStack.push_back(NId); + G.disconnectAllNeighborsFromNode(NId); + } else + break; + } + + return NodeStack; + } + + class SpillCostComparator { + public: + SpillCostComparator(const Graph& G) : G(G) {} + bool operator()(NodeId N1Id, NodeId N2Id) { + PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id); + PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id); + return N1SC < N2SC; + } + private: + const Graph& G; + }; + + Graph& G; + typedef std::set<NodeId> NodeSet; + NodeSet OptimallyReducibleNodes; + NodeSet ConservativelyAllocatableNodes; + NodeSet NotProvablyAllocatableNodes; + }; + + typedef Graph<RegAllocSolverImpl> Graph; + + Solution solve(Graph& G) { + if (G.empty()) + return Solution(); + RegAllocSolverImpl RegAllocSolver(G); + return RegAllocSolver.solve(); + } + + } +} + +#endif // LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H diff --git a/include/llvm/CodeGen/PBQP/Solution.h b/include/llvm/CodeGen/PBQP/Solution.h index 091805d00f..3556e60f39 100644 --- a/include/llvm/CodeGen/PBQP/Solution.h +++ b/include/llvm/CodeGen/PBQP/Solution.h @@ -26,7 +26,7 @@ namespace PBQP { class Solution { private: - typedef std::map<Graph::NodeId, unsigned> SelectionsMap; + typedef std::map<GraphBase::NodeId, unsigned> SelectionsMap; SelectionsMap selections; unsigned r0Reductions, r1Reductions, r2Reductions, rNReductions; @@ -72,14 +72,14 @@ namespace PBQP { /// \brief Set the selection for a given node. /// @param nodeId Node id. /// @param selection Selection for nodeId. - void setSelection(Graph::NodeId nodeId, unsigned selection) { + void setSelection(GraphBase::NodeId nodeId, unsigned selection) { selections[nodeId] = selection; } /// \brief Get a node's selection. /// @param nodeId Node id. /// @return The selection for nodeId; - unsigned getSelection(Graph::NodeId nodeId) const { + unsigned getSelection(GraphBase::NodeId nodeId) const { SelectionsMap::const_iterator sItr = selections.find(nodeId); assert(sItr != selections.end() && "No selection for node."); return sItr->second; diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index ae4a2fa0bf..5d68f8637b 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -207,9 +207,9 @@ public: /// Fully developed targets will not generally override this. virtual void addMachinePasses(); - /// createTargetScheduler - Create an instance of ScheduleDAGInstrs to be run - /// within the standard MachineScheduler pass for this function and target at - /// the current optimization level. + /// Create an instance of ScheduleDAGInstrs to be run within the standard + /// MachineScheduler pass for this function and target at the current + /// optimization level. /// /// This can also be used to plug a new MachineSchedStrategy into an instance /// of the standard ScheduleDAGMI: @@ -221,6 +221,13 @@ public: return 0; } + /// Similar to createMachineScheduler but used when postRA machine scheduling + /// is enabled. + virtual ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const { + return 0; + } + protected: // Helper to verify the analysis is really immutable. void setOpt(bool &Opt, bool Val); @@ -363,6 +370,10 @@ namespace llvm { createMachineFunctionPrinterPass(raw_ostream &OS, const std::string &Banner =""); + /// createCodeGenPreparePass - Transform the code to expose more pattern + /// matching during instruction selection. + FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = 0); + /// MachineLoopInfo - This pass is a loop analysis pass. extern char &MachineLoopInfoID; @@ -403,6 +414,9 @@ namespace llvm { /// MachineScheduler - This pass schedules machine instructions. extern char &MachineSchedulerID; + /// PostMachineScheduler - This pass schedules machine instructions postRA. + extern char &PostMachineSchedulerID; + /// SpillPlacement analysis. Suggest optimal placement of spill code between /// basic blocks. extern char &SpillPlacementID; @@ -568,6 +582,11 @@ namespace llvm { /// bundles (created earlier, e.g. during pre-RA scheduling). extern char &FinalizeMachineBundlesID; + /// StackMapLiveness - This pass analyses the register live-out set of + /// stackmap/patchpoint intrinsics and attaches the calculated information to + /// the intrinsic for later emission to the StackMap. + extern char &StackMapLivenessID; + } // End llvm namespace #endif diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h index 705086c22b..0af8915551 100644 --- a/include/llvm/CodeGen/PseudoSourceValue.h +++ b/include/llvm/CodeGen/PseudoSourceValue.h @@ -29,7 +29,7 @@ namespace llvm { /// printCustom - Implement printing for PseudoSourceValue. This is called /// from Value::print or Value's operator<<. /// - virtual void printCustom(raw_ostream &O) const; + void printCustom(raw_ostream &O) const override; public: explicit PseudoSourceValue(enum ValueTy Subclass = PseudoSourceValueVal); @@ -93,13 +93,13 @@ namespace llvm { return V->getValueID() == FixedStackPseudoSourceValueVal; } - virtual bool isConstant(const MachineFrameInfo *MFI) const; + bool isConstant(const MachineFrameInfo *MFI) const override; - virtual bool isAliased(const MachineFrameInfo *MFI) const; + bool isAliased(const MachineFrameInfo *MFI) const override; - virtual bool mayAlias(const MachineFrameInfo *) const; + bool mayAlias(const MachineFrameInfo *) const override; - virtual void printCustom(raw_ostream &OS) const; + void printCustom(raw_ostream &OS) const override; int getFrameIndex() const { return FI; } }; diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h index 7472e5a62d..efd7c6188c 100644 --- a/include/llvm/CodeGen/RegAllocPBQP.h +++ b/include/llvm/CodeGen/RegAllocPBQP.h @@ -17,9 +17,9 @@ #define LLVM_CODEGEN_REGALLOCPBQP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/PBQP/Graph.h" -#include "llvm/CodeGen/PBQP/Solution.h" +#include "llvm/CodeGen/PBQP/RegAllocSolver.h" #include <map> #include <set> @@ -29,30 +29,30 @@ namespace llvm { class MachineBlockFrequencyInfo; class MachineFunction; class TargetRegisterInfo; - template<class T> class OwningPtr; + + typedef PBQP::RegAlloc::Graph PBQPRAGraph; /// This class wraps up a PBQP instance representing a register allocation /// problem, plus the structures necessary to map back from the PBQP solution /// to a register allocation solution. (i.e. The PBQP-node <--> vreg map, /// and the PBQP option <--> storage location map). - class PBQPRAProblem { public: typedef SmallVector<unsigned, 16> AllowedSet; - PBQP::Graph& getGraph() { return graph; } + PBQPRAGraph& getGraph() { return graph; } - const PBQP::Graph& getGraph() const { return graph; } + const PBQPRAGraph& getGraph() const { return graph; } /// Record the mapping between the given virtual register and PBQP node, /// and the set of allowed pregs for the vreg. /// /// If you are extending /// PBQPBuilder you are unlikely to need this: Nodes and options for all - /// vregs will already have been set up for you by the base class. + /// vregs will already have been set up for you by the base class. template <typename AllowedRegsItr> - void recordVReg(unsigned vreg, PBQP::Graph::NodeId nodeId, + void recordVReg(unsigned vreg, PBQPRAGraph::NodeId nodeId, AllowedRegsItr arBegin, AllowedRegsItr arEnd) { assert(node2VReg.find(nodeId) == node2VReg.end() && "Re-mapping node."); assert(vreg2Node.find(vreg) == vreg2Node.end() && "Re-mapping vreg."); @@ -64,10 +64,10 @@ namespace llvm { } /// Get the virtual register corresponding to the given PBQP node. - unsigned getVRegForNode(PBQP::Graph::NodeId nodeId) const; + unsigned getVRegForNode(PBQPRAGraph::NodeId nodeId) const; /// Get the PBQP node corresponding to the given virtual register. - PBQP::Graph::NodeId getNodeForVReg(unsigned vreg) const; + PBQPRAGraph::NodeId getNodeForVReg(unsigned vreg) const; /// Returns true if the given PBQP option represents a physical register, /// false otherwise. @@ -92,16 +92,16 @@ namespace llvm { private: - typedef std::map<PBQP::Graph::NodeId, unsigned> Node2VReg; - typedef DenseMap<unsigned, PBQP::Graph::NodeId> VReg2Node; + typedef std::map<PBQPRAGraph::NodeId, unsigned> Node2VReg; + typedef DenseMap<unsigned, PBQPRAGraph::NodeId> VReg2Node; typedef DenseMap<unsigned, AllowedSet> AllowedSetMap; - PBQP::Graph graph; + PBQPRAGraph graph; Node2VReg node2VReg; VReg2Node vreg2Node; AllowedSetMap allowedSets; - + }; /// Builds PBQP instances to represent register allocation problems. Includes @@ -114,7 +114,7 @@ namespace llvm { public: typedef std::set<unsigned> RegSet; - + /// Default constructor. PBQPBuilder() {} @@ -139,12 +139,12 @@ namespace llvm { /// Extended builder which adds coalescing constraints to a problem. class PBQPBuilderWithCoalescing : public PBQPBuilder { public: - + /// Build a PBQP instance to represent the register allocation problem for /// the given MachineFunction. - virtual PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis, - const MachineBlockFrequencyInfo *mbfi, - const RegSet &vregs); + PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis, + const MachineBlockFrequencyInfo *mbfi, + const RegSet &vregs) override; private: @@ -157,8 +157,9 @@ namespace llvm { PBQP::PBQPNum benefit); }; - FunctionPass* createPBQPRegisterAllocator(OwningPtr<PBQPBuilder> &builder, - char *customPassID=0); + FunctionPass * + createPBQPRegisterAllocator(std::unique_ptr<PBQPBuilder> &builder, + char *customPassID = 0); } #endif /* LLVM_CODEGEN_REGALLOCPBQP_H */ diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h index f20a9fce2a..7ae9111028 100644 --- a/include/llvm/CodeGen/ResourcePriorityQueue.h +++ b/include/llvm/CodeGen/ResourcePriorityQueue.h @@ -81,17 +81,17 @@ namespace llvm { delete ResourcesModel; } - bool isBottomUp() const { return false; } + bool isBottomUp() const override { return false; } - void initNodes(std::vector<SUnit> &sunits); + void initNodes(std::vector<SUnit> &sunits) override; - void addNode(const SUnit *SU) { + void addNode(const SUnit *SU) override { NumNodesSolelyBlocking.resize(SUnits->size(), 0); } - void updateNode(const SUnit *SU) {} + void updateNode(const SUnit *SU) override {} - void releaseState() { + void releaseState() override { SUnits = 0; } @@ -116,18 +116,18 @@ namespace llvm { signed regPressureDelta(SUnit *SU, bool RawPressure = false); signed rawRegPressureDelta (SUnit *SU, unsigned RCId); - bool empty() const { return Queue.empty(); } + bool empty() const override { return Queue.empty(); } - virtual void push(SUnit *U); + void push(SUnit *U) override; - virtual SUnit *pop(); + SUnit *pop() override; - virtual void remove(SUnit *SU); + void remove(SUnit *SU) override; - virtual void dump(ScheduleDAG* DAG) const; + void dump(ScheduleDAG* DAG) const override; /// scheduledNode - Main resource tracking point. - void scheduledNode(SUnit *Node); + void scheduledNode(SUnit *Node) override; bool isResourceAvailable(SUnit *SU); void reserveResources(SUnit *SU); diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index ccba1b0364..4886e5c506 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -184,6 +184,12 @@ namespace llvm { || Contents.OrdKind == MustAliasMem); } + /// isBarrier - Test if this is an Order dependence that is marked + /// as a barrier. + bool isBarrier() const { + return getKind() == Order && Contents.OrdKind == Barrier; + } + /// isMustAlias - Test if this is an Order dependence that is marked /// as "must alias", meaning that the SUnits at either end of the edge /// have a memory dependence on a known memory location. @@ -248,7 +254,7 @@ namespace llvm { /// SUnit - Scheduling unit. This is a node in the scheduling DAG. class SUnit { private: - enum LLVM_ENUM_INT_TYPE(unsigned) { BoundaryID = ~0u }; + enum : unsigned { BoundaryID = ~0u }; SDNode *Node; // Representative node. MachineInstr *Instr; // Alternatively, a MachineInstr. @@ -292,6 +298,8 @@ namespace llvm { bool isScheduleHigh : 1; // True if preferable to schedule high. bool isScheduleLow : 1; // True if preferable to schedule low. bool isCloned : 1; // True if this node has been cloned. + bool isUnbuffered : 1; // Uses an unbuffered resource. + bool hasReservedResource : 1; // Uses a reserved resource. Sched::Preference SchedulingPref; // Scheduling preference. private: @@ -316,7 +324,8 @@ namespace llvm { isTwoAddress(false), isCommutable(false), hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false), isAvailable(false), isScheduled(false), isScheduleHigh(false), - isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None), + isScheduleLow(false), isCloned(false), isUnbuffered(false), + hasReservedResource(false), SchedulingPref(Sched::None), isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {} @@ -330,7 +339,8 @@ namespace llvm { isTwoAddress(false), isCommutable(false), hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false), isAvailable(false), isScheduled(false), isScheduleHigh(false), - isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None), + isScheduleLow(false), isCloned(false), isUnbuffered(false), + hasReservedResource(false), SchedulingPref(Sched::None), isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {} @@ -343,7 +353,8 @@ namespace llvm { isTwoAddress(false), isCommutable(false), hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false), isAvailable(false), isScheduled(false), isScheduleHigh(false), - isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None), + isScheduleLow(false), isCloned(false), isUnbuffered(false), + hasReservedResource(false), SchedulingPref(Sched::None), isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {} diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index fe4f3c2de3..72bbe8b718 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -15,8 +15,8 @@ #ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H #define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H -#include "llvm/ADT/SparseSet.h" #include "llvm/ADT/SparseMultiSet.h" +#include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/Support/Compiler.h" @@ -43,7 +43,7 @@ namespace llvm { }; /// Record a physical register access. - /// For non data-dependent uses, OpIdx == -1. + /// For non-data-dependent uses, OpIdx == -1. struct PhysRegSUOper { SUnit *SU; int OpIdx; @@ -88,9 +88,13 @@ namespace llvm { /// isPostRA flag indicates vregs cannot be present. bool IsPostRA; + /// True if the DAG builder should remove kill flags (in preparation for + /// rescheduling). + bool RemoveKillFlags; + /// The standard DAG builder does not normally include terminators as DAG /// nodes because it does not create the necessary dependencies to prevent - /// reordering. A specialized scheduler can overide + /// reordering. A specialized scheduler can override /// TargetInstrInfo::isSchedulingBoundary then enable this flag to indicate /// it has taken responsibility for scheduling the terminator correctly. bool CanHandleTerminators; @@ -145,15 +149,21 @@ namespace llvm { DbgValueVector DbgValues; MachineInstr *FirstDbgValue; + /// Set of live physical registers for updating kill flags. + BitVector LiveRegs; + public: explicit ScheduleDAGInstrs(MachineFunction &mf, const MachineLoopInfo &mli, const MachineDominatorTree &mdt, bool IsPostRAFlag, + bool RemoveKillFlags = false, LiveIntervals *LIS = 0); virtual ~ScheduleDAGInstrs() {} + bool isPostRA() const { return IsPostRA; } + /// \brief Expose LiveIntervals for use in DAG mutators and such. LiveIntervals *getLIS() const { return LIS; } @@ -219,20 +229,31 @@ namespace llvm { /// the level of the whole MachineFunction. By default does nothing. virtual void finalizeSchedule() {} - virtual void dumpNode(const SUnit *SU) const; + void dumpNode(const SUnit *SU) const override; /// Return a label for a DAG node that points to an instruction. - virtual std::string getGraphNodeLabel(const SUnit *SU) const; + std::string getGraphNodeLabel(const SUnit *SU) const override; /// Return a label for the region of code covered by the DAG. - virtual std::string getDAGName() const; + std::string getDAGName() const override; + /// \brief Fix register kill flags that scheduling has made invalid. + void fixupKills(MachineBasicBlock *MBB); protected: void initSUnits(); void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx); void addPhysRegDeps(SUnit *SU, unsigned OperIdx); void addVRegDefDeps(SUnit *SU, unsigned OperIdx); void addVRegUseDeps(SUnit *SU, unsigned OperIdx); + + /// \brief PostRA helper for rewriting kill flags. + void startBlockForKills(MachineBasicBlock *BB); + + /// \brief Toggle a register operand kill flag. + /// + /// Other adjustments may be made to the instruction if necessary. Return + /// true if the operand has been deleted, false if not. + bool toggleKillFlag(MachineInstr *MI, MachineOperand &MO); }; /// newSUnit - Creates a new SUnit and return a ptr to it. diff --git a/include/llvm/CodeGen/ScheduleHazardRecognizer.h b/include/llvm/CodeGen/ScheduleHazardRecognizer.h index 9dfa3446ef..8a40e7212f 100644 --- a/include/llvm/CodeGen/ScheduleHazardRecognizer.h +++ b/include/llvm/CodeGen/ScheduleHazardRecognizer.h @@ -70,6 +70,22 @@ public: /// emitted, to advance the hazard state. virtual void EmitInstruction(SUnit *) {} + /// PreEmitNoops - This callback is invoked prior to emitting an instruction. + /// It should return the number of noops to emit prior to the provided + /// instruction. + /// Note: This is only used during PostRA scheduling. EmitNoop is not called + /// for these noops. + virtual unsigned PreEmitNoops(SUnit *) { + return 0; + } + + /// ShouldPreferAnother - This callback may be invoked if getHazardType + /// returns NoHazard. If, even though there is no hazard, it would be better to + /// schedule another available instruction, this callback should return true. + virtual bool ShouldPreferAnother(SUnit *) { + return false; + } + /// AdvanceCycle - This callback is invoked whenever the next top-down /// instruction to be scheduled cannot issue in the current cycle, either /// because of latency or resource conflicts. This should increment the diff --git a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h index c2103fb233..fbbbb0c936 100644 --- a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h +++ b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h @@ -110,15 +110,15 @@ public: /// atIssueLimit - Return true if no more instructions may be issued in this /// cycle. - virtual bool atIssueLimit() const; + bool atIssueLimit() const override; // Stalls provides an cycle offset at which SU will be scheduled. It will be // negative for bottom-up scheduling. - virtual HazardType getHazardType(SUnit *SU, int Stalls); - virtual void Reset(); - virtual void EmitInstruction(SUnit *SU); - virtual void AdvanceCycle(); - virtual void RecedeCycle(); + HazardType getHazardType(SUnit *SU, int Stalls) override; + void Reset() override; + void EmitInstruction(SUnit *SU) override; + void AdvanceCycle() override; + void RecedeCycle() override; }; } diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 82becca315..a30656a112 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -36,7 +36,6 @@ class MDNode; class SDDbgValue; class TargetLowering; class TargetSelectionDAGInfo; -class TargetTransformInfo; class SDVTListNode : public FoldingSetNode { friend struct FoldingSetTrait<SDVTListNode>; @@ -169,7 +168,6 @@ void checkForCycles(const SelectionDAG *DAG); class SelectionDAG { const TargetMachine &TM; const TargetSelectionDAGInfo &TSI; - const TargetTransformInfo *TTI; const TargetLowering *TLI; MachineFunction *MF; LLVMContext *Context; @@ -269,8 +267,7 @@ public: /// init - Prepare this SelectionDAG to process code in the given /// MachineFunction. /// - void init(MachineFunction &mf, const TargetTransformInfo *TTI, - const TargetLowering *TLI); + void init(MachineFunction &mf, const TargetLowering *TLI); /// clear - Clear state and free memory necessary to make this /// SelectionDAG ready to process a new block. @@ -281,7 +278,6 @@ public: const TargetMachine &getTarget() const { return TM; } const TargetLowering &getTargetLoweringInfo() const { return *TLI; } const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return TSI; } - const TargetTransformInfo *getTargetTransformInfo() const { return TTI; } LLVMContext *getContext() const {return Context; } /// viewGraph - Pop up a GraphViz/gv window with the DAG rendered using 'dot'. @@ -401,18 +397,22 @@ public: //===--------------------------------------------------------------------===// // Node creation methods. // - SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false); - SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false); - SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false); + SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false, + bool isOpaque = false); + SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false, + bool isOpaque = false); + SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false, + bool isOpaque = false); SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false); - SDValue getTargetConstant(uint64_t Val, EVT VT) { - return getConstant(Val, VT, true); + SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) { + return getConstant(Val, VT, true, isOpaque); } - SDValue getTargetConstant(const APInt &Val, EVT VT) { - return getConstant(Val, VT, true); + SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) { + return getConstant(Val, VT, true, isOpaque); } - SDValue getTargetConstant(const ConstantInt &Val, EVT VT) { - return getConstant(Val, VT, true); + SDValue getTargetConstant(const ConstantInt &Val, EVT VT, + bool isOpaque = false) { + return getConstant(Val, VT, true, isOpaque); } // The forms below that take a double should only be used for simple // constants that can be exactly represented in VT. No checks are made. @@ -692,12 +692,14 @@ public: SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result (if relevant) @@ -726,9 +728,13 @@ public: /// getAtomic - Gets a node for an atomic op, produces result and chain and /// takes N operands. SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList, - SDValue* Ops, unsigned NumOps, MachineMemOperand *MMO, - AtomicOrdering Ordering, + SDValue *Ops, unsigned NumOps, MachineMemOperand *MMO, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, SynchronizationScope SynchScope); + SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList, + SDValue *Ops, unsigned NumOps, MachineMemOperand *MMO, + AtomicOrdering Ordering, SynchronizationScope SynchScope); /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, @@ -1143,7 +1149,7 @@ public: /// low/high part. std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) { EVT LoVT, HiVT; - llvm::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); + std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType()); return SplitVector(N, DL, LoVT, HiVT); } diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index b5ec8cb075..b92b6ec424 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -30,7 +30,6 @@ namespace llvm { class MachineInstr; class TargetLowering; class TargetLibraryInfo; - class TargetTransformInfo; class FunctionLoweringInfo; class ScheduleHazardRecognizer; class GCFunctionInfo; @@ -43,7 +42,6 @@ class SelectionDAGISel : public MachineFunctionPass { public: TargetMachine &TM; const TargetLibraryInfo *LibInfo; - const TargetTransformInfo *TTI; FunctionLoweringInfo *FuncInfo; MachineFunction *MF; MachineRegisterInfo *RegInfo; @@ -62,9 +60,9 @@ public: return TM.getTargetLowering(); } - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + void getAnalysisUsage(AnalysisUsage &AU) const override; - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; virtual void EmitFunctionEntryCode() {} @@ -125,6 +123,8 @@ public: OPC_CheckChild3Type, OPC_CheckChild4Type, OPC_CheckChild5Type, OPC_CheckChild6Type, OPC_CheckChild7Type, OPC_CheckInteger, + OPC_CheckChild0Integer, OPC_CheckChild1Integer, OPC_CheckChild2Integer, + OPC_CheckChild3Integer, OPC_CheckChild4Integer, OPC_CheckCondCode, OPC_CheckValueType, OPC_CheckComplexPat, diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 70c15e6c6e..fd915b01d6 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -29,9 +29,9 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/DataTypes.h" -#include "llvm/Support/DebugLoc.h" #include "llvm/Support/MathExtras.h" #include <cassert> @@ -70,6 +70,10 @@ namespace ISD { /// BUILD_VECTOR where all of the elements are 0 or undef. bool isBuildVectorAllZeros(const SDNode *N); + /// \brief Return true if the specified node is a BUILD_VECTOR node of + /// all ConstantSDNode or undef. + bool isBuildVectorOfConstantSDNodes(const SDNode *N); + /// isScalarToVector - Return true if the specified node is a /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low /// element is not an undef. @@ -116,7 +120,7 @@ public: return !operator==(O); } bool operator<(const SDValue &O) const { - return Node < O.Node || (Node == O.Node && ResNo < O.ResNo); + return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); } SDValue getValue(unsigned R) const { @@ -141,6 +145,10 @@ public: return getValueType().getSizeInBits(); } + unsigned getScalarValueSizeInBits() const { + return getValueType().getScalarType().getSizeInBits(); + } + // Forwarding methods - These forward to the corresponding methods in SDNode. inline unsigned getOpcode() const; inline unsigned getNumOperands() const; @@ -404,7 +412,7 @@ public: /// hasOneUse - Return true if there is exactly one use of this node. /// bool hasOneUse() const { - return !use_empty() && llvm::next(use_begin()) == use_end(); + return !use_empty() && std::next(use_begin()) == use_end(); } /// use_size - Return the number of uses of this node. This method takes @@ -1085,15 +1093,27 @@ public: class AtomicSDNode : public MemSDNode { SDUse Ops[4]; - void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) { + /// For cmpxchg instructions, the ordering requirements when a store does not + /// occur. + AtomicOrdering FailureOrdering; + + void InitAtomic(AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope) { // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. - assert((Ordering & 15) == Ordering && + assert((SuccessOrdering & 15) == SuccessOrdering && + "Ordering may not require more than 4 bits!"); + assert((FailureOrdering & 15) == FailureOrdering && "Ordering may not require more than 4 bits!"); assert((SynchScope & 1) == SynchScope && "SynchScope may not require more than 1 bit!"); - SubclassData |= Ordering << 8; + SubclassData |= SuccessOrdering << 8; SubclassData |= SynchScope << 12; - assert(getOrdering() == Ordering && "Ordering encoding error!"); + this->FailureOrdering = FailureOrdering; + assert(getSuccessOrdering() == SuccessOrdering && + "Ordering encoding error!"); + assert(getFailureOrdering() == FailureOrdering && + "Ordering encoding error!"); assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } @@ -1107,12 +1127,11 @@ public: // SrcVal: address to update as a Value (used for MemOperand) // Align: alignment of memory AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, - EVT MemVT, - SDValue Chain, SDValue Ptr, - SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, + MachineMemOperand *MMO, AtomicOrdering Ordering, + SynchronizationScope SynchScope) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + InitAtomic(Ordering, Ordering, SynchScope); InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, @@ -1121,7 +1140,7 @@ public: SDValue Val, MachineMemOperand *MMO, AtomicOrdering Ordering, SynchronizationScope SynchScope) : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, Ordering, SynchScope); InitOperands(Ops, Chain, Ptr, Val); } AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, @@ -1130,15 +1149,16 @@ public: MachineMemOperand *MMO, AtomicOrdering Ordering, SynchronizationScope SynchScope) : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, Ordering, SynchScope); InitOperands(Ops, Chain, Ptr); } AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue* AllOps, SDUse *DynOps, unsigned NumOps, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope) : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); assert((DynOps || NumOps <= array_lengthof(Ops)) && "Too many ops for internal storage!"); InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps); @@ -1147,6 +1167,16 @@ public: const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } + AtomicOrdering getSuccessOrdering() const { + return getOrdering(); + } + + // Not quite enough room in SubclassData for everything, so failure gets its + // own field. + AtomicOrdering getFailureOrdering() const { + return FailureOrdering; + } + bool isCompareAndSwap() const { unsigned Op = getOpcode(); return Op == ISD::ATOMIC_CMP_SWAP; @@ -1246,9 +1276,10 @@ public: class ConstantSDNode : public SDNode { const ConstantInt *Value; friend class SelectionDAG; - ConstantSDNode(bool isTarget, const ConstantInt *val, EVT VT) + ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), getSDVTList(VT)), Value(val) { + SubclassData |= (uint16_t)isOpaque; } public: @@ -1261,6 +1292,8 @@ public: bool isNullValue() const { return Value->isNullValue(); } bool isAllOnesValue() const { return Value->isAllOnesValue(); } + bool isOpaque() const { return SubclassData & 1; } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Constant || N->getOpcode() == ISD::TargetConstant; @@ -1486,7 +1519,16 @@ public: /// undefined. isBigEndian describes the endianness of the target. bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, - unsigned MinSplatBits = 0, bool isBigEndian = false); + unsigned MinSplatBits = 0, + bool isBigEndian = false) const; + + /// getConstantSplatValue - Check if this is a constant splat, and if so, + /// return the splat value only if it is a ConstantSDNode. Otherwise + /// return nullptr. This is a simpler form of isConstantSplat. + /// Get the constant splat only if you care about the splat value. + ConstantSDNode *getConstantSplatValue() const; + + bool isConstant() const; static inline bool classof(const SDNode *N) { return N->getOpcode() == ISD::BUILD_VECTOR; diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 984796af86..1cc34d57c9 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -377,10 +377,10 @@ namespace llvm { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); } - virtual void getAnalysisUsage(AnalysisUsage &au) const; - virtual void releaseMemory(); + void getAnalysisUsage(AnalysisUsage &au) const override; + void releaseMemory() override; - virtual bool runOnMachineFunction(MachineFunction &fn); + bool runOnMachineFunction(MachineFunction &fn) override; /// Dump the indexes. void dump() const; @@ -545,7 +545,7 @@ namespace llvm { std::lower_bound(idx2MBBMap.begin(), idx2MBBMap.end(), start); if (itr == idx2MBBMap.end()) { - itr = prior(itr); + itr = std::prev(itr); return itr->second; } @@ -553,7 +553,7 @@ namespace llvm { if (itr->first < end) return 0; - itr = prior(itr); + itr = std::prev(itr); if (itr->first <= start) return itr->second; @@ -581,11 +581,11 @@ namespace llvm { if (Late) { // Insert mi's index immediately before the following instruction. nextItr = getIndexAfter(mi).listEntry(); - prevItr = prior(nextItr); + prevItr = std::prev(nextItr); } else { // Insert mi's index immediately after the preceding instruction. prevItr = getIndexBefore(mi).listEntry(); - nextItr = llvm::next(prevItr); + nextItr = std::next(prevItr); } // Get a number for the new instr, or 0 if there's no room currently. @@ -638,7 +638,7 @@ namespace llvm { /// Add the given MachineBasicBlock into the maps. void insertMBBInMaps(MachineBasicBlock *mbb) { MachineFunction::iterator nextMBB = - llvm::next(MachineFunction::iterator(mbb)); + std::next(MachineFunction::iterator(mbb)); IndexListEntry *startEntry = 0; IndexListEntry *endEntry = 0; diff --git a/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/include/llvm/CodeGen/StackMapLivenessAnalysis.h new file mode 100644 index 0000000000..6ba7256e44 --- /dev/null +++ b/include/llvm/CodeGen/StackMapLivenessAnalysis.h @@ -0,0 +1,65 @@ +//===--- StackMapLivenessAnalysis - StackMap Liveness Analysis --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass calculates the liveness for each basic block in a function and +// attaches the register live-out information to a stackmap or patchpoint +// intrinsic if present. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H +#define LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H + +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineFunctionPass.h" + + +namespace llvm { + +/// \brief This pass calculates the liveness information for each basic block in +/// a function and attaches the register live-out information to a stackmap or +/// patchpoint intrinsic if present. +/// +/// This is an optional pass that has to be explicitly enabled via the +/// -enable-stackmap-liveness and/or -enable-patchpoint-liveness flag. The pass +/// skips functions that don't have any stackmap or patchpoint intrinsics. The +/// information provided by this pass is optional and not required by the +/// aformentioned intrinsics to function. +class StackMapLiveness : public MachineFunctionPass { + MachineFunction *MF; + const TargetRegisterInfo *TRI; + LivePhysRegs LiveRegs; +public: + static char ID; + + /// \brief Default construct and initialize the pass. + StackMapLiveness(); + + /// \brief Tell the pass manager which passes we depend on and what + /// information we preserve. + void getAnalysisUsage(AnalysisUsage &AU) const override; + + /// \brief Calculate the liveness information for the given machine function. + bool runOnMachineFunction(MachineFunction &MF) override; + +private: + /// \brief Performs the actual liveness calculation for the function. + bool calculateLiveness(); + + /// \brief Add the current register live set to the instruction. + void addLiveOutSetToMI(MachineInstr &MI); + + /// \brief Create a register mask and initialize it with the registers from + /// the register live set. + uint32_t *createRegisterMask() const; +}; + +} // llvm namespace + +#endif // LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index e90f22e5b6..a62ab6edd4 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -1,4 +1,5 @@ //===------------------- StackMaps.h - StackMaps ----------------*- C++ -*-===// + // // The LLVM Compiler Infrastructure // @@ -10,6 +11,7 @@ #ifndef LLVM_STACKMAPS #define LLVM_STACKMAPS +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" #include <map> @@ -92,19 +94,28 @@ public: : LocType(LocType), Size(Size), Reg(Reg), Offset(Offset) {} }; - // Typedef a function pointer for functions that parse sequences of operands - // and return a Location, plus a new "next" operand iterator. - typedef std::pair<Location, MachineInstr::const_mop_iterator> - (*OperandParser)(MachineInstr::const_mop_iterator, - MachineInstr::const_mop_iterator, const TargetMachine&); + struct LiveOutReg { + unsigned short Reg; + unsigned short RegNo; + unsigned short Size; + + LiveOutReg() : Reg(0), RegNo(0), Size(0) {} + LiveOutReg(unsigned short Reg, unsigned short RegNo, unsigned short Size) + : Reg(Reg), RegNo(RegNo), Size(Size) {} + + void MarkInvalid() { Reg = 0; } + + // Only sort by the dwarf register number. + bool operator< (const LiveOutReg &LO) const { return RegNo < LO.RegNo; } + static bool IsInvalid(const LiveOutReg &LO) { return LO.Reg == 0; } + }; // OpTypes are used to encode information about the following logical // operand (which may consist of several MachineOperands) for the // OpParser. typedef enum { DirectMemRefOp, IndirectMemRefOp, ConstantOp } OpType; - StackMaps(AsmPrinter &AP, OperandParser OpParser) - : AP(AP), OpParser(OpParser) {} + StackMaps(AsmPrinter &AP) : AP(AP) {} /// \brief Generate a stackmap record for a stackmap instruction. /// @@ -121,15 +132,19 @@ public: private: typedef SmallVector<Location, 8> LocationVec; + typedef SmallVector<LiveOutReg, 8> LiveOutVec; + typedef MapVector<const MCSymbol *, uint64_t> FnStackSizeMap; struct CallsiteInfo { const MCExpr *CSOffsetExpr; - unsigned ID; + uint64_t ID; LocationVec Locations; + LiveOutVec LiveOuts; CallsiteInfo() : CSOffsetExpr(0), ID(0) {} - CallsiteInfo(const MCExpr *CSOffsetExpr, unsigned ID, - LocationVec Locations) - : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations) {} + CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID, + LocationVec &Locations, LiveOutVec &LiveOuts) + : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations), + LiveOuts(LiveOuts) {} }; typedef std::vector<CallsiteInfo> CallsiteInfoList; @@ -155,16 +170,29 @@ private: }; AsmPrinter &AP; - OperandParser OpParser; CallsiteInfoList CSInfos; ConstantPool ConstPool; + FnStackSizeMap FnStackSize; + + MachineInstr::const_mop_iterator + parseOperand(MachineInstr::const_mop_iterator MOI, + MachineInstr::const_mop_iterator MOE, + LocationVec &Locs, LiveOutVec &LiveOuts) const; + + /// \brief Create a live-out register record for the given register @p Reg. + LiveOutReg createLiveOutReg(unsigned Reg, + const TargetRegisterInfo *TRI) const; + + /// \brief Parse the register live-out mask and return a vector of live-out + /// registers that need to be recorded in the stackmap. + LiveOutVec parseRegisterLiveOutMask(const uint32_t *Mask) const; /// This should be called by the MC lowering code _immediately_ before /// lowering the MI to an MCInst. It records where the operands for the /// instruction are stored, and outputs a label to record the offset of /// the call from the start of the text section. In special cases (e.g. AnyReg /// calling convention) the return register is also recorded if requested. - void recordStackMapOpers(const MachineInstr &MI, uint32_t ID, + void recordStackMapOpers(const MachineInstr &MI, uint64_t ID, MachineInstr::const_mop_iterator MOI, MachineInstr::const_mop_iterator MOE, bool recordResult = false); diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h index d09a933a66..0b8b8c0662 100644 --- a/include/llvm/CodeGen/StackProtector.h +++ b/include/llvm/CodeGen/StackProtector.h @@ -19,12 +19,12 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Triple.h" -#include "llvm/ADT/ValueMap.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/ValueMap.h" #include "llvm/Pass.h" #include "llvm/Target/TargetLowering.h" namespace llvm { -class DominatorTree; class Function; class Module; class PHINode; @@ -114,13 +114,14 @@ public: initializeStackProtectorPass(*PassRegistry::getPassRegistry()); } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.addPreserved<DominatorTree>(); + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addPreserved<DominatorTreeWrapperPass>(); } SSPLayoutKind getSSPLayout(const AllocaInst *AI) const; + void adjustForColoring(const AllocaInst *From, const AllocaInst *To); - virtual bool runOnFunction(Function &Fn); + bool runOnFunction(Function &Fn) override; }; } // end namespace llvm diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 5b22c9c685..16fed32aa1 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -38,81 +38,81 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile { public: virtual ~TargetLoweringObjectFileELF() {} - virtual void emitPersonalityValue(MCStreamer &Streamer, + void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM, + const MCSymbol *Sym) const override; + + /// Given a constant with the SectionKind, return a section that it should be + /// placed in. + const MCSection *getSectionForConstant(SectionKind Kind) const override; + + const MCSection *getExplicitSectionGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; + + const MCSection *SelectSectionForGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; + + /// Return an MCExpr to use for a reference to the specified type info global + /// variable from exception handling information. + const MCExpr * + getTTypeGlobalReference(const GlobalValue *GV, unsigned Encoding, + Mangler &Mang, const TargetMachine &TM, + MachineModuleInfo *MMI, + MCStreamer &Streamer) const override; + + // The symbol that gets passed to .cfi_personality. + MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang, const TargetMachine &TM, - const MCSymbol *Sym) const; + MachineModuleInfo *MMI) const override; - /// getSectionForConstant - Given a constant with the SectionKind, return a - /// section that it should be placed in. - virtual const MCSection *getSectionForConstant(SectionKind Kind) const; + void InitializeELF(bool UseInitArray_); + const MCSection * + getStaticCtorSection(unsigned Priority = 65535) const override; + const MCSection * + getStaticDtorSection(unsigned Priority = 65535) const override; +}; - virtual const MCSection * - getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; - virtual const MCSection * - SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; +class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { +public: + virtual ~TargetLoweringObjectFileMachO() {} - /// getTTypeGlobalReference - Return an MCExpr to use for a reference to the - /// specified type info global variable from exception handling information. - virtual const MCExpr * - getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, unsigned Encoding, - MCStreamer &Streamer) const; + /// Extract the dependent library name from a linker option string. Returns + /// StringRef() if the option does not specify a library. + StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override; - // getCFIPersonalitySymbol - The symbol that gets passed to .cfi_personality. - virtual MCSymbol * - getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI) const; + /// Emit the module flags that specify the garbage collection information. + void emitModuleFlags(MCStreamer &Streamer, + ArrayRef<Module::ModuleFlagEntry> ModuleFlags, + Mangler &Mang, const TargetMachine &TM) const override; - void InitializeELF(bool UseInitArray_); - virtual const MCSection * - getStaticCtorSection(unsigned Priority = 65535) const; - virtual const MCSection * - getStaticDtorSection(unsigned Priority = 65535) const; -}; + bool isSectionAtomizableBySymbols(const MCSection &Section) const override; + const MCSection * + SelectSectionForGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; + const MCSection * + getExplicitSectionGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; -class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { -public: - virtual ~TargetLoweringObjectFileMachO() {} + const MCSection *getSectionForConstant(SectionKind Kind) const override; + + /// The mach-o version of this method defaults to returning a stub reference. + const MCExpr * + getTTypeGlobalReference(const GlobalValue *GV, unsigned Encoding, + Mangler &Mang, const TargetMachine &TM, + MachineModuleInfo *MMI, + MCStreamer &Streamer) const override; - /// emitModuleFlags - Emit the module flags that specify the garbage - /// collection information. - virtual void emitModuleFlags(MCStreamer &Streamer, - ArrayRef<Module::ModuleFlagEntry> ModuleFlags, - Mangler *Mang, const TargetMachine &TM) const; - - virtual const MCSection * - SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; - - virtual const MCSection * - getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; - - virtual const MCSection *getSectionForConstant(SectionKind Kind) const; - - /// shouldEmitUsedDirectiveFor - This hook allows targets to selectively - /// decide not to emit the UsedDirective for some symbols in llvm.used. - /// FIXME: REMOVE this (rdar://7071300) - virtual bool shouldEmitUsedDirectiveFor(const GlobalValue *GV, - Mangler *) const; - - /// getTTypeGlobalReference - The mach-o version of this method - /// defaults to returning a stub reference. - virtual const MCExpr * - getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, unsigned Encoding, - MCStreamer &Streamer) const; - - // getCFIPersonalitySymbol - The symbol that gets passed to .cfi_personality. - virtual MCSymbol * - getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI) const; + // The symbol that gets passed to .cfi_personality. + MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang, + const TargetMachine &TM, + MachineModuleInfo *MMI) const override; }; @@ -121,19 +121,25 @@ class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile { public: virtual ~TargetLoweringObjectFileCOFF() {} - virtual const MCSection * - getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; - - virtual const MCSection * - SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, - Mangler *Mang, const TargetMachine &TM) const; - - /// emitModuleFlags - Emit Obj-C garbage collection and linker options. Only - /// linker option emission is implemented for COFF. - virtual void emitModuleFlags(MCStreamer &Streamer, - ArrayRef<Module::ModuleFlagEntry> ModuleFlags, - Mangler *Mang, const TargetMachine &TM) const; + const MCSection * + getExplicitSectionGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; + + const MCSection * + SelectSectionForGlobal(const GlobalValue *GV, + SectionKind Kind, Mangler &Mang, + const TargetMachine &TM) const override; + + /// Extract the dependent library name from a linker option string. Returns + /// StringRef() if the option does not specify a library. + StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override; + + /// Emit Obj-C garbage collection and linker options. Only linker option + /// emission is implemented for COFF. + void emitModuleFlags(MCStreamer &Streamer, + ArrayRef<Module::ModuleFlagEntry> ModuleFlags, + Mangler &Mang, const TargetMachine &TM) const override; }; } // end namespace llvm diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index 8ef26b7ca5..4e178d057b 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -98,6 +98,14 @@ public: return SchedModel.getProcResource(PIdx); } +#ifndef NDEBUG + const char *getResourceName(unsigned PIdx) const { + if (!PIdx) + return "MOps"; + return SchedModel.getProcResource(PIdx)->Name; + } +#endif + typedef const MCWriteProcResEntry *ProcResIter; // \brief Get an iterator into the processor resources consumed by this @@ -150,7 +158,7 @@ public: /// model. /// /// Compute and return the expected latency of this instruction independent of - /// a particular use. computeOperandLatency is the prefered API, but this is + /// a particular use. computeOperandLatency is the preferred API, but this is /// occasionally useful to help estimate instruction cost. /// /// If UseDefaultDefLatency is false and no new machine sched model is diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index 79f323341f..8cf26fa30d 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -16,559 +16,14 @@ #ifndef LLVM_CODEGEN_VALUETYPES_H #define LLVM_CODEGEN_VALUETYPES_H -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" +#include "llvm/CodeGen/MachineValueType.h" #include <cassert> #include <string> namespace llvm { - class Type; - class LLVMContext; - struct EVT; - - /// MVT - Machine Value Type. Every type that is supported natively by some - /// processor targeted by LLVM occurs here. This means that any legal value - /// type can be represented by an MVT. - class MVT { - public: - enum SimpleValueType { - // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are - // considered extended value types. - INVALID_SIMPLE_VALUE_TYPE = -1, - - // If you change this numbering, you must change the values in - // ValueTypes.td as well! - Other = 0, // This is a non-standard value - i1 = 1, // This is a 1 bit integer value - i8 = 2, // This is an 8 bit integer value - i16 = 3, // This is a 16 bit integer value - i32 = 4, // This is a 32 bit integer value - i64 = 5, // This is a 64 bit integer value - i128 = 6, // This is a 128 bit integer value - - FIRST_INTEGER_VALUETYPE = i1, - LAST_INTEGER_VALUETYPE = i128, - - f16 = 7, // This is a 16 bit floating point value - f32 = 8, // This is a 32 bit floating point value - f64 = 9, // This is a 64 bit floating point value - f80 = 10, // This is a 80 bit floating point value - f128 = 11, // This is a 128 bit floating point value - ppcf128 = 12, // This is a PPC 128-bit floating point value - - FIRST_FP_VALUETYPE = f16, - LAST_FP_VALUETYPE = ppcf128, - - v2i1 = 13, // 2 x i1 - v4i1 = 14, // 4 x i1 - v8i1 = 15, // 8 x i1 - v16i1 = 16, // 16 x i1 - v32i1 = 17, // 32 x i1 - v64i1 = 18, // 64 x i1 - - v1i8 = 19, // 1 x i8 - v2i8 = 20, // 2 x i8 - v4i8 = 21, // 4 x i8 - v8i8 = 22, // 8 x i8 - v16i8 = 23, // 16 x i8 - v32i8 = 24, // 32 x i8 - v64i8 = 25, // 64 x i8 - v1i16 = 26, // 1 x i16 - v2i16 = 27, // 2 x i16 - v4i16 = 28, // 4 x i16 - v8i16 = 29, // 8 x i16 - v16i16 = 30, // 16 x i16 - v32i16 = 31, // 32 x i16 - v1i32 = 32, // 1 x i32 - v2i32 = 33, // 2 x i32 - v4i32 = 34, // 4 x i32 - v8i32 = 35, // 8 x i32 - v16i32 = 36, // 16 x i32 - v1i64 = 37, // 1 x i64 - v2i64 = 38, // 2 x i64 - v4i64 = 39, // 4 x i64 - v8i64 = 40, // 8 x i64 - v16i64 = 41, // 16 x i64 - - FIRST_INTEGER_VECTOR_VALUETYPE = v2i1, - LAST_INTEGER_VECTOR_VALUETYPE = v16i64, - - v2f16 = 42, // 2 x f16 - v4f16 = 43, // 4 x f16 - v8f16 = 44, // 8 x f16 - v1f32 = 45, // 1 x f32 - v2f32 = 46, // 2 x f32 - v4f32 = 47, // 4 x f32 - v8f32 = 48, // 8 x f32 - v16f32 = 49, // 16 x f32 - v1f64 = 50, // 1 x f64 - v2f64 = 51, // 2 x f64 - v4f64 = 52, // 4 x f64 - v8f64 = 53, // 8 x f64 - - FIRST_FP_VECTOR_VALUETYPE = v2f16, - LAST_FP_VECTOR_VALUETYPE = v8f64, - - FIRST_VECTOR_VALUETYPE = v2i1, - LAST_VECTOR_VALUETYPE = v8f64, - - x86mmx = 54, // This is an X86 MMX value - - Glue = 55, // This glues nodes together during pre-RA sched - - isVoid = 56, // This has no value - - Untyped = 57, // This value takes a register, but has - // unspecified type. The register class - // will be determined by the opcode. - - LAST_VALUETYPE = 58, // This always remains at the end of the list. - - // This is the current maximum for LAST_VALUETYPE. - // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors - // This value must be a multiple of 32. - MAX_ALLOWED_VALUETYPE = 64, - - // Metadata - This is MDNode or MDString. - Metadata = 250, - - // iPTRAny - An int value the size of the pointer of the current - // target to any address space. This must only be used internal to - // tblgen. Other than for overloading, we treat iPTRAny the same as iPTR. - iPTRAny = 251, - - // vAny - A vector with any length and element size. This is used - // for intrinsics that have overloadings based on vector types. - // This is only for tblgen's consumption! - vAny = 252, - - // fAny - Any floating-point or vector floating-point value. This is used - // for intrinsics that have overloadings based on floating-point types. - // This is only for tblgen's consumption! - fAny = 253, - - // iAny - An integer or vector integer value of any bit width. This is - // used for intrinsics that have overloadings based on integer bit widths. - // This is only for tblgen's consumption! - iAny = 254, - - // iPTR - An int value the size of the pointer of the current - // target. This should only be used internal to tblgen! - iPTR = 255 - }; - - SimpleValueType SimpleTy; - - MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {} - MVT(SimpleValueType SVT) : SimpleTy(SVT) { } - - bool operator>(const MVT& S) const { return SimpleTy > S.SimpleTy; } - bool operator<(const MVT& S) const { return SimpleTy < S.SimpleTy; } - bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; } - bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; } - bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; } - bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; } - - /// isFloatingPoint - Return true if this is a FP, or a vector FP type. - bool isFloatingPoint() const { - return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE && - SimpleTy <= MVT::LAST_FP_VALUETYPE) || - (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE && - SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE)); - } - - /// isInteger - Return true if this is an integer, or a vector integer type. - bool isInteger() const { - return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE && - SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) || - (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE && - SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE)); - } - - /// isVector - Return true if this is a vector value type. - bool isVector() const { - return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE && - SimpleTy <= MVT::LAST_VECTOR_VALUETYPE); - } - - /// is16BitVector - Return true if this is a 16-bit vector type. - bool is16BitVector() const { - return (SimpleTy == MVT::v2i8 || SimpleTy == MVT::v1i16 || - SimpleTy == MVT::v16i1); - } - - /// is32BitVector - Return true if this is a 32-bit vector type. - bool is32BitVector() const { - return (SimpleTy == MVT::v4i8 || SimpleTy == MVT::v2i16 || - SimpleTy == MVT::v1i32); - } - - /// is64BitVector - Return true if this is a 64-bit vector type. - bool is64BitVector() const { - return (SimpleTy == MVT::v8i8 || SimpleTy == MVT::v4i16 || - SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 || - SimpleTy == MVT::v1f64 || SimpleTy == MVT::v2f32); - } - - /// is128BitVector - Return true if this is a 128-bit vector type. - bool is128BitVector() const { - return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || - SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || - SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64); - } - - /// is256BitVector - Return true if this is a 256-bit vector type. - bool is256BitVector() const { - return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 || - SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 || - SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64); - } - - /// is512BitVector - Return true if this is a 512-bit vector type. - bool is512BitVector() const { - return (SimpleTy == MVT::v8f64 || SimpleTy == MVT::v16f32 || - SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 || - SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32); - } - - /// is1024BitVector - Return true if this is a 1024-bit vector type. - bool is1024BitVector() const { - return (SimpleTy == MVT::v16i64); - } - - /// isPow2VectorType - Returns true if the given vector is a power of 2. - bool isPow2VectorType() const { - unsigned NElts = getVectorNumElements(); - return !(NElts & (NElts - 1)); - } - - /// getPow2VectorType - Widens the length of the given vector MVT up to - /// the nearest power of 2 and returns that type. - MVT getPow2VectorType() const { - if (isPow2VectorType()) - return *this; - - unsigned NElts = getVectorNumElements(); - unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts); - return MVT::getVectorVT(getVectorElementType(), Pow2NElts); - } - - /// getScalarType - If this is a vector type, return the element type, - /// otherwise return this. - MVT getScalarType() const { - return isVector() ? getVectorElementType() : *this; - } - - MVT getVectorElementType() const { - switch (SimpleTy) { - default: - llvm_unreachable("Not a vector MVT!"); - case v2i1 : - case v4i1 : - case v8i1 : - case v16i1 : - case v32i1 : - case v64i1: return i1; - case v1i8 : - case v2i8 : - case v4i8 : - case v8i8 : - case v16i8: - case v32i8: - case v64i8: return i8; - case v1i16: - case v2i16: - case v4i16: - case v8i16: - case v16i16: - case v32i16: return i16; - case v1i32: - case v2i32: - case v4i32: - case v8i32: - case v16i32: return i32; - case v1i64: - case v2i64: - case v4i64: - case v8i64: - case v16i64: return i64; - case v2f16: - case v4f16: - case v8f16: return f16; - case v1f32: - case v2f32: - case v4f32: - case v8f32: - case v16f32: return f32; - case v1f64: - case v2f64: - case v4f64: - case v8f64: return f64; - } - } - - unsigned getVectorNumElements() const { - switch (SimpleTy) { - default: - llvm_unreachable("Not a vector MVT!"); - case v32i1: - case v32i8: - case v32i16: return 32; - case v64i1: - case v64i8: return 64; - case v16i1: - case v16i8: - case v16i16: - case v16i32: - case v16i64: - case v16f32: return 16; - case v8i1 : - case v8i8 : - case v8i16: - case v8i32: - case v8i64: - case v8f16: - case v8f32: - case v8f64: return 8; - case v4i1: - case v4i8: - case v4i16: - case v4i32: - case v4i64: - case v4f16: - case v4f32: - case v4f64: return 4; - case v2i1: - case v2i8: - case v2i16: - case v2i32: - case v2i64: - case v2f16: - case v2f32: - case v2f64: return 2; - case v1i8: - case v1i16: - case v1i32: - case v1i64: - case v1f32: - case v1f64: return 1; - } - } - - unsigned getSizeInBits() const { - switch (SimpleTy) { - default: - llvm_unreachable("getSizeInBits called on extended MVT."); - case Other: - llvm_unreachable("Value type is non-standard value, Other."); - case iPTR: - llvm_unreachable("Value type size is target-dependent. Ask TLI."); - case iPTRAny: - case iAny: - case fAny: - case vAny: - llvm_unreachable("Value type is overloaded."); - case Metadata: - llvm_unreachable("Value type is metadata."); - case i1 : return 1; - case v2i1: return 2; - case v4i1: return 4; - case i8 : - case v1i8: - case v8i1: return 8; - case i16 : - case f16: - case v16i1: - case v2i8: - case v1i16: return 16; - case f32 : - case i32 : - case v32i1: - case v4i8: - case v2i16: - case v2f16: - case v1f32: - case v1i32: return 32; - case x86mmx: - case f64 : - case i64 : - case v64i1: - case v8i8: - case v4i16: - case v2i32: - case v1i64: - case v4f16: - case v2f32: - case v1f64: return 64; - case f80 : return 80; - case f128: - case ppcf128: - case i128: - case v16i8: - case v8i16: - case v4i32: - case v2i64: - case v8f16: - case v4f32: - case v2f64: return 128; - case v32i8: - case v16i16: - case v8i32: - case v4i64: - case v8f32: - case v4f64: return 256; - case v64i8: - case v32i16: - case v16i32: - case v8i64: - case v16f32: - case v8f64: return 512; - case v16i64:return 1024; - } - } - - /// getStoreSize - Return the number of bytes overwritten by a store - /// of the specified value type. - unsigned getStoreSize() const { - return (getSizeInBits() + 7) / 8; - } - - /// getStoreSizeInBits - Return the number of bits overwritten by a store - /// of the specified value type. - unsigned getStoreSizeInBits() const { - return getStoreSize() * 8; - } - - /// Return true if this has more bits than VT. - bool bitsGT(MVT VT) const { - return getSizeInBits() > VT.getSizeInBits(); - } - - /// Return true if this has no less bits than VT. - bool bitsGE(MVT VT) const { - return getSizeInBits() >= VT.getSizeInBits(); - } - - /// Return true if this has less bits than VT. - bool bitsLT(MVT VT) const { - return getSizeInBits() < VT.getSizeInBits(); - } - - /// Return true if this has no more bits than VT. - bool bitsLE(MVT VT) const { - return getSizeInBits() <= VT.getSizeInBits(); - } - - - static MVT getFloatingPointVT(unsigned BitWidth) { - switch (BitWidth) { - default: - llvm_unreachable("Bad bit width!"); - case 16: - return MVT::f16; - case 32: - return MVT::f32; - case 64: - return MVT::f64; - case 80: - return MVT::f80; - case 128: - return MVT::f128; - } - } - - static MVT getIntegerVT(unsigned BitWidth) { - switch (BitWidth) { - default: - return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); - case 1: - return MVT::i1; - case 8: - return MVT::i8; - case 16: - return MVT::i16; - case 32: - return MVT::i32; - case 64: - return MVT::i64; - case 128: - return MVT::i128; - } - } - - static MVT getVectorVT(MVT VT, unsigned NumElements) { - switch (VT.SimpleTy) { - default: - break; - case MVT::i1: - if (NumElements == 2) return MVT::v2i1; - if (NumElements == 4) return MVT::v4i1; - if (NumElements == 8) return MVT::v8i1; - if (NumElements == 16) return MVT::v16i1; - if (NumElements == 32) return MVT::v32i1; - if (NumElements == 64) return MVT::v64i1; - break; - case MVT::i8: - if (NumElements == 1) return MVT::v1i8; - if (NumElements == 2) return MVT::v2i8; - if (NumElements == 4) return MVT::v4i8; - if (NumElements == 8) return MVT::v8i8; - if (NumElements == 16) return MVT::v16i8; - if (NumElements == 32) return MVT::v32i8; - if (NumElements == 64) return MVT::v64i8; - break; - case MVT::i16: - if (NumElements == 1) return MVT::v1i16; - if (NumElements == 2) return MVT::v2i16; - if (NumElements == 4) return MVT::v4i16; - if (NumElements == 8) return MVT::v8i16; - if (NumElements == 16) return MVT::v16i16; - if (NumElements == 32) return MVT::v32i16; - break; - case MVT::i32: - if (NumElements == 1) return MVT::v1i32; - if (NumElements == 2) return MVT::v2i32; - if (NumElements == 4) return MVT::v4i32; - if (NumElements == 8) return MVT::v8i32; - if (NumElements == 16) return MVT::v16i32; - break; - case MVT::i64: - if (NumElements == 1) return MVT::v1i64; - if (NumElements == 2) return MVT::v2i64; - if (NumElements == 4) return MVT::v4i64; - if (NumElements == 8) return MVT::v8i64; - if (NumElements == 16) return MVT::v16i64; - break; - case MVT::f16: - if (NumElements == 2) return MVT::v2f16; - if (NumElements == 4) return MVT::v4f16; - if (NumElements == 8) return MVT::v8f16; - break; - case MVT::f32: - if (NumElements == 1) return MVT::v1f32; - if (NumElements == 2) return MVT::v2f32; - if (NumElements == 4) return MVT::v4f32; - if (NumElements == 8) return MVT::v8f32; - if (NumElements == 16) return MVT::v16f32; - break; - case MVT::f64: - if (NumElements == 1) return MVT::v1f64; - if (NumElements == 2) return MVT::v2f64; - if (NumElements == 4) return MVT::v4f64; - if (NumElements == 8) return MVT::v8f64; - break; - } - return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); - } - - /// Return the value type corresponding to the specified type. This returns - /// all pointers as iPTR. If HandleUnknown is true, unknown types are - /// returned as Other, otherwise they are invalid. - static MVT getVT(Type *Ty, bool HandleUnknown = false); - - }; + class LLVMContext; + class Type; /// EVT - Extended Value Type. Capable of holding value types which are not /// native for any processor (such as the i12345 type), as well as the types @@ -782,6 +237,10 @@ namespace llvm { return getExtendedSizeInBits(); } + unsigned getScalarSizeInBits() const { + return getScalarType().getSizeInBits(); + } + /// getStoreSize - Return the number of bytes overwritten by a store /// of the specified value type. unsigned getStoreSize() const { @@ -821,6 +280,14 @@ namespace llvm { return getIntegerVT(Context, (EVTSize + 1) / 2); } + /// \brief Return a VT for an integer vector type with the size of the + /// elements doubled. The typed returned may be an extended type. + EVT widenIntegerVectorElementType(LLVMContext &Context) const { + EVT EltVT = getVectorElementType(); + EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits()); + return EVT::getVectorVT(Context, EltVT, getVectorNumElements()); + } + /// isPow2VectorType - Returns true if the given vector is a power of 2. bool isPow2VectorType() const { unsigned NElts = getVectorNumElements(); @@ -880,18 +347,18 @@ namespace llvm { static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth); static EVT getExtendedVectorVT(LLVMContext &C, EVT VT, unsigned NumElements); - bool isExtendedFloatingPoint() const; - bool isExtendedInteger() const; - bool isExtendedVector() const; - bool isExtended16BitVector() const; - bool isExtended32BitVector() const; - bool isExtended64BitVector() const; - bool isExtended128BitVector() const; - bool isExtended256BitVector() const; - bool isExtended512BitVector() const; - bool isExtended1024BitVector() const; + bool isExtendedFloatingPoint() const LLVM_READONLY; + bool isExtendedInteger() const LLVM_READONLY; + bool isExtendedVector() const LLVM_READONLY; + bool isExtended16BitVector() const LLVM_READONLY; + bool isExtended32BitVector() const LLVM_READONLY; + bool isExtended64BitVector() const LLVM_READONLY; + bool isExtended128BitVector() const LLVM_READONLY; + bool isExtended256BitVector() const LLVM_READONLY; + bool isExtended512BitVector() const LLVM_READONLY; + bool isExtended1024BitVector() const LLVM_READONLY; EVT getExtendedVectorElementType() const; - unsigned getExtendedVectorNumElements() const; + unsigned getExtendedVectorNumElements() const LLVM_READONLY; unsigned getExtendedSizeInBits() const; }; diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h index 3bc6ebd563..89b5a9f668 100644 --- a/include/llvm/CodeGen/VirtRegMap.h +++ b/include/llvm/CodeGen/VirtRegMap.h @@ -70,9 +70,9 @@ namespace llvm { static char ID; VirtRegMap() : MachineFunctionPass(ID), Virt2PhysMap(NO_PHYS_REG), Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) { } - virtual bool runOnMachineFunction(MachineFunction &MF); + bool runOnMachineFunction(MachineFunction &MF) override; - virtual void getAnalysisUsage(AnalysisUsage &AU) const { + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -177,7 +177,7 @@ namespace llvm { /// the specified stack slot void assignVirt2StackSlot(unsigned virtReg, int frameIndex); - void print(raw_ostream &OS, const Module* M = 0) const; + void print(raw_ostream &OS, const Module* M = 0) const override; void dump() const; }; |