diff options
| -rw-r--r-- | include/llvm/CodeGen/CallingConvLower.h | 4 | ||||
| -rw-r--r-- | include/llvm/CodeGen/SelectionDAG.h | 2 | ||||
| -rw-r--r-- | include/llvm/CodeGen/SelectionDAGNodes.h | 85 | ||||
| -rw-r--r-- | include/llvm/CodeGen/ValueTypes.h | 187 | ||||
| -rw-r--r-- | include/llvm/Target/TargetLowering.h | 54 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 532 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 669 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 95 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 357 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/TargetLowering.cpp | 52 | ||||
| -rw-r--r-- | lib/Target/X86/X86ISelLowering.cpp | 12 | ||||
| -rw-r--r-- | lib/VMCore/ValueTypes.cpp | 65 |
12 files changed, 951 insertions, 1163 deletions
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index 4308c377d1..959d0528c6 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -48,10 +48,10 @@ private: LocInfo HTP : 7; /// ValVT - The type of the value being assigned. - MVT::ValueType ValVT : 8; + MVT::ValueType ValVT; /// LocVT - The type of the location being assigned to. - MVT::ValueType LocVT : 8; + MVT::ValueType LocVT; public: static CCValAssign getReg(unsigned ValNo, MVT::ValueType ValVT, diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 56cb17dee6..9388df1e6d 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -319,8 +319,6 @@ public: unsigned Alignment=0); SDOperand getIndexedLoad(SDOperand OrigLoad, SDOperand Base, SDOperand Offset, ISD::MemIndexedMode AM); - SDOperand getVecLoad(unsigned Count, MVT::ValueType VT, SDOperand Chain, - SDOperand Ptr, SDOperand SV); /// getStore - Helper function to build ISD::STORE nodes. /// diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 5a7d47fdc6..792859a9b1 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -237,58 +237,30 @@ namespace ISD { // FCOPYSIGN(f32, f64) is allowed. FCOPYSIGN, - /// VBUILD_VECTOR(ELT1, ELT2, ELT3, ELT4,..., COUNT,TYPE) - Return a vector - /// with the specified, possibly variable, elements. The number of elements - /// is required to be a power of two. - VBUILD_VECTOR, - /// BUILD_VECTOR(ELT1, ELT2, ELT3, ELT4,...) - Return a vector /// with the specified, possibly variable, elements. The number of elements /// is required to be a power of two. BUILD_VECTOR, - /// VINSERT_VECTOR_ELT(VECTOR, VAL, IDX, COUNT,TYPE) - Given a vector - /// VECTOR, an element ELEMENT, and a (potentially variable) index IDX, - /// return a vector with the specified element of VECTOR replaced with VAL. - /// COUNT and TYPE specify the type of vector, as is standard for V* nodes. - VINSERT_VECTOR_ELT, - - /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR (a legal packed - /// type) with the element at IDX replaced with VAL. + /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element + /// at IDX replaced with VAL. INSERT_VECTOR_ELT, - /// VEXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR - /// (an MVT::Vector value) identified by the (potentially variable) element - /// number IDX. - VEXTRACT_VECTOR_ELT, - /// EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR - /// (a legal vector type vector) identified by the (potentially variable) - /// element number IDX. + /// identified by the (potentially variable) element number IDX. EXTRACT_VECTOR_ELT, - /// VCONCAT_VECTORS(VECTOR0, VECTOR1, ..., COUNT,TYPE) - Given a number of - /// values of MVT::Vector type with the same length and element type, this - /// produces a concatenated MVT::Vector result value, with length equal to - /// the sum of the input vectors. This can only be used before - /// legalization. - VCONCAT_VECTORS, + /// CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of + /// vector type with the same length and element type, this produces a + /// concatenated vector result value, with length equal to the sum of the + /// input vectors. + CONCAT_VECTORS, - /// VEXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an - /// MVT::Vector value) starting with the (potentially variable) - /// element number IDX, which must be a multiple of the result vector - /// length. This can only be used before legalization. - VEXTRACT_SUBVECTOR, + /// EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an + /// vector value) starting with the (potentially variable) element number + /// IDX, which must be a multiple of the result vector length. + EXTRACT_SUBVECTOR, - /// VVECTOR_SHUFFLE(VEC1, VEC2, SHUFFLEVEC, COUNT,TYPE) - Returns a vector, - /// of the same type as VEC1/VEC2. SHUFFLEVEC is a VBUILD_VECTOR of - /// constant int values that indicate which value each result element will - /// get. The elements of VEC1/VEC2 are enumerated in order. This is quite - /// similar to the Altivec 'vperm' instruction, except that the indices must - /// be constants and are in terms of the element size of VEC1/VEC2, not in - /// terms of bytes. - VVECTOR_SHUFFLE, - /// VECTOR_SHUFFLE(VEC1, VEC2, SHUFFLEVEC) - Returns a vector, of the same /// type as VEC1/VEC2. SHUFFLEVEC is a BUILD_VECTOR of constant int values /// (regardless of whether its datatype is legal or not) that indicate @@ -298,34 +270,6 @@ namespace ISD { /// of the element size of VEC1/VEC2, not in terms of bytes. VECTOR_SHUFFLE, - /// X = VBIT_CONVERT(Y) and X = VBIT_CONVERT(Y, COUNT,TYPE) - This node - /// represents a conversion from or to an ISD::Vector type. - /// - /// This is lowered to a BIT_CONVERT of the appropriate input/output types. - /// The input and output are required to have the same size and at least one - /// is required to be a vector (if neither is a vector, just use - /// BIT_CONVERT). - /// - /// If the result is a vector, this takes three operands (like any other - /// vector producer) which indicate the size and type of the vector result. - /// Otherwise it takes one input. - VBIT_CONVERT, - - /// BINOP(LHS, RHS, COUNT,TYPE) - /// Simple abstract vector operators. Unlike the integer and floating point - /// binary operators, these nodes also take two additional operands: - /// a constant element count, and a value type node indicating the type of - /// the elements. The order is op0, op1, count, type. All vector opcodes, - /// including VLOAD and VConstant must currently have count and type as - /// their last two operands. - VADD, VSUB, VMUL, VSDIV, VUDIV, - VAND, VOR, VXOR, - - /// VSELECT(COND,LHS,RHS, COUNT,TYPE) - Select for MVT::Vector values. - /// COND is a boolean value. This node return LHS if COND is true, RHS if - /// COND is false. - VSELECT, - /// SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a /// scalar value into the low element of the resultant vector type. The top /// elements of the vector are undefined. @@ -432,11 +376,6 @@ namespace ISD { // indexed memory ops). LOAD, STORE, - // Abstract vector version of LOAD. VLOAD has a constant element count as - // the first operand, followed by a value type node indicating the type of - // the elements, a token chain, a pointer operand, and a SRCVALUE node. - VLOAD, - // TRUNCSTORE - This operators truncates (for integer) or rounds (for FP) a // value and stores it to memory in one operation. This can be used for // either integer or floating point operands. The first four operands of diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index a42cee181b..c9983868e8 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -17,16 +17,17 @@ #define LLVM_CODEGEN_VALUETYPES_H #include <cassert> +#include <string> #include "llvm/Support/DataTypes.h" namespace llvm { class Type; -/// MVT namespace - This namespace defines the ValueType enum, which contains -/// the various low-level value types. +/// MVT namespace - This namespace defines the SimpleValueType enum, which +/// contains the various low-level value types, and the ValueType typedef. /// namespace MVT { // MVT = Machine Value Types - enum ValueType { + enum SimpleValueType { // If you change this numbering, you must change the values in ValueTypes.td // well! Other = 0, // This is a non-standard value @@ -45,10 +46,6 @@ namespace MVT { // MVT = Machine Value Types isVoid = 12, // This has no value - Vector = 13, // This is an abstract vector type, which will - // be expanded into a target vector type, or scalars - // if no matching vector type is available. - v8i8 = 14, // 8 x i8 v4i16 = 15, // 4 x i16 v2i32 = 16, // 2 x i32 @@ -76,64 +73,55 @@ namespace MVT { // MVT = Machine Value Types iPTR = 255 }; - /// MVT::isInteger - Return true if this is a simple integer, or a packed - /// vector integer type. - static inline bool isInteger(ValueType VT) { - return (VT >= i1 && VT <= i128) || (VT >= v8i8 && VT <= v2i64); + /// MVT::ValueType - This type holds low-level value types. Valid values + /// include any of the values in the SimpleValueType enum, or any value + /// returned from a function in the MVT namespace that has a ValueType + /// return type. Any value type equal to one of the SimpleValueType enum + /// values is a "simple" value type. All other value types are "extended". + /// + /// Note that simple doesn't necessary mean legal for the target machine. + /// All legal value types must be simple, but often there are some simple + /// value types that are not legal. + typedef uint32_t ValueType; + + static const int SimpleTypeBits = 8; + + static const uint32_t SimpleTypeMask = + (~uint32_t(0) << (32 - SimpleTypeBits)) >> (32 - SimpleTypeBits); + + /// MVT::isExtendedValueType - Test if the given ValueType is extended + /// (as opposed to being simple). + static inline bool isExtendedValueType(ValueType VT) { + return VT & ~SimpleTypeMask; } - /// MVT::isFloatingPoint - Return true if this is a simple FP, or a packed - /// vector FP type. - static inline bool isFloatingPoint(ValueType VT) { - return (VT >= f32 && VT <= f128) || (VT >= v2f32 && VT <= v2f64); + /// MVT::isInteger - Return true if this is an integer, or a vector integer + /// type. + static inline bool isInteger(ValueType VT) { + ValueType SVT = VT & SimpleTypeMask; + return (SVT >= i1 && SVT <= i128) || (SVT >= v8i8 && SVT <= v2i64); } - /// MVT::isVector - Return true if this is a packed vector type (i.e. not - /// MVT::Vector). - static inline bool isVector(ValueType VT) { - return VT >= FIRST_VECTOR_VALUETYPE && VT <= LAST_VECTOR_VALUETYPE; + /// MVT::isFloatingPoint - Return true if this is an FP, or a vector FP type. + static inline bool isFloatingPoint(ValueType VT) { + ValueType SVT = VT & SimpleTypeMask; + return (SVT >= f32 && SVT <= f128) || (SVT >= v2f32 && SVT <= v2f64); } - /// MVT::getSizeInBits - Return the size of the specified value type in bits. - /// - static inline unsigned getSizeInBits(ValueType VT) { - switch (VT) { - default: assert(0 && "ValueType has no known size!"); - case MVT::i1 : return 1; - case MVT::i8 : return 8; - case MVT::i16 : return 16; - case MVT::f32 : - case MVT::i32 : return 32; - case MVT::f64 : - case MVT::i64 : - case MVT::v8i8: - case MVT::v4i16: - case MVT::v2i32: - case MVT::v1i64: - case MVT::v2f32: return 64; - case MVT::f80 : return 80; - case MVT::f128: - case MVT::i128: - case MVT::v16i8: - case MVT::v8i16: - case MVT::v4i32: - case MVT::v2i64: - case MVT::v4f32: - case MVT::v2f64: return 128; - } + /// MVT::isVector - Return true if this is a vector value type. + static inline bool isVector(ValueType VT) { + return (VT >= FIRST_VECTOR_VALUETYPE && VT <= LAST_VECTOR_VALUETYPE) || + isExtendedValueType(VT); } - /// MVT::getVectorType - Returns the ValueType that represents a vector - /// NumElements in length, where each element is of type VT. If there is no - /// ValueType that represents this vector, a ValueType of Other is returned. - /// - ValueType getVectorType(ValueType VT, unsigned NumElements); - - /// MVT::getVectorElementType - Given a packed vector type, return the type of + /// MVT::getVectorElementType - Given a vector type, return the type of /// each element. static inline ValueType getVectorElementType(ValueType VT) { switch (VT) { - default: assert(0 && "Invalid vector type!"); + default: + if (isExtendedValueType(VT)) + return VT & SimpleTypeMask; + assert(0 && "Invalid vector type!"); case v8i8 : case v16i8: return i8; case v4i16: @@ -148,11 +136,14 @@ namespace MVT { // MVT = Machine Value Types } } - /// MVT::getVectorNumElements - Given a packed vector type, return the number - /// of elements it contains. + /// MVT::getVectorNumElements - Given a vector type, return the + /// number of elements it contains. static inline unsigned getVectorNumElements(ValueType VT) { switch (VT) { - default: assert(0 && "Invalid vector type!"); + default: + if (isExtendedValueType(VT)) + return ((VT & ~SimpleTypeMask) >> SimpleTypeBits) - 1; + assert(0 && "Invalid vector type!"); case v16i8: return 16; case v8i8 : case v8i16: return 8; @@ -167,11 +158,84 @@ namespace MVT { // MVT = Machine Value Types } } + /// MVT::getSizeInBits - Return the size of the specified value type + /// in bits. + /// + static inline unsigned getSizeInBits(ValueType VT) { + switch (VT) { + default: + if (isExtendedValueType(VT)) + return getSizeInBits(getVectorElementType(VT)) * + getVectorNumElements(VT); + assert(0 && "ValueType has no known size!"); + case MVT::i1 : return 1; + case MVT::i8 : return 8; + case MVT::i16 : return 16; + case MVT::f32 : + case MVT::i32 : return 32; + case MVT::f64 : + case MVT::i64 : + case MVT::v8i8: + case MVT::v4i16: + case MVT::v2i32: + case MVT::v1i64: + case MVT::v2f32: return 64; + case MVT::f80 : return 80; + case MVT::f128: + case MVT::i128: + case MVT::v16i8: + case MVT::v8i16: + case MVT::v4i32: + case MVT::v2i64: + case MVT::v4f32: + case MVT::v2f64: return 128; + } + } + + /// MVT::getVectorType - Returns the ValueType that represents a vector + /// NumElements in length, where each element is of type VT. + /// + static inline ValueType getVectorType(ValueType VT, unsigned NumElements) { + switch (VT) { + default: + break; + case MVT::i8: + if (NumElements == 8) return MVT::v8i8; + if (NumElements == 16) return MVT::v16i8; + break; + case MVT::i16: + if (NumElements == 4) return MVT::v4i16; + if (NumElements == 8) return MVT::v8i16; + break; + case MVT::i32: + if (NumElements == 2) return MVT::v2i32; + if (NumElements == 4) return MVT::v4i32; + break; + case MVT::i64: + if (NumElements == 1) return MVT::v1i64; + if (NumElements == 2) return MVT::v2i64; + break; + case MVT::f32: + if (NumElements == 2) return MVT::v2f32; + if (NumElements == 4) return MVT::v4f32; + break; + case MVT::f64: + if (NumElements == 2) return MVT::v2f64; + break; + } + ValueType Result = VT | ((NumElements + 1) << SimpleTypeBits); + assert(getVectorElementType(Result) == VT && + "Bad vector element type!"); + assert(getVectorNumElements(Result) == NumElements && + "Bad vector length!"); + return Result; + } + /// MVT::getIntVectorWithNumElements - Return any integer vector type that has /// the specified number of elements. static inline ValueType getIntVectorWithNumElements(unsigned NumElts) { switch (NumElts) { - default: assert(0 && "Invalid vector type!"); + default: return getVectorType(i8, NumElts); case 1: return v1i64; case 2: return v2i32; case 4: return v4i16; @@ -196,7 +260,7 @@ namespace MVT { // MVT = Machine Value Types /// MVT::getValueTypeString - This function returns value type as a string, /// e.g. "i32". - const char *getValueTypeString(ValueType VT); + std::string getValueTypeString(ValueType VT); /// MVT::getTypeForValueType - This method returns an LLVM type corresponding /// to the specified ValueType. For integer types, this returns an unsigned @@ -204,9 +268,8 @@ namespace MVT { // MVT = Machine Value Types const Type *getTypeForValueType(ValueType VT); /// MVT::getValueType - Return the value type corresponding to the specified - /// type. This returns all vectors as MVT::Vector and all pointers as - /// MVT::iPTR. If HandleUnknown is true, unknown types are returned as Other, - /// otherwise they are invalid. + /// type. This returns all pointers as MVT::iPTR. If HandleUnknown is true, + /// unknown types are returned as Other, otherwise they are invalid. ValueType getValueType(const Type *Ty, bool HandleUnknown = false); } diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index 5b98667602..88682f4334 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -120,6 +120,7 @@ public: /// getRegClassFor - Return the register class that should be used for the /// specified value type. This may only be called on legal types. TargetRegisterClass *getRegClassFor(MVT::ValueType VT) const { + assert(!MVT::isExtendedValueType(VT)); TargetRegisterClass *RC = RegClassForVT[VT]; assert(RC && "This value type is not natively supported!"); return RC; @@ -129,7 +130,9 @@ public: /// specified value type. This means that it has a register that directly /// holds it without promotions or expansions. bool isTypeLegal(MVT::ValueType VT) const { - return RegClassForVT[VT] != 0; + return !MVT::isExtendedValueType(VT) ? + RegClassForVT[VT] != 0 : + false; } class ValueTypeActionImpl { @@ -147,9 +150,12 @@ public: } LegalizeAction getTypeAction(MVT::ValueType VT) const { - return (LegalizeAction)((ValueTypeActions[VT>>4] >> ((2*VT) & 31)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((ValueTypeActions[VT>>4] >> ((2*VT) & 31)) & 3) : + Expand; } void setTypeAction(MVT::ValueType VT, LegalizeAction Action) { + assert(!MVT::isExtendedValueType(VT)); assert(unsigned(VT >> 4) < sizeof(ValueTypeActions)/sizeof(ValueTypeActions[0])); ValueTypeActions[VT>>4] |= Action << ((VT*2) & 31); @@ -175,6 +181,10 @@ public: /// to get to the smaller register. For illegal floating point types, this /// returns the integer type to transform to. MVT::ValueType getTypeToTransformTo(MVT::ValueType VT) const { + if (MVT::isExtendedValueType(VT)) + return MVT::getVectorType(MVT::getVectorElementType(VT), + MVT::getVectorNumElements(VT) / 2); + return TransformToType[VT]; } @@ -183,12 +193,13 @@ public: /// that are larger than the largest integer register or illegal floating /// point types), this returns the largest legal type it will be expanded to. MVT::ValueType getTypeToExpandTo(MVT::ValueType VT) const { + assert(!MVT::isExtendedValueType(VT)); while (true) { switch (getTypeAction(VT)) { case Legal: return VT; case Expand: - VT = TransformToType[VT]; + VT = getTypeToTransformTo(VT); break; default: assert(false && "Type is not legal nor is it to be expanded!"); @@ -199,17 +210,17 @@ public: } /// getVectorTypeBreakdown - Vector types are broken down into some number of - /// legal first class types. For example, <8 x float> maps to 2 MVT::v4f32 + /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. - /// Similarly, <2 x long> turns into 4 MVT::i32 values with both PPC and X86. + /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. /// /// This method returns the number of registers needed, and the VT for each /// register. It also returns the VT of the VectorType elements before they /// are promoted/expanded. /// - unsigned getVectorTypeBreakdown(const VectorType *PTy, - MVT::ValueType &PTyElementVT, - MVT::ValueType &PTyLegalElementVT) const; + unsigned getVectorTypeBreakdown(MVT::ValueType VT, + MVT::ValueType &ElementVT, + MVT::ValueType &LegalElementVT) const; typedef std::vector<double>::const_iterator legal_fpimm_iterator; legal_fpimm_iterator legal_fpimm_begin() const { @@ -242,7 +253,9 @@ public: /// expanded to some other code sequence, or the target has a custom expander /// for it. LegalizeAction getOperationAction(unsigned Op, MVT::ValueType VT) const { - return (LegalizeAction)((OpActions[Op] >> (2*VT)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((OpActions[Op] >> (2*VT)) & 3) : + Expand; } /// isOperationLegal - Return true if the specified operation is legal on this @@ -257,7 +270,9 @@ public: /// expanded to some other code sequence, or the target has a custom expander /// for it. LegalizeAction getLoadXAction(unsigned LType, MVT::ValueType VT) const { - return (LegalizeAction)((LoadXActions[LType] >> (2*VT)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((LoadXActions[LType] >> (2*VT)) & 3) : + Expand; } /// isLoadXLegal - Return true if the specified load with extension is legal @@ -272,7 +287,9 @@ public: /// expanded to some other code sequence, or the target has a custom expander /// for it. LegalizeAction getStoreXAction(MVT::ValueType VT) const { - return (LegalizeAction)((StoreXActions >> (2*VT)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((StoreXActions >> (2*VT)) & 3) : + Expand; } /// isStoreXLegal - Return true if the specified store with truncation is @@ -287,7 +304,9 @@ public: /// for it. LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT::ValueType VT) const { - return (LegalizeAction)((IndexedModeActions[0][IdxMode] >> (2*VT)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((IndexedModeActions[0][IdxMode] >> (2*VT)) & 3) : + Expand; } /// isIndexedLoadLegal - Return true if the specified indexed load is legal @@ -303,7 +322,9 @@ public: /// for it. LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT::ValueType VT) const { - return (LegalizeAction)((IndexedModeActions[1][IdxMode] >> (2*VT)) & 3); + return !MVT::isExtendedValueType(VT) ? + (LegalizeAction)((IndexedModeActions[1][IdxMode] >> (2*VT)) & 3) : + Expand; } /// isIndexedStoreLegal - Return true if the specified indexed load is legal @@ -352,7 +373,11 @@ public: /// registers, but may be more than one for types (like i64) that are split /// into pieces. unsigned getNumRegisters(MVT::ValueType VT) const { - return NumRegistersForVT[VT]; + if (!MVT::isExtendedValueType(VT)) + return NumRegistersForVT[VT]; + + MVT::ValueType VT1, VT2; + return getVectorTypeBreakdown(VT, VT1, VT2); } /// hasTargetDAGCombine - If true, the target has custom DAG combine @@ -648,6 +673,7 @@ protected: /// regclass for the specified value type. This indicates the selector can /// handle values of that class natively. void addRegisterClass(MVT::ValueType VT, TargetRegisterClass *RC) { + assert(!MVT::isExtendedValueType(VT)); AvailableRegClasses.push_back(std::make_pair(VT, RC)); RegClassForVT[VT] = RC; } diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 32ab5d323e..cb7276e023 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -227,7 +227,7 @@ namespace { SDOperand visitAND(SDNode *N); SDOperand visitOR(SDNode *N); SDOperand visitXOR(SDNode *N); - SDOperand visitVBinOp(SDNode *N, ISD::NodeType IntOp, ISD::NodeType FPOp); + SDOperand SimplifyVBinOp(SDNode *N); SDOperand visitSHL(SDNode *N); SDOperand visitSRA(SDNode *N); SDOperand visitSRL(SDNode *N); @@ -243,7 +243,6 @@ namespace { SDOperand visitSIGN_EXTEND_INREG(SDNode *N); SDOperand visitTRUNCATE(SDNode *N); SDOperand visitBIT_CONVERT(SDNode *N); - SDOperand visitVBIT_CONVERT(SDNode *N); SDOperand visitFADD(SDNode *N); SDOperand visitFSUB(SDNode *N); SDOperand visitFMUL(SDNode *N); @@ -264,10 +263,9 @@ namespace { SDOperand visitLOAD(SDNode *N); SDOperand visitSTORE(SDNode *N); SDOperand visitINSERT_VECTOR_ELT(SDNode *N); - SDOperand visitVINSERT_VECTOR_ELT(SDNode *N); - SDOperand visitVBUILD_VECTOR(SDNode *N); + SDOperand visitBUILD_VECTOR(SDNode *N); + SDOperand visitCONCAT_VECTORS(SDNode *N); SDOperand visitVECTOR_SHUFFLE(SDNode *N); - SDOperand visitVVECTOR_SHUFFLE(SDNode *N); SDOperand XformToShuffleWithZero(SDNode *N); SDOperand ReassociateOps(unsigned Opc, SDOperand LHS, SDOperand RHS); @@ -280,7 +278,7 @@ namespace { bool NotExtCompare = false); SDOperand SimplifySetCC(MVT::ValueType VT, SDOperand N0, SDOperand N1, ISD::CondCode Cond, bool foldBooleans = true); - SDOperand ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *, MVT::ValueType); + SDOperand ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *, MVT::ValueType); SDOperand BuildSDIV(SDNode *N); SDOperand BuildUDIV(SDNode *N); SDNode *MatchRotate(SDOperand LHS, SDOperand RHS); @@ -657,7 +655,6 @@ SDOperand DAGCombiner::visit(SDNode *N) { case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); case ISD::TRUNCATE: return visitTRUNCATE(N); case ISD::BIT_CONVERT: return visitBIT_CONVERT(N); - case ISD::VBIT_CONVERT: return visitVBIT_CONVERT(N); case ISD::FADD: return visitFADD(N); case ISD::FSUB: return visitFSUB(N); case ISD::FMUL: return visitFMUL(N); @@ -678,18 +675,9 @@ SDOperand DAGCombiner::visit(SDNode *N) { case ISD::LOAD: return visitLOAD(N); case ISD::STORE: return visitSTORE(N); case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); - case ISD::VINSERT_VECTOR_ELT: return visitVINSERT_VECTOR_ELT(N); - case ISD::VBUILD_VECTOR: return visitVBUILD_VECTOR(N); + case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); + case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); - case ISD::VVECTOR_SHUFFLE: return visitVVECTOR_SHUFFLE(N); - case ISD::VADD: return visitVBinOp(N, ISD::ADD , ISD::FADD); - case ISD::VSUB: return visitVBinOp(N, ISD::SUB , ISD::FSUB); - case ISD::VMUL: return visitVBinOp(N, ISD::MUL , ISD::FMUL); - case ISD::VSDIV: return visitVBinOp(N, ISD::SDIV, ISD::FDIV); - case ISD::VUDIV: return visitVBinOp(N, ISD::UDIV, ISD::UDIV); - case ISD::VAND: return visitVBinOp(N, ISD::AND , ISD::AND); - case ISD::VOR: return visitVBinOp(N, ISD::OR , ISD::OR); - case ISD::VXOR: return visitVBinOp(N, ISD::XOR , ISD::XOR); } return SDOperand(); } @@ -856,6 +844,10 @@ SDOperand DAGCombiner::visitADD(SDNode *N) { ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); MVT::ValueType VT = N0.getValueType(); + + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; // fold (add c1, c2) -> c1+c2 if (N0C && N1C) @@ -928,6 +920,10 @@ SDOperand DAGCombiner::visitADD(SDNode *N) { if (Result.Val) return Result; } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1004,6 +1000,10 @@ SDOperand DAGCombiner::visitSUB(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val); MVT::ValueType VT = N0.getValueType(); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (sub x, x) -> 0 if (N0 == N1) return DAG.getConstant(0, N->getValueType(0)); @@ -1024,6 +1024,10 @@ SDOperand DAGCombiner::visitSUB(SDNode *N) { SDOperand Result = combineSelectAndUse(N, N1, N0, DAG); if (Result.Val) return Result; } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1034,6 +1038,10 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); MVT::ValueType VT = N0.getValueType(); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (mul c1, c2) -> c1*c2 if (N0C && N1C) return DAG.getNode(ISD::MUL, VT, N0, N1); @@ -1098,6 +1106,11 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) { SDOperand RMUL = ReassociateOps(ISD::MUL, N0, N1); if (RMUL.Val != 0) return RMUL; + + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1108,6 +1121,10 @@ SDOperand DAGCombiner::visitSDIV(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (sdiv c1, c2) -> c1/c2 if (N0C && N1C && !N1C->isNullValue()) return DAG.getNode(ISD::SDIV, VT, N0, N1); @@ -1162,6 +1179,11 @@ SDOperand DAGCombiner::visitSDIV(SDNode *N) { SDOperand Op = BuildSDIV(N); if (Op.Val) return Op; } + + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1172,6 +1194,10 @@ SDOperand DAGCombiner::visitUDIV(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (udiv c1, c2) -> c1/c2 if (N0C && N1C && !N1C->isNullValue()) return DAG.getNode(ISD::UDIV, VT, N0, N1); @@ -1198,6 +1224,11 @@ SDOperand DAGCombiner::visitUDIV(SDNode *N) { SDOperand Op = BuildUDIV(N); if (Op.Val) return Op; } + + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1229,6 +1260,10 @@ SDOperand DAGCombiner::visitSREM(SDNode *N) { return Sub; } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1267,6 +1302,10 @@ SDOperand DAGCombiner::visitUREM(SDNode *N) { return Sub; } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1274,6 +1313,7 @@ SDOperand DAGCombiner::visitMULHS(SDNode *N) { SDOperand N0 = N->getOperand(0); SDOperand N1 = N->getOperand(1); ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); + MVT::ValueType VT = N->getValueType(0); // fold (mulhs x, 0) -> 0 if (N1C && N1C->isNullValue()) @@ -1283,6 +1323,10 @@ SDOperand DAGCombiner::visitMULHS(SDNode *N) { return DAG.getNode(ISD::SRA, N0.getValueType(), N0, DAG.getConstant(MVT::getSizeInBits(N0.getValueType())-1, TLI.getShiftAmountTy())); + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1290,6 +1334,7 @@ SDOperand DAGCombiner::visitMULHU(SDNode *N) { SDOperand N0 = N->getOperand(0); SDOperand N1 = N->getOperand(1); ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); + MVT::ValueType VT = N->getValueType(0); // fold (mulhu x, 0) -> 0 if (N1C && N1C->isNullValue()) @@ -1297,6 +1342,10 @@ SDOperand DAGCombiner::visitMULHU(SDNode *N) { // fold (mulhu x, 1) -> 0 if (N1C && N1C->getValue() == 1) return DAG.getConstant(0, N0.getValueType()); + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1336,6 +1385,10 @@ SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { return DAG.getNode(N0.getOpcode(), VT, ORNode, N0.getOperand(1)); } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -1347,6 +1400,10 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); MVT::ValueType VT = N1.getValueType(); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (and c1, c2) -> c1&c2 if (N0C && N1C) return DAG.getNode(ISD::AND, VT, N0, N1); @@ -1528,6 +1585,10 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { MVT::ValueType VT = N1.getValueType(); unsigned OpSizeInBits = MVT::getSizeInBits(VT); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (or c1, c2) -> c1|c2 if (N0C && N1C) return DAG.getNode(ISD::OR, VT, N0, N1); @@ -1807,6 +1868,10 @@ SDOperand DAGCombiner::visitXOR(SDNode *N) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); MVT::ValueType VT = N0.getValueType(); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (xor c1, c2) -> c1^c2 if (N0C && N1C) return DAG.getNode(ISD::XOR, VT, N0, N1); @@ -2742,6 +2807,30 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { SDOperand N0 = N->getOperand(0); MVT::ValueType VT = N->getValueType(0); + // If the input is a BUILD_VECTOR with all constant elements, fold this now. + // Only do this before legalize, since afterward the target may be depending + // on the bitconvert. + // First check to see if this is all constant. + if (!AfterLegalize && + N0.getOpcode() == ISD::BUILD_VECTOR && N0.Val->hasOneUse() && + MVT::isVector(VT)) { + bool isSimple = true; + for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) + if (N0.getOperand(i).getOpcode() != ISD::UNDEF && + N0.getOperand(i).getOpcode() != ISD::Constant && + N0.getOperand(i).getOpcode() != ISD::ConstantFP) { + isSimple = false; + break; + } + + MVT::ValueType DestEltVT = MVT::getVectorElementType(N->getValueType(0)); + assert(!MVT::isVector(DestEltVT) && + "Element type of vector ValueType must not be vector!"); + if (isSimple) { + return ConstantFoldBIT_CONVERTofBUILD_VECTOR(N0.Val, DestEltVT); + } + } + // If the input is a constant, let getNode() fold it. if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) { SDOperand Res = DAG.getNode(ISD::BIT_CONVERT, VT, N0); @@ -2774,37 +2863,11 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { return SDOperand(); } -SDOperand DAGCombiner::visitVBIT_CONVERT(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); - - // If the input is a VBUILD_VECTOR with all constant elements, fold this now. - // First check to see if this is all constant. - if (N0.getOpcode() == ISD::VBUILD_VECTOR && N0.Val->hasOneUse() && - VT == MVT::Vector) { - bool isSimple = true; - for (unsigned i = 0, e = N0.getNumOperands()-2; i != e; ++i) - if (N0.getOperand(i).getOpcode() != ISD::UNDEF && - N0.getOperand(i).getOpcode() != ISD::Constant && - N0.getOperand(i).getOpcode() != ISD::ConstantFP) { - isSimple = false; - break; - } - - MVT::ValueType DestEltVT = cast<VTSDNode>(N->getOperand(2))->getVT(); - if (isSimple && !MVT::isVector(DestEltVT)) { - return ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(N0.Val, DestEltVT); - } - } - - return SDOperand(); -} - -/// ConstantFoldVBIT_CONVERTofVBUILD_VECTOR - We know that BV is a vbuild_vector +/// ConstantFoldBIT_CONVERTofBUILD_VECTOR - We know that BV is a build_vector /// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the /// destination element value type. SDOperand DAGCombiner:: -ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { +ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { MVT::ValueType SrcEltVT = BV->getOperand(0).getValueType(); // If this is already the right type, we're done. @@ -2817,13 +2880,14 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { // type, convert each element. This handles FP<->INT cases. if (SrcBitSize == DstBitSize) { SmallVector<SDOperand, 8> Ops; - for (unsigned i = 0, e = BV->getNumOperands()-2; i != e; ++i) { + for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, DstEltVT, BV->getOperand(i))); AddToWorkList(Ops.back().Val); } - Ops.push_back(*(BV->op_end()-2)); // Add num elements. - Ops.push_back(DAG.getValueType(DstEltVT)); - return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); + MVT::ValueType VT = + MVT::getVectorType(DstEltVT, + MVT::getVectorNumElements(BV->getValueType(0))); + return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } // Otherwise, we're growing or shrinking the elements. To avoid having to @@ -2834,7 +2898,7 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { // same sizes. assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!"); MVT::ValueType IntVT = SrcEltVT == MVT::f32 ? MVT::i32 : MVT::i64; - BV = ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(BV, IntVT).Val; + BV = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, IntVT).Val; SrcEltVT = IntVT; } @@ -2843,10 +2907,10 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { if (MVT::isFloatingPoint(DstEltVT)) { assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!"); MVT::ValueType TmpVT = DstEltVT == MVT::f32 ? MVT::i32 : MVT::i64; - SDNode *Tmp = ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(BV, TmpVT).Val; + SDNode *Tmp = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, TmpVT).Val; // Next, convert to FP elements of the same size. - return ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(Tmp, DstEltVT); + return ConstantFoldBIT_CONVERTofBUILD_VECTOR(Tmp, DstEltVT); } // Okay, we know the src/dst types are both integers of differing types. @@ -2856,7 +2920,7 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { unsigned NumInputsPerOutput = DstBitSize/SrcBitSize; SmallVector<SDOperand, 8> Ops; - for (unsigned i = 0, e = BV->getNumOperands()-2; i != e; + for (unsigned i = 0, e = BV->getNumOperands(); i != e; i += NumInputsPerOutput) { bool isLE = TLI.isLittleEndian(); uint64_t NewBits = 0; @@ -2877,16 +2941,16 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { Ops.push_back(DAG.getConstant(NewBits, DstEltVT)); } - Ops.push_back(DAG.getConstant(Ops.size(), MVT::i32)); // Add num elements. - Ops.push_back(DAG.getValueType(DstEltVT)); // Add element size. - return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); + MVT::ValueType VT = MVT::getVectorType(DstEltVT, + Ops.size()); + return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } // Finally, this must be the case where we are shrinking elements: each input // turns into multiple outputs. unsigned NumOutputsPerInput = SrcBitSize/DstBitSize; SmallVector<SDOperand, 8> Ops; - for (unsigned i = 0, e = BV->getNumOperands()-2; i != e; ++i) { + for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { if (BV->getOperand(i).getOpcode() == ISD::UNDEF) { for (unsigned j = 0; j != NumOutputsPerInput; ++j) Ops.push_back(DAG.getNode(ISD::UNDEF, DstEltVT)); @@ -2904,9 +2968,8 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { if (!TLI.isLittleEndian()) std::reverse(Ops.end()-NumOutputsPerInput, Ops.end()); } - Ops.push_back(DAG.getConstant(Ops.size(), MVT::i32)); // Add num elements. - Ops.push_back(DAG.getValueType(DstEltVT)); // Add element size. - return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); + MVT::ValueType VT = MVT::getVectorType(DstEltVT, Ops.size()); + return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } @@ -2918,6 +2981,10 @@ SDOperand DAGCombiner::visitFADD(SDNode *N) { ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (fadd c1, c2) -> c1+c2 if (N0CFP && N1CFP) return DAG.getNode(ISD::FADD, VT, N0, N1); @@ -2937,6 +3004,10 @@ SDOperand DAGCombiner::visitFADD(SDNode *N) { return DAG.getNode(ISD::FADD, VT, N0.getOperand(0), DAG.getNode(ISD::FADD, VT, N0.getOperand(1), N1)); + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -2947,6 +3018,10 @@ SDOperand DAGCombiner::visitFSUB(SDNode *N) { ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (fsub c1, c2) -> c1-c2 if (N0CFP && N1CFP) return DAG.getNode(ISD::FSUB, VT, N0, N1); @@ -2954,6 +3029,10 @@ SDOperand DAGCombiner::visitFSUB(SDNode *N) { if (isNegatibleForFree(N1)) return DAG.getNode(ISD::FADD, VT, N0, GetNegatedExpression(N1, DAG)); + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -2964,6 +3043,10 @@ SDOperand DAGCombiner::visitFMUL(SDNode *N) { ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (fmul c1, c2) -> c1*c2 if (N0CFP && N1CFP) return DAG.getNode(ISD::FMUL, VT, N0, N1); @@ -2994,6 +3077,10 @@ SDOperand DAGCombiner::visitFMUL(SDNode *N) { return DAG.getNode(ISD::FMUL, VT, N0.getOperand(0), DAG.getNode(ISD::FMUL, VT, N0.getOperand(1), N1)); + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -3004,6 +3091,10 @@ SDOperand DAGCombiner::visitFDIV(SDNode *N) { ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); MVT::ValueType VT = N->getValueType(0); + // fold vector ops + SDOperand FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.Val) return FoldedVOp; + // fold (fdiv c1, c2) -> c1/c2 if (N0CFP && N1CFP) return DAG.getNode(ISD::FDIV, VT, N0, N1); @@ -3020,6 +3111,10 @@ SDOperand DAGCombiner::visitFDIV(SDNode *N) { } } + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -3033,6 +3128,11 @@ SDOperand DAGCombiner::visitFREM(SDNode *N) { // fold (frem c1, c2) -> fmod(c1,c2) if (N0CFP && N1CFP) return DAG.getNode(ISD::FREM, VT, N0, N1); + + // If either operand is undef, the result is undef + if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, VT); + return SDOperand(); } @@ -3735,53 +3835,32 @@ SDOperand DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { return SDOperand(); } -SDOperand DAGCombiner::visitVINSERT_VECTOR_ELT(SDNode *N) { - SDOperand InVec = N->getOperand(0); - SDOperand InVal = N->getOperand(1); - SDOperand EltNo = N->getOperand(2); - SDOperand NumElts = N->getOperand(3); - SDOperand EltType = N->getOperand(4); - - // If the invec is a VBUILD_VECTOR and if EltNo is a constant, build a new - // vector with the inserted element. - if (InVec.getOpcode() == ISD::VBUILD_VECTOR && isa<ConstantSDNode>(EltNo)) { - unsigned Elt = cast<ConstantSDNode>(EltNo)->getValue(); - SmallVector<SDOperand, 8> Ops(InVec.Val->op_begin(), InVec.Val->op_end()); - if (Elt < Ops.size()-2) - Ops[Elt] = InVal; - return DAG.getNode(ISD::VBUILD_VECTOR, InVec.getValueType(), - &Ops[0], Ops.size()); - } - - return SDOperand(); -} - -SDOperand DAGCombiner::visitVBUILD_VECTOR(SDNode *N) { - unsigned NumInScalars = N->getNumOperands()-2; - SDOperand NumElts = N->getOperand(NumInScalars); - SDOperand EltType = N->getOperand(NumInScalars+1); +SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) { + unsigned NumInScalars = N->getNumOperands(); + MVT::ValueType VT = N->getValueType(0); + unsigned NumElts = MVT::getVectorNumElements(VT); + MVT::ValueType EltType = MVT::getVectorElementType(VT); - // Check to see if this is a VBUILD_VECTOR of a bunch of VEXTRACT_VECTOR_ELT - // operations. If so, and if the EXTRACT_ELT vector inputs come from at most - // two distinct vectors, turn this into a shuffle node. + // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT + // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from + // at most two distinct vectors, turn this into a shuffle node. SDOperand VecIn1, VecIn2; for (unsigned i = 0; i != NumInScalars; ++i) { // Ignore undef inputs. if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; - // If this input is something other than a VEXTRACT_VECTOR_ELT with a + // If this input is something other than a EXTRACT_VECTOR_ELT with a // constant index, bail out. - if (N->getOperand(i).getOpcode() != ISD::VEXTRACT_VECTOR_ELT || + if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT || !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) { VecIn1 = VecIn2 = SDOperand(0, 0); break; } - // If the input vector type disagrees with the result of the vbuild_vector, + // If the input vector type disagrees with the result of the build_vector, // we can't make a shuffle. SDOperand ExtractedFromVec = N->getOperand(i).getOperand(0); - if (*(ExtractedFromVec.Val->op_end()-2) != NumElts || - *(ExtractedFromVec.Val->op_end()-1) != EltType) { + if (ExtractedFromVec.getValueType() != VT) { VecIn1 = VecIn2 = SDOperand(0, 0); break; } @@ -3825,158 +3904,49 @@ SDOperand DAGCombiner::visitVBUILD_VECTOR(SDNode *N) { } // Add count and size info. - BuildVecIndices.push_back(NumElts); - BuildVecIndices.push_back(DAG.getValueType(TLI.getPointerTy())); + MVT::ValueType BuildVecVT = + MVT::getVectorType(TLI.getPointerTy(), NumElts); - // Return the new VVECTOR_SHUFFLE node. + // Return the new VECTOR_SHUFFLE node. SDOperand Ops[5]; Ops[0] = VecIn1; if (VecIn2.Val) { Ops[1] = VecIn2; } else { - // Use an undef vbuild_vector as input for the second operand. + // Use an undef build_vector as input for the second operand. std::vector<SDOperand> UnOps(NumInScalars, DAG.getNode(ISD::UNDEF, - cast<VTSDNode>(EltType)->getVT())); - UnOps.push_back(NumElts); - UnOps.push_back(EltType); - Ops[1] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + EltType)); + Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, VT, &UnOps[0], UnOps.size()); AddToWorkList(Ops[1].Val); } - Ops[2] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, BuildVecVT, &BuildVecIndices[0], BuildVecIndices.size()); - Ops[3] = NumElts; - Ops[4] = EltType; - return DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, Ops, 5); + return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Ops, 3); } return SDOperand(); } -SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { - SDOperand ShufMask = N->getOperand(2); - unsigned NumElts = ShufMask.getNumOperands(); +SDOperand DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { + // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of + // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector + // inputs come from at most two distinct vectors, turn this into a shuffle + // node. - // If the shuffle mask is an identity operation on the LHS, return the LHS. - bool isIdentity = true; - for (unsigned i = 0; i != NumElts; ++i) { - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i) { - isIdentity = false; - break; - } - } - if (isIdentity) return N->getOperand(0); - - // If the shuffle mask is an identity operation on the RHS, return the RHS. - isIdentity = true; - for (unsigned i = 0; i != NumElts; ++i) { - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i+NumElts) { - isIdentity = false; - break; - } - } - if (isIdentity) return N->getOperand(1); - - // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not - // needed at all. - bool isUnary = true; - bool isSplat = true; - int VecNum = -1; - unsigned BaseIdx = 0; - for (unsigned i = 0; i != NumElts; ++i) - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) { - unsigned Idx = cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue(); - int V = (Idx < NumElts) ? 0 : 1; - if (VecNum == -1) { - VecNum = V; - BaseIdx = Idx; - } else { - if (BaseIdx != Idx) - isSplat = false; - if (VecNum != V) { - isUnary = false; - break; - } - } - } - - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - // Normalize unary shuffle so the RHS is undef. - if (isUnary && VecNum == 1) - std::swap(N0, N1); - - // If it is a splat, check if the argument vector is a build_vector with - // all scalar elements the same. - if (isSplat) { - SDNode *V = N0.Val; - if (V->getOpcode() == ISD::BIT_CONVERT) - V = V->getOperand(0).Val; - if (V->getOpcode() == ISD::BUILD_VECTOR) { - unsigned NumElems = V->getNumOperands()-2; - if (NumElems > BaseIdx) { - SDOperand Base; - bool AllSame = true; - for (unsigned i = 0; i != NumElems; ++i) { - if (V->getOperand(i).getOpcode() != ISD::UNDEF) { - Base = V->getOperand(i); - break; - } - } - // Splat of <u, u, u, u>, return <u, u, u, u> - if (!Base.Val) - return N0; - for (unsigned i = 0; i != NumElems; ++i) { - if (V->getOperand(i).getOpcode() != ISD::UNDEF && - V->getOperand(i) != Base) { - AllSame = false; - break; - } - } - // Splat of <x, x, x, x>, return <x, x, x, x> - if (AllSame) - return N0; - } - } + // If we only have one input vector, we don't need to do any concatenation. + if (N->getNumOperands() == 1) { + return N->getOperand(0); } - // If it is a unary or the LHS and the RHS are the same node, turn the RHS - // into an undef. - if (isUnary || N0 == N1) { - if (N0.getOpcode() == ISD::UNDEF) - return DAG.getNode(ISD::UNDEF, N->getValueType(0)); - // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the - // first operand. - SmallVector<SDOperand, 8> MappedOps; - for (unsigned i = 0, e = ShufMask.getNumOperands(); i != e; ++i) { - if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF || - cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() < NumElts) { - MappedOps.push_back(ShufMask.getOperand(i)); - } else { - unsigned NewIdx = - cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() - NumElts; - MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32)); - } - } - ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(), - &MappedOps[0], MappedOps.size()); - AddToWorkList(ShufMask.Val); - return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0), - N0, - DAG.getNode(ISD::UNDEF, N->getValueType(0)), - ShufMask); - } - return SDOperand(); } -SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) { +SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { SDOperand ShufMask = N->getOperand(2); - unsigned NumElts = ShufMask.getNumOperands()-2; - + unsigned NumElts = ShufMask.getNumOperands(); + // If the shuffle mask is an identity operation on the LHS, return the LHS. bool isIdentity = true; for (unsigned i = 0; i != NumElts; ++i) { @@ -3987,7 +3957,7 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) { } } if (isIdentity) return N->getOperand(0); - + // If the shuffle mask is an identity operation on the RHS, return the RHS. isIdentity = true; for (unsigned i = 0; i != NumElts; ++i) { @@ -4033,19 +4003,17 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) { if (isSplat) { SDNode *V = N0.Val; - // If this is a vbit convert that changes the element type of the vector but + // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to // look though conversions that change things like v4f32 to v2f64. - if (V->getOpcode() == ISD::VBIT_CONVERT) { + if (V->getOpcode() == ISD::BIT_CONVERT) { SDOperand ConvInput = V->getOperand(0); - if (ConvInput.getValueType() == MVT::Vector && - NumElts == - ConvInput.getConstantOperandVal(ConvInput.getNumOperands()-2)) + if (MVT::getVectorNumElements(ConvInput.getValueType()) == NumElts) V = ConvInput.Val; } - if (V->getOpcode() == ISD::VBUILD_VECTOR) { - unsigned NumElems = V->getNumOperands()-2; + if (V->getOpcode() == ISD::BUILD_VECTOR) { + unsigned NumElems = V->getNumOperands(); if (NumElems > BaseIdx) { SDOperand Base; bool AllSame = true; @@ -4088,48 +4056,33 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) { MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32)); } } - // Add the type/#elts values. - MappedOps.push_back(ShufMask.getOperand(NumElts)); - MappedOps.push_back(ShufMask.getOperand(NumElts+1)); - - ShufMask = DAG.getNode(ISD::VBUILD_VECTOR, ShufMask.getValueType(), + ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(), &MappedOps[0], MappedOps.size()); AddToWorkList(ShufMask.Val); - - // Build the undef vector. - SDOperand UDVal = DAG.getNode(ISD::UNDEF, MappedOps[0].getValueType()); - for (unsigned i = 0; i != NumElts; ++i) - MappedOps[i] = UDVal; - MappedOps[NumElts ] = *(N0.Val->op_end()-2); - MappedOps[NumElts+1] = *(N0.Val->op_end()-1); - UDVal = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, - &MappedOps[0], MappedOps.size()); - - return DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, - N0, UDVal, ShufMask, - MappedOps[NumElts], MappedOps[NumElts+1]); + return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0), + N0, + DAG.getNode(ISD::UNDEF, N->getValueType(0)), + ShufMask); } - + return SDOperand(); } /// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform -/// a VAND to a vector_shuffle with the destination vector and a zero vector. -/// e.g. VAND V, <0xffffffff, 0, 0xffffffff, 0>. ==> +/// an AND to a vector_shuffle with the destination vector and a zero vector. +/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> /// vector_shuffle V, Zero, <0, 4, 2, 4> SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) { SDOperand LHS = N->getOperand(0); SDOperand RHS = N->getOperand(1); - if (N->getOpcode() == ISD::VAND) { - SDOperand DstVecSize = *(LHS.Val->op_end()-2); - SDOperand DstVecEVT = *(LHS.Val->op_end()-1); - if (RHS.getOpcode() == ISD::VBIT_CONVERT) + if (N->getOpcode() == ISD::AND) { + if (RHS.getOpcode() == ISD::BIT_CONVERT) RHS = RHS.getOperand(0); - if (RHS.getOpcode() == ISD::VBUILD_VECTOR) { + if (RHS.getOpcode() == ISD::BUILD_VECTOR) { std::vector<SDOperand> IdxOps; unsigned NumOps = RHS.getNumOperands(); - unsigned NumElts = NumOps-2; - MVT::ValueType EVT = cast<VTSDNode>(RHS.getOperand(NumOps-1))->getVT(); + unsigned NumElts = NumOps; + MVT::ValueType EVT = MVT::getVectorElementType(RHS.getValueType()); for (unsigned i = 0; i != NumElts; ++i) { SDOperand Elt = RHS.getOperand(i); if (!isa<ConstantSDNode>(Elt)) @@ -4146,30 +4099,21 @@ SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) { if (!TLI.isVectorClearMaskLegal(IdxOps, EVT, DAG)) return SDOperand(); - // Return the new VVECTOR_SHUFFLE node. - SDOperand NumEltsNode = DAG.getConstant(NumElts, MVT::i32); - SDOperand EVTNode = DAG.getValueType(EVT); + // Return the new VECTOR_SHUFFLE node. + MVT::ValueType VT = MVT::getVectorType(EVT, NumElts); std::vector<SDOperand> Ops; - LHS = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, LHS, NumEltsNode, - EVTNode); + LHS = DAG.getNode(ISD::BIT_CONVERT, VT, LHS); Ops.push_back(LHS); AddToWorkList(LHS.Val); std::vector<SDOperand> ZeroOps(NumElts, DAG.getConstant(0, EVT)); - ZeroOps.push_back(NumEltsNode); - ZeroOps.push_back(EVTNode); - Ops.push_back(DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT, &ZeroOps[0], ZeroOps.size())); - IdxOps.push_back(NumEltsNode); - IdxOps.push_back(EVTNode); - Ops.push_back(DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT, &IdxOps[0], IdxOps.size())); - Ops.push_back(NumEltsNode); - Ops.push_back(EVTNode); - SDOperand Result = DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, + SDOperand Result = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, &Ops[0], Ops.size()); - if (NumEltsNode != DstVecSize || EVTNode != DstVecEVT) { - Result = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Result, - DstVecSize, DstVecEVT); + if (VT != LHS.getValueType()) { + Result = DAG.getNode(ISD::BIT_CONVERT, LHS.getValueType(), Result); } return Result; } @@ -4177,24 +4121,28 @@ SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) { return SDOperand(); } -/// visitVBinOp - Visit a binary vector operation, like VADD. IntOp indicates -/// the scalar operation of the vop if it is operating on an integer vector -/// (e.g. ADD) and FPOp indicates the FP version (e.g. FADD). -SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp, - ISD::NodeType FPOp) { - MVT::ValueType EltType = cast<VTSDNode>(*(N->op_end()-1))->getVT(); - ISD::NodeType ScalarOp = MVT::isInteger(EltType) ? IntOp : FPOp; +/// SimplifyVBinOp - Visit a binary vector operation, like ADD. +SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) { + // After legalize, the target may be depending on adds and other + // binary ops to provide legal ways to construct constants or other + // things. Simplifying them may result in a loss of legality. + if (AfterLegalize) return SDOperand(); + + MVT::ValueType VT = N->getValueType(0); + if (!MVT::isVector(VT)) return SDOperand(); + + MVT::ValueType EltType = MVT::getVectorElementType(VT); SDOperand LHS = N->getOperand(0); SDOperand RHS = N->getOperand(1); SDOperand Shuffle = XformToShuffleWithZero(N); if (Shuffle.Val) return Shuffle; - // If the LHS and RHS are VBUILD_VECTOR nodes, see if we can constant fold + // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold // this operation. - if (LHS.getOpcode() == ISD::VBUILD_VECTOR && - RHS.getOpcode() == ISD::VBUILD_VECTOR) { + if (LHS.getOpcode() == ISD::BUILD_VECTOR && + RHS.getOpcode() == ISD::BUILD_VECTOR) { SmallVector<SDOperand, 8> Ops; - for (unsigned i = 0, e = LHS.getNumOperands()-2; i != e; ++i) { + for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { SDOperand LHSOp = LHS.getOperand(i); SDOperand RHSOp = RHS.getOperand(i); // If these two elements can't be folded, bail out. @@ -4206,14 +4154,15 @@ SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp, RHSOp.getOpcode() != ISD::ConstantFP)) break; // Can't fold divide by zero. - if (N->getOpcode() == ISD::VSDIV || N->getOpcode() == ISD::VUDIV) { + if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV || + N->getOpcode() == ISD::FDIV) { if ((RHSOp.getOpcode() == ISD::Constant && cast<ConstantSDNode>(RHSOp.Val)->isNullValue()) || (RHSOp.getOpcode() == ISD::ConstantFP && !cast<ConstantFPSDNode>(RHSOp.Val)->getValue())) break; } - Ops.push_back(DAG.getNode(ScalarOp, EltType, LHSOp, RHSOp)); + Ops.push_back(DAG.getNode(N->getOpcode(), EltType, LHSOp, RHSOp)); AddToWorkList(Ops.back().Val); assert((Ops.back().getOpcode() == ISD::UNDEF || Ops.back().getOpcode() == ISD::Constant || @@ -4221,10 +4170,9 @@ SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp, "Scalar binop didn't fold!"); } - if (Ops.size() == LHS.getNumOperands()-2) { - Ops.push_back(*(LHS.Val->op_end()-2)); - Ops.push_back(*(LHS.Val->op_end()-1)); - return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); + if (Ops.size() == LHS.getNumOperands()) { + MVT::ValueType VT = LHS.getValueType(); + return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } } diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 57357442fb..5f41452055 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -111,10 +111,10 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// to avoid splitting the same node more than once. std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes; - /// PackedNodes - For nodes that need to be packed from MVT::Vector types to - /// concrete vector types, this contains the mapping of ones we have already + /// ScalarizedNodes - For nodes that need to be converted from vector types to + /// scalar types, this contains the mapping of ones we have already /// processed to the result. - std::map<SDOperand, SDOperand> PackedNodes; + std::map<SDOperand, SDOperand> ScalarizedNodes; void AddLegalizedOperand(SDOperand From, SDOperand To) { LegalizedNodes.insert(std::make_pair(From, To)); @@ -149,7 +149,7 @@ public: void LegalizeDAG(); private: - /// HandleOp - Legalize, Promote, Expand or Pack the specified operand as + /// HandleOp - Legalize, Promote, or Expand the specified operand as /// appropriate for its type. void HandleOp(SDOperand Op); @@ -173,15 +173,13 @@ private: /// types. void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi); - /// SplitVectorOp - Given an operand of MVT::Vector type, break it down into - /// two smaller values of MVT::Vector type. + /// SplitVectorOp - Given an operand of vector type, break it down into + /// two smaller values. void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi); - /// PackVectorOp - Given an operand of MVT::Vector type, convert it into the - /// equivalent operation that returns a packed value (e.g. MVT::V4F32). When - /// this is called, we know that PackedVT is the right type for the result and - /// we know that this type is legal for the target. - SDOperand PackVectorOp(SDOperand O, MVT::ValueType PackedVT); + /// ScalarizeVectorOp - Given an operand of vector type, convert it into the + /// equivalent operation that returns a scalar value. + SDOperand ScalarizeVectorOp(SDOperand O); /// isShuffleLegal - Return true if a vector shuffle is legal with the /// specified mask and type. Targets can specify exactly which masks they @@ -224,8 +222,7 @@ private: void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt, SDOperand &Lo, SDOperand &Hi); - SDOperand LowerVEXTRACT_VECTOR_ELT(SDOperand Op); - SDOperand LowerVEXTRACT_SUBVECTOR(SDOperand Op); + SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op); SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op); SDOperand getIntPtrConstant(uint64_t Val) { @@ -279,22 +276,6 @@ SDNode *SelectionDAGLegalize::isShuffleLegal(MVT::ValueType VT, return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0; } -/// getScalarizedOpcode - Return the scalar opcode that corresponds to the -/// specified vector opcode. -static unsigned getScalarizedOpcode(unsigned VecOp, MVT::ValueType VT) { - switch (VecOp) { - default: assert(0 && "Don't know how to scalarize this opcode!"); - case ISD::VADD: return MVT::isInteger(VT) ? ISD::ADD : ISD::FADD; - case ISD::VSUB: return MVT::isInteger(VT) ? ISD::SUB : ISD::FSUB; - case ISD::VMUL: return MVT::isInteger(VT) ? ISD::MUL : ISD::FMUL; - case ISD::VSDIV: return MVT::isInteger(VT) ? ISD::SDIV: ISD::FDIV; - case ISD::VUDIV: return MVT::isInteger(VT) ? ISD::UDIV: ISD::FDIV; - case ISD::VAND: return MVT::isInteger(VT) ? ISD::AND : 0; - case ISD::VOR: return MVT::isInteger(VT) ? ISD::OR : 0; - case ISD::VXOR: return MVT::isInteger(VT) ? ISD::XOR : 0; - } -} - SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) : TLI(dag.getTargetLoweringInfo()), DAG(dag), ValueTypeActions(TLI.getValueTypeActions()) { @@ -369,7 +350,7 @@ void SelectionDAGLegalize::LegalizeDAG() { LegalizedNodes.clear(); PromotedNodes.clear(); SplitNodes.clear(); - PackedNodes.clear(); + ScalarizedNodes.clear(); // Remove dead nodes now. DAG.RemoveDeadNodes(); @@ -473,38 +454,29 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, return false; } -/// HandleOp - Legalize, Promote, Expand or Pack the specified operand as +/// HandleOp - Legalize, Promote, or Expand the specified operand as /// appropriate for its type. void SelectionDAGLegalize::HandleOp(SDOperand Op) { - switch (getTypeAction(Op.getValueType())) { + MVT::ValueType VT = Op.getValueType(); + switch (getTypeAction(VT)) { default: assert(0 && "Bad type action!"); - case Legal: LegalizeOp(Op); break; - case Promote: PromoteOp(Op); break; + case Legal: (void)LegalizeOp(Op); break; + case Promote: (void)PromoteOp(Op); break; case Expand: - if (Op.getValueType() != MVT::Vector) { + if (!MVT::isVector(VT)) { + // If this is an illegal scalar, expand it into its two component + // pieces. SDOperand X, Y; ExpandOp(Op, X, Y); + } else if (MVT::getVectorNumElements(VT) == 1) { + // If this is an illegal single element vector, convert it to a + // scalar operation. + (void)ScalarizeVectorOp(Op); } else { - SDNode *N = Op.Val; - unsigned NumOps = N->getNumOperands(); - unsigned NumElements = - cast<ConstantSDNode>(N->getOperand(NumOps-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(NumOps-1))->getVT(); - MVT::ValueType PackedVT = MVT::getVectorType(EVT, NumElements); - if (PackedVT != MVT::Other && TLI.isTypeLegal(PackedVT)) { - // In the common case, this is a legal vector type, convert it to the - // packed operation and type now. - PackVectorOp(Op, PackedVT); - } else if (NumElements == 1) { - // Otherwise, if this is a single element vector, convert it to a - // scalar operation. - PackVectorOp(Op, EVT); - } else { - // Otherwise, this is a multiple element vector that isn't supported. - // Split it in half and legalize both parts. - SDOperand X, Y; - SplitVectorOp(Op, X, Y); - } + // Otherwise, this is an illegal multiple element vector. + // Split it in half and legalize both parts. + SDOperand X, Y; + SplitVectorOp(Op, X, Y); } break; } @@ -556,6 +528,8 @@ SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT::ValueType NVT, SelectionDAG &DAG, TargetLowering &TLI) { MVT::ValueType VT = Node->getValueType(0); MVT::ValueType SrcVT = Node->getOperand(1).getValueType(); + assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && + "fcopysign expansion only supported for f32 and f64"); MVT::ValueType SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; // First get the sign bit of second operand. @@ -1158,34 +1132,17 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case ISD::EXTRACT_VECTOR_ELT: - Tmp1 = LegalizeOp(Node->getOperand(0)); + Tmp1 = Node->getOperand(0); Tmp2 = LegalizeOp(Node->getOperand(1)); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); - - switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, - Tmp1.getValueType())) { - default: assert(0 && "This action is not supported yet!"); - case TargetLowering::Legal: - break; - case TargetLowering::Custom: - Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { - Result = Tmp3; - break; - } - // FALLTHROUGH - case TargetLowering::Expand: - Result = ExpandEXTRACT_VECTOR_ELT(Result); - break; - } + Result = ExpandEXTRACT_VECTOR_ELT(Result); break; - case ISD::VEXTRACT_VECTOR_ELT: - Result = LegalizeOp(LowerVEXTRACT_VECTOR_ELT(Op)); - break; - - case ISD::VEXTRACT_SUBVECTOR: - Result = LegalizeOp(LowerVEXTRACT_SUBVECTOR(Op)); + case ISD::EXTRACT_SUBVECTOR: + Tmp1 = Node->getOperand(0); + Tmp2 = LegalizeOp(Node->getOperand(1)); + Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); + Result = ExpandEXTRACT_SUBVECTOR(Result); break; case ISD::CALLSEQ_START: { @@ -1688,7 +1645,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3); break; case Expand: - if (Tmp2.getValueType() != MVT::Vector) { + if (!MVT::isVector(Tmp2.getValueType())) { SDOperand Lo, Hi; ExpandOp(Tmp2, Lo, Hi); @@ -1703,20 +1660,20 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = LegalizeOp(Result); } else { SDNode *InVal = Tmp2.Val; - unsigned NumElems = - cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); + unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0)); + MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0)); - // Figure out if there is a Packed type corresponding to this Vector + // Figure out if there is a simple type corresponding to this Vector // type. If so, convert to the vector type. MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); - if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) { + if (TLI.isTypeLegal(TVT)) { // Turn this into a return of the vector type. - Tmp2 = PackVectorOp(Tmp2, TVT); + Tmp2 = LegalizeOp(Tmp2); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); } else if (NumElems == 1) { // Turn this into a return of the scalar type. - Tmp2 = PackVectorOp(Tmp2, EVT); + Tmp2 = ScalarizeVectorOp(Tmp2); + Tmp2 = LegalizeOp(Tmp2); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); // FIXME: Returns of gcc generic vectors smaller than a legal type @@ -1756,7 +1713,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case Expand: { SDOperand Lo, Hi; - assert(Node->getOperand(i).getValueType() != MVT::Vector && + assert(!MVT::isExtendedValueType(Node->getOperand(i).getValueType())&& "FIXME: TODO: implement returning non-legal vector types!"); ExpandOp(Node->getOperand(i), Lo, Hi); NewValues.push_back(Lo); @@ -1853,27 +1810,30 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // If this is a vector type, then we have to calculate the increment as // the product of the element size in bytes, and the number of elements // in the high half of the vector. - if (ST->getValue().getValueType() == MVT::Vector) { + if (MVT::isVector(ST->getValue().getValueType())) { SDNode *InVal = ST->getValue().Val; - unsigned NumElems = - cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); + unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0)); + MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0)); - // Figure out if there is a Packed type corresponding to this Vector + // Figure out if there is a simple type corresponding to this Vector // type. If so, convert to the vector type. MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); - if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) { + if (TLI.isTypeLegal(TVT)) { // Turn this into a normal store of the vector type. - Tmp3 = PackVectorOp(Node->getOperand(1), TVT); + Tmp3 = LegalizeOp(Node->getOperand(1)); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), - ST->getSrcValueOffset()); + ST->getSrcValueOffset(), + ST->isVolatile(), + ST->getAlignment()); Result = LegalizeOp(Result); break; } else if (NumElems == 1) { // Turn this into a normal store of the scalar type. - Tmp3 = PackVectorOp(Node->getOperand(1), EVT); + Tmp3 = ScalarizeVectorOp(Node->getOperand(1)); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), - ST->getSrcValueOffset()); + ST->getSrcValueOffset(), + ST->isVolatile(), + ST->getAlignment()); // The scalarized value type may not be legal, e.g. it might require // promotion or expansion. Relegalize the scalar store. Result = LegalizeOp(Result); @@ -2864,6 +2824,30 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::BIT_CONVERT: if (!isTypeLegal(Node->getOperand(0).getValueType())) { Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0)); + } else if (MVT::isVector(Op.getOperand(0).getValueType())) { + // The input has to be a vector type, we have to either scalarize it, pack + // it, or convert it based on whether the input vector type is legal. + SDNode *InVal = Node->getOperand(0).Val; + unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0)); + MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0)); + + // Figure out if there is a simple type corresponding to this Vector + // type. If so, convert to the vector type. + MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); + if (TLI.isTypeLegal(TVT)) { + // Turn this into a bit convert of the packed input. + Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), + LegalizeOp(Node->getOperand(0))); + break; + } else if (NumElems == 1) { + // Turn this into a bit convert of the scalar input. + Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), + ScalarizeVectorOp(Node->getOperand(0))); + break; + } else { + // FIXME: UNIMP! Store then reload + assert(0 && "Cast from unsupported vector type not implemented yet!"); + } } else { switch (TLI.getOperationAction(ISD::BIT_CONVERT, Node->getOperand(0).getValueType())) { @@ -2878,35 +2862,6 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } } break; - case ISD::VBIT_CONVERT: { - assert(Op.getOperand(0).getValueType() == MVT::Vector && - "Can only have VBIT_CONVERT where input or output is MVT::Vector!"); - - // The input has to be a vector type, we have to either scalarize it, pack - // it, or convert it based on whether the input vector type is legal. - SDNode *InVal = Node->getOperand(0).Val; - unsigned NumElems = - cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); - if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) { - // Turn this into a bit convert of the packed input. - Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), - PackVectorOp(Node->getOperand(0), TVT)); - break; - } else if (NumElems == 1) { - // Turn this into a bit convert of the scalar input. - Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), - PackVectorOp(Node->getOperand(0), EVT)); - break; - } else { - // FIXME: UNIMP! Store then reload - assert(0 && "Cast from unsupported vector type not implemented yet!"); - } - } // Conversion operators. The source and destination have different types. case ISD::SINT_TO_FP: @@ -3568,11 +3523,8 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { break; } break; - case ISD::VEXTRACT_VECTOR_ELT: - Result = PromoteOp(LowerVEXTRACT_VECTOR_ELT(Op)); - break; - case ISD::VEXTRACT_SUBVECTOR: - Result = PromoteOp(LowerVEXTRACT_SUBVECTOR(Op)); + case ISD::EXTRACT_SUBVECTOR: + Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op)); break; case ISD::EXTRACT_VECTOR_ELT: Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op)); @@ -3589,66 +3541,90 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { return Result; } -/// LowerVEXTRACT_VECTOR_ELT - Lower a VEXTRACT_VECTOR_ELT operation into a -/// EXTRACT_VECTOR_ELT operation, to memory operations, or to scalar code based -/// on the vector type. The return type of this matches the element type of the -/// vector, which may not be legal for the target. -SDOperand SelectionDAGLegalize::LowerVEXTRACT_VECTOR_ELT(SDOperand Op) { +/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into +/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, +/// based on the vector type. The return type of this matches the element type +/// of the vector, which may not be legal for the target. +SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { // We know that operand #0 is the Vec vector. If the index is a constant // or if the invec is a supported hardware type, we can use it. Otherwise, // lower to a store then an indexed load. SDOperand Vec = Op.getOperand(0); - SDOperand Idx = LegalizeOp(Op.getOperand(1)); + SDOperand Idx = Op.getOperand(1); SDNode *InVal = Vec.Val; - unsigned NumElems = cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); + MVT::ValueType TVT = InVal->getValueType(0); + unsigned NumElems = MVT::getVectorNumElements(TVT); - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); - if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) { - // Turn this into a packed extract_vector_elt operation. - Vec = PackVectorOp(Vec, TVT); - return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Op.getValueType(), Vec, Idx); - } else if (NumElems == 1) { + switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) { + default: assert(0 && "This action is not supported yet!"); + case TargetLowering::Custom: { + Vec = LegalizeOp(Vec); + Op = DAG.UpdateNodeOperands(Op, Vec, Idx); + SDOperand Tmp3 = TLI.LowerOperation(Op, DAG); + if (Tmp3.Val) + return Tmp3; + break; + } + case TargetLowering::Legal: + if (isTypeLegal(TVT)) { + Vec = LegalizeOp(Vec); + Op = DAG.UpdateNodeOperands(Op, Vec, Idx); + Op = LegalizeOp(Op); + } + break; + case TargetLowering::Expand: + break; + } + + if (NumElems == 1) { // This must be an access of the only element. Return it. - return PackVectorOp(Vec, EVT); - } else if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { + Op = ScalarizeVectorOp(Vec); + } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) { + ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); SDOperand Lo, Hi; SplitVectorOp(Vec, Lo, Hi); if (CIdx->getValue() < NumElems/2) { Vec = Lo; } else { Vec = Hi; - Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType()); + Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, + Idx.getValueType()); } - + // It's now an extract from the appropriate high or low part. Recurse. Op = DAG.UpdateNodeOperands(Op, Vec, Idx); - return LowerVEXTRACT_VECTOR_ELT(Op); + Op = ExpandEXTRACT_VECTOR_ELT(Op); } else { - // Variable index case for extract element. - // FIXME: IMPLEMENT STORE/LOAD lowering. Need alignment of stack slot!! - assert(0 && "unimp!"); - return SDOperand(); + // Store the value to a temporary stack slot, then LOAD the scalar + // element back out. + SDOperand StackPtr = CreateStackTemporary(Vec.getValueType()); + SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); + + // Add the offset to the index. + unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8; + Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx, + DAG.getConstant(EltSize, Idx.getValueType())); + StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr); + + Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0); } + return Op; } -/// LowerVEXTRACT_SUBVECTOR - Lower a VEXTRACT_SUBVECTOR operation. For now +/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now /// we assume the operation can be split if it is not already legal. -SDOperand SelectionDAGLegalize::LowerVEXTRACT_SUBVECTOR(SDOperand Op) { +SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) { // We know that operand #0 is the Vec vector. For now we assume the index // is a constant and that the extracted result is a supported hardware type. SDOperand Vec = Op.getOperand(0); SDOperand Idx = LegalizeOp(Op.getOperand(1)); - SDNode *InVal = Vec.Val; - unsigned NumElems = cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); + unsigned NumElems = MVT::getVectorNumElements(Vec.getValueType()); if (NumElems == MVT::getVectorNumElements(Op.getValueType())) { // This must be an access of the desired vector length. Return it. - return PackVectorOp(Vec, Op.getValueType()); + return Vec; } ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx); @@ -3663,30 +3639,9 @@ SDOperand SelectionDAGLegalize::LowerVEXTRACT_SUBVECTOR(SDOperand Op) { // It's now an extract from the appropriate high or low part. Recurse. Op = DAG.UpdateNodeOperands(Op, Vec, Idx); - return LowerVEXTRACT_SUBVECTOR(Op); -} - -/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into -/// memory traffic. -SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { - SDOperand Vector = Op.getOperand(0); - SDOperand Idx = Op.getOperand(1); - - // If the target doesn't support this, store the value to a temporary - // stack slot, then LOAD the scalar element back out. - SDOperand StackPtr = CreateStackTemporary(Vector.getValueType()); - SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vector, StackPtr, NULL, 0); - - // Add the offset to the index. - unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8; - Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx, - DAG.getConstant(EltSize, Idx.getValueType())); - StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr); - - return DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0); + return ExpandEXTRACT_SUBVECTOR(Op); } - /// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC /// with condition CC on the current target. This usually involves legalizing /// or promoting the arguments. In the case where LHS and RHS must be expanded, @@ -4748,7 +4703,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ SDNode *Node = Op.Val; assert(getTypeAction(VT) == Expand && "Not an expanded type!"); assert(((MVT::isInteger(NVT) && NVT < VT) || MVT::isFloatingPoint(VT) || - VT == MVT::Vector) && + MVT::isVector(VT)) && "Cannot expand to FP value or to larger int value!"); // See if we already expanded it. @@ -4870,9 +4825,10 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ SDOperand Ch = LD->getChain(); // Legalize the chain. SDOperand Ptr = LD->getBasePtr(); // Legalize the pointer. ISD::LoadExtType ExtType = LD->getExtensionType(); + unsigned SVOffset = LD->getSrcValueOffset(); if (ExtType == ISD::NON_EXTLOAD) { - Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset()); + Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset); if (VT == MVT::f32 || VT == MVT::f64) { // f32->i32 or f64->i64 one to one expansion. // Remember that we legalized the chain. @@ -4887,8 +4843,8 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ unsigned IncrementSize = MVT::getSizeInBits(Lo.getValueType())/8; Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, getIntPtrConstant(IncrementSize)); - // FIXME: This creates a bogus srcvalue! - Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset()); + SVOffset += IncrementSize; + Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset); // Build a factor node to remember that this load is independent of the // other one. @@ -4905,7 +4861,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (VT == MVT::f64 && EVT == MVT::f32) { // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(), - LD->getSrcValueOffset()); + SVOffset); // Remember that we legalized the chain. AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1))); ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi); @@ -4914,10 +4870,10 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (EVT == NVT) Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), - LD->getSrcValueOffset()); + SVOffset); else Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(), - LD->getSrcValueOffset(), EVT); + SVOffset, EVT); // Remember that we legalized the chain. AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1))); @@ -5504,17 +5460,17 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ assert(isNew && "Value already expanded?!?"); } -/// SplitVectorOp - Given an operand of MVT::Vector type, break it down into -/// two smaller values of MVT::Vector type. +/// SplitVectorOp - Given an operand of vector type, break it down into +/// two smaller values, still of vector type. void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi) { - assert(Op.getValueType() == MVT::Vector && "Cannot split non-vector type!"); + assert(MVT::isVector(Op.getValueType()) && "Cannot split non-vector type!"); SDNode *Node = Op.Val; - unsigned NumElements = cast<ConstantSDNode>(*(Node->op_end()-2))->getValue(); + unsigned NumElements = MVT::getVectorNumElements(Node->getValueType(0)); assert(NumElements > 1 && "Cannot split a single element vector!"); unsigned NewNumElts = NumElements/2; - SDOperand NewNumEltsNode = DAG.getConstant(NewNumElts, MVT::i32); - SDOperand TypeNode = *(Node->op_end()-1); + MVT::ValueType NewEltVT = MVT::getVectorElementType(Node->getValueType(0)); + MVT::ValueType NewVT = MVT::getVectorType(NewEltVT, NewNumElts); // See if we already split it. std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I @@ -5531,64 +5487,73 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, Node->dump(&DAG); #endif assert(0 && "Unhandled operation in SplitVectorOp!"); - case ISD::VBUILD_VECTOR: { + case ISD::BUILD_PAIR: + Lo = Node->getOperand(0); + Hi = Node->getOperand(1); + break; + case ISD::BUILD_VECTOR: { SmallVector<SDOperand, 8> LoOps(Node->op_begin(), Node->op_begin()+NewNumElts); - LoOps.push_back(NewNumEltsNode); - LoOps.push_back(TypeNode); - Lo = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &LoOps[0], LoOps.size()); + Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &LoOps[0], LoOps.size()); SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts, - Node->op_end()-2); - HiOps.push_back(NewNumEltsNode); - HiOps.push_back(TypeNode); - Hi = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &HiOps[0], HiOps.size()); + Node->op_end()); + Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &HiOps[0], HiOps.size()); break; } - case ISD::VCONCAT_VECTORS: { - unsigned NewNumSubvectors = (Node->getNumOperands() - 2) / 2; - SmallVector<SDOperand, 8> LoOps(Node->op_begin(), - Node->op_begin()+NewNumSubvectors); - LoOps.push_back(NewNumEltsNode); - LoOps.push_back(TypeNode); - Lo = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &LoOps[0], LoOps.size()); + case ISD::CONCAT_VECTORS: { + unsigned NewNumSubvectors = Node->getNumOperands() / 2; + if (NewNumSubvectors == 1) { + Lo = Node->getOperand(0); + Hi = Node->getOperand(1); + } else { + SmallVector<SDOperand, 8> LoOps(Node->op_begin(), + Node->op_begin()+NewNumSubvectors); + Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &LoOps[0], LoOps.size()); - SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors, - Node->op_end()-2); - HiOps.push_back(NewNumEltsNode); - HiOps.push_back(TypeNode); - Hi = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &HiOps[0], HiOps.size()); + SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors, + Node->op_end()); + Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &HiOps[0], HiOps.size()); + } break; } - case ISD::VADD: - case ISD::VSUB: - case ISD::VMUL: - case ISD::VSDIV: - case ISD::VUDIV: - case ISD::VAND: - case ISD::VOR: - case ISD::VXOR: { + case ISD::ADD: + case ISD::SUB: + case ISD::MUL: + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::SDIV: + case ISD::UDIV: + case ISD::FDIV: + case ISD::AND: + case ISD::OR: + case ISD::XOR: { SDOperand LL, LH, RL, RH; SplitVectorOp(Node->getOperand(0), LL, LH); SplitVectorOp(Node->getOperand(1), RL, RH); - Lo = DAG.getNode(Node->getOpcode(), MVT::Vector, LL, RL, - NewNumEltsNode, TypeNode); - Hi = DAG.getNode(Node->getOpcode(), MVT::Vector, LH, RH, - NewNumEltsNode, TypeNode); + Lo = DAG.getNode(Node->getOpcode(), NewVT, LL, RL); + Hi = DAG.getNode(Node->getOpcode(), NewVT, LH, RH); break; } - case ISD::VLOAD: { - SDOperand Ch = Node->getOperand(0); // Legalize the chain. - SDOperand Ptr = Node->getOperand(1); // Legalize the pointer. - MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT(); - - Lo = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2)); - unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(EVT)/8; + case ISD::LOAD: { + LoadSDNode *LD = cast<LoadSDNode>(Node); + SDOperand Ch = LD->getChain(); + SDOperand Ptr = LD->getBasePtr(); + const Value *SV = LD->getSrcValue(); + int SVOffset = LD->getSrcValueOffset(); + unsigned Alignment = LD->getAlignment(); + bool isVolatile = LD->isVolatile(); + + Lo = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); + unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(NewEltVT)/8; Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, getIntPtrConstant(IncrementSize)); - // FIXME: This creates a bogus srcvalue! - Hi = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2)); + SVOffset += IncrementSize; + if (Alignment > IncrementSize) + Alignment = IncrementSize; + Hi = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); // Build a factor node to remember that this load is independent of the // other one. @@ -5599,42 +5564,31 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); break; } - case ISD::VBIT_CONVERT: { + case ISD::BIT_CONVERT: { // We know the result is a vector. The input may be either a vector or a // scalar value. - if (Op.getOperand(0).getValueType() != MVT::Vector) { + if (!MVT::isVector(Op.getOperand(0).getValueType())) { // Lower to a store/load. FIXME: this could be improved probably. SDOperand Ptr = CreateStackTemporary(Op.getOperand(0).getValueType()); SDOperand St = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), Ptr, NULL, 0); - MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT(); - St = DAG.getVecLoad(NumElements, EVT, St, Ptr, DAG.getSrcValue(0)); + St = DAG.getLoad(NewVT, St, Ptr, NULL, 0); SplitVectorOp(St, Lo, Hi); } else { // If the input is a vector type, we have to either scalarize it, pack it // or convert it based on whether the input vector type is legal. SDNode *InVal = Node->getOperand(0).Val; - unsigned NumElems = - cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); - - // If the input is from a single element vector, scalarize the vector, - // then treat like a scalar. - if (NumElems == 1) { - SDOperand Scalar = PackVectorOp(Op.getOperand(0), EVT); - Scalar = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Scalar, - Op.getOperand(1), Op.getOperand(2)); - SplitVectorOp(Scalar, Lo, Hi); - } else { + unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0)); + + assert(NumElems > 1); + { // Split the input vector. SplitVectorOp(Op.getOperand(0), Lo, Hi); // Convert each of the pieces now. - Lo = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Lo, - NewNumEltsNode, TypeNode); - Hi = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Hi, - NewNumEltsNode, TypeNode); + Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT, Lo); + Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT, Hi); } break; } @@ -5648,18 +5602,18 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, } -/// PackVectorOp - Given an operand of MVT::Vector type, convert it into the -/// equivalent operation that returns a scalar (e.g. MVT::f32) or packed value -/// (e.g. MVT::v4f32). When this is called, we know that PackedVT is the right -/// type for the result. -SDOperand SelectionDAGLegalize::PackVectorOp(SDOperand Op, - MVT::ValueType NewVT) { - assert(Op.getValueType() == MVT::Vector && "Bad PackVectorOp invocation!"); +/// ScalarizeVectorOp - Given an operand of vector type, convert it into the +/// equivalent operation that returns a scalar (e.g. F32) value. +SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { + assert(MVT::isVector(Op.getValueType()) && + "Bad ScalarizeVectorOp invocation!"); SDNode *Node = Op.Val; + MVT::ValueType NewVT = MVT::getVectorElementType(Op.getValueType()); + assert(MVT::getVectorNumElements(Op.getValueType()) == 1); - // See if we already packed it. - std::map<SDOperand, SDOperand>::iterator I = PackedNodes.find(Op); - if (I != PackedNodes.end()) return I->second; + // See if we already scalarized it. + std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op); + if (I != ScalarizedNodes.end()) return I->second; SDOperand Result; switch (Node->getOpcode()) { @@ -5667,146 +5621,89 @@ SDOperand SelectionDAGLegalize::PackVectorOp(SDOperand Op, #ifndef NDEBUG Node->dump(&DAG); cerr << "\n"; #endif - assert(0 && "Unknown vector operation in PackVectorOp!"); - case ISD::VADD: - case ISD::VSUB: - case ISD::VMUL: - case ISD::VSDIV: - case ISD::VUDIV: - case ISD::VAND: - case ISD::VOR: - case ISD::VXOR: - Result = DAG.getNode(getScalarizedOpcode(Node->getOpcode(), NewVT), + assert(0 && "Unknown vector operation in ScalarizeVectorOp!"); + case ISD::ADD: + case ISD::FADD: + case ISD::SUB: + case ISD::FSUB: + case ISD::MUL: + case ISD::FMUL: + case ISD::SDIV: + case ISD::UDIV: + case ISD::FDIV: + case ISD::SREM: + case ISD::UREM: + case ISD::FREM: + case ISD::AND: + case ISD::OR: + case ISD::XOR: + Result = DAG.getNode(Node->getOpcode(), NewVT, - PackVectorOp(Node->getOperand(0), NewVT), - PackVectorOp(Node->getOperand(1), NewVT)); + ScalarizeVectorOp(Node->getOperand(0)), + ScalarizeVectorOp(Node->getOperand(1))); break; - case ISD::VLOAD: { - SDOperand Ch = LegalizeOp(Node->getOperand(0)); // Legalize the chain. - SDOperand Ptr = LegalizeOp(Node->getOperand(1)); // Legalize the pointer. - - SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2)); - Result = DAG.getLoad(NewVT, Ch, Ptr, SV->getValue(), SV->getOffset()); + case ISD::FNEG: + case ISD::FABS: + case ISD::FSQRT: + case ISD::FSIN: + case ISD::FCOS: + Result = DAG.getNode(Node->getOpcode(), + NewVT, + ScalarizeVectorOp(Node->getOperand(0))); + break; + case ISD::LOAD: { + LoadSDNode *LD = cast<LoadSDNode>(Node); + SDOperand Ch = LegalizeOp(LD->getChain()); // Legalize the chain. + SDOperand Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. + const Value *SV = LD->getSrcValue(); + int SVOffset = LD->getSrcValueOffset(); + Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, + LD->isVolatile(), LD->getAlignment()); + // Remember that we legalized the chain. AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); break; } - case ISD::VBUILD_VECTOR: - if (Node->getOperand(0).getValueType() == NewVT) { - // Returning a scalar? - Result = Node->getOperand(0); - } else { - // Returning a BUILD_VECTOR? - - // If all elements of the build_vector are undefs, return an undef. - bool AllUndef = true; - for (unsigned i = 0, e = Node->getNumOperands()-2; i != e; ++i) - if (Node->getOperand(i).getOpcode() != ISD::UNDEF) { - AllUndef = false; - break; - } - if (AllUndef) { - Result = DAG.getNode(ISD::UNDEF, NewVT); - } else { - Result = DAG.getNode(ISD::BUILD_VECTOR, NewVT, Node->op_begin(), - Node->getNumOperands()-2); - } - } + case ISD::BUILD_VECTOR: + Result = Node->getOperand(0); + break; + case ISD::INSERT_VECTOR_ELT: + // Returning the inserted scalar element. + Result = Node->getOperand(1); break; - case ISD::VCONCAT_VECTORS: + case ISD::CONCAT_VECTORS: assert(Node->getOperand(0).getValueType() == NewVT && "Concat of non-legal vectors not yet supported!"); Result = Node->getOperand(0); break; - case ISD::VINSERT_VECTOR_ELT: - if (!MVT::isVector(NewVT)) { - // Returning a scalar? Must be the inserted element. - Result = Node->getOperand(1); - } else { - Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT, - PackVectorOp(Node->getOperand(0), NewVT), - Node->getOperand(1), Node->getOperand(2)); - } + case ISD::VECTOR_SHUFFLE: { + // Figure out if the scalar is the LHS or RHS and return it. + SDOperand EltNum = Node->getOperand(2).getOperand(0); + if (cast<ConstantSDNode>(EltNum)->getValue()) + Result = ScalarizeVectorOp(Node->getOperand(1)); + else + Result = ScalarizeVectorOp(Node->getOperand(0)); break; - case ISD::VEXTRACT_SUBVECTOR: - Result = PackVectorOp(Node->getOperand(0), NewVT); + } + case ISD::EXTRACT_SUBVECTOR: + Result = Node->getOperand(0); assert(Result.getValueType() == NewVT); break; - case ISD::VVECTOR_SHUFFLE: - if (!MVT::isVector(NewVT)) { - // Returning a scalar? Figure out if it is the LHS or RHS and return it. - SDOperand EltNum = Node->getOperand(2).getOperand(0); - if (cast<ConstantSDNode>(EltNum)->getValue()) - Result = PackVectorOp(Node->getOperand(1), NewVT); - else - Result = PackVectorOp(Node->getOperand(0), NewVT); - } else { - // Otherwise, return a VECTOR_SHUFFLE node. First convert the index - // vector from a VBUILD_VECTOR to a BUILD_VECTOR. - std::vector<SDOperand> BuildVecIdx(Node->getOperand(2).Val->op_begin(), - Node->getOperand(2).Val->op_end()-2); - MVT::ValueType BVT = MVT::getIntVectorWithNumElements(BuildVecIdx.size()); - SDOperand BV = DAG.getNode(ISD::BUILD_VECTOR, BVT, - Node->getOperand(2).Val->op_begin(), - Node->getOperand(2).Val->getNumOperands()-2); - - Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, - PackVectorOp(Node->getOperand(0), NewVT), - PackVectorOp(Node->getOperand(1), NewVT), BV); - } - break; - case ISD::VBIT_CONVERT: - if (Op.getOperand(0).getValueType() != MVT::Vector) - Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0)); - else { - // If the input is a vector type, we have to either scalarize it, pack it - // or convert it based on whether the input vector type is legal. - SDNode *InVal = Node->getOperand(0).Val; - unsigned NumElems = - cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue(); - MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems); - if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) { - // Turn this into a bit convert of the packed input. - Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, - PackVectorOp(Node->getOperand(0), TVT)); - break; - } else if (NumElems == 1) { - // Turn this into a bit convert of the scalar input. - Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, - PackVectorOp(Node->getOperand(0), EVT)); - break; - } else { - // If the input vector type isn't legal, then go through memory. - SDOperand Ptr = CreateStackTemporary(NewVT); - // Get the alignment for the store. - const TargetData &TD = *TLI.getTargetData(); - unsigned Align = - TD.getABITypeAlignment(MVT::getTypeForValueType(NewVT)); - - SDOperand St = DAG.getStore(DAG.getEntryNode(), - Node->getOperand(0), Ptr, NULL, 0, false, - Align); - Result = DAG.getLoad(NewVT, St, Ptr, 0, 0); - break; - } - } + case ISD::BIT_CONVERT: + Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0)); break; - case ISD::VSELECT: + case ISD::SELECT: Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0), - PackVectorOp(Op.getOperand(1), NewVT), - PackVectorOp(Op.getOperand(2), NewVT)); + ScalarizeVectorOp(Op.getOperand(1)), + ScalarizeVectorOp(Op.getOperand(2))); break; } if (TLI.isTypeLegal(NewVT)) Result = LegalizeOp(Result); - bool isNew = PackedNodes.insert(std::make_pair(Op, Result)).second; - assert(isNew && "Value already packed?"); + bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second; + assert(isNew && "Value already scalarized?"); return Result; } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index d70823379a..5780eff67a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -673,7 +673,9 @@ SDOperand SelectionDAG::getConstant(uint64_t Val, MVT::ValueType VT, bool isT) { SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT, bool isTarget) { assert(MVT::isFloatingPoint(VT) && "Cannot create integer FP constant!"); - if (VT == MVT::f32) + MVT::ValueType EltVT = + MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT; + if (EltVT == MVT::f32) Val = (float)Val; // Mask out extra precision. // Do the map lookup using the actual bit pattern for the floating point @@ -681,15 +683,21 @@ SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT, // we don't have issues with SNANs. unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; FoldingSetNodeID ID; - AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); + AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); ID.AddDouble(Val); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); - SDNode *N = new ConstantFPSDNode(isTarget, Val, VT); + SDNode *N = new ConstantFPSDNode(isTarget, Val, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - return SDOperand(N, 0); + SDOperand Result(N, 0); + if (MVT::isVector(VT)) { + SmallVector<SDOperand, 8> Ops; + Ops.assign(MVT::getVectorNumElements(VT), Result); + Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); + } + return Result; } SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV, @@ -1952,6 +1960,23 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, if (EVT == VT) return N1; // Not actually extending break; } + case ISD::EXTRACT_VECTOR_ELT: + assert(N2C && "Bad EXTRACT_VECTOR_ELT!"); + + // EXTRACT_VECTOR_ELT of BUILD_PAIR is often formed while lowering is + // expanding copies of large vectors from registers. + if (N1.getOpcode() == ISD::BUILD_PAIR) { + unsigned NewNumElts = MVT::getVectorNumElements(N1.getValueType()) / 2; + bool Low = N2C->getValue() < NewNumElts; + return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(!Low), + Low ? N2 : getConstant(N2C->getValue() - NewNumElts, + N2.getValueType())); + } + // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is + // expanding large vector constants. + if (N1.getOpcode() == ISD::BUILD_VECTOR) + return N1.getOperand(N2C->getValue()); + break; case ISD::EXTRACT_ELEMENT: assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!"); @@ -2045,14 +2070,10 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, MVT::getVectorNumElements(VT) == N3.getNumOperands() && "Illegal VECTOR_SHUFFLE node!"); break; - case ISD::VBIT_CONVERT: - // Fold vbit_convert nodes from a type to themselves. - if (N1.getValueType() == MVT::Vector) { - assert(isa<ConstantSDNode>(*(N1.Val->op_end()-2)) && - isa<VTSDNode>(*(N1.Val->op_end()-1)) && "Malformed vector input!"); - if (*(N1.Val->op_end()-2) == N2 && *(N1.Val->op_end()-1) == N3) - return N1; - } + case ISD::BIT_CONVERT: + // Fold bit_convert nodes from a type to themselves. + if (N1.getValueType() == VT) + return N1; break; } @@ -2095,7 +2116,7 @@ SDOperand SelectionDAG::getLoad(MVT::ValueType VT, bool isVolatile, unsigned Alignment) { if (Alignment == 0) { // Ensure that codegen never sees alignment 0 const Type *Ty = 0; - if (VT != MVT::Vector && VT != MVT::iPTR) { + if (VT != MVT::iPTR) { Ty = MVT::getTypeForValueType(VT); } else if (SV) { const PointerType *PT = dyn_cast<PointerType>(SV->getType()); @@ -2149,7 +2170,7 @@ SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT, if (Alignment == 0) { // Ensure that codegen never sees alignment 0 const Type *Ty = 0; - if (VT != MVT::Vector && VT != MVT::iPTR) { + if (VT != MVT::iPTR) { Ty = MVT::getTypeForValueType(VT); } else if (SV) { const PointerType *PT = dyn_cast<PointerType>(SV->getType()); @@ -2211,14 +2232,6 @@ SelectionDAG::getIndexedLoad(SDOperand OrigLoad, SDOperand Base, return SDOperand(N, 0); } -SDOperand SelectionDAG::getVecLoad(unsigned Count, MVT::ValueType EVT, - SDOperand Chain, SDOperand Ptr, - SDOperand SV) { - SDOperand Ops[] = { Chain, Ptr, SV, getConstant(Count, MVT::i32), - getValueType(EVT) }; - return getNode(ISD::VLOAD, getVTList(MVT::Vector, MVT::Other), Ops, 5); -} - SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val, SDOperand Ptr, const Value *SV, int SVOffset, bool isVolatile, unsigned Alignment) { @@ -2226,7 +2239,7 @@ SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val, if (Alignment == 0) { // Ensure that codegen never sees alignment 0 const Type *Ty = 0; - if (VT != MVT::Vector && VT != MVT::iPTR) { + if (VT != MVT::iPTR) { Ty = MVT::getTypeForValueType(VT); } else if (SV) { const PointerType *PT = dyn_cast<PointerType>(SV->getType()); @@ -2271,7 +2284,7 @@ SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val, if (Alignment == 0) { // Ensure that codegen never sees alignment 0 const Type *Ty = 0; - if (VT != MVT::Vector && VT != MVT::iPTR) { + if (VT != MVT::iPTR) { Ty = MVT::getTypeForValueType(VT); } else if (SV) { const PointerType *PT = dyn_cast<PointerType>(SV->getType()); @@ -2462,7 +2475,18 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList, } SDVTList SelectionDAG::getVTList(MVT::ValueType VT) { - return makeVTList(SDNode::getValueTypeList(VT), 1); + if (!MVT::isExtendedValueType(VT)) + return makeVTList(SDNode::getValueTypeList(VT), 1); + + for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(), + E = VTList.end(); I != E; ++I) { + if (I->size() == 1 && (*I)[0] == VT) + return makeVTList(&(*I)[0], 1); + } + std::vector<MVT::ValueType> V; + V.push_back(VT); + VTList.push_front(V); + return makeVTList(&(*VTList.begin())[0], 1); } SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2) { @@ -2496,7 +2520,7 @@ SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2, SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) { switch (NumVTs) { case 0: assert(0 && "Cannot have nodes without results!"); - case 1: return makeVTList(SDNode::getValueTypeList(VTs[0]), 1); + case 1: return getVTList(VTs[0]); case 2: return getVTList(VTs[0], VTs[1]); case 3: return getVTList(VTs[0], VTs[1], VTs[2]); default: break; @@ -3394,30 +3418,16 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FDIV: return "fdiv"; case ISD::FREM: return "frem"; case ISD::FCOPYSIGN: return "fcopysign"; - case ISD::VADD: return "vadd"; - case ISD::VSUB: return "vsub"; - case ISD::VMUL: return "vmul"; - case ISD::VSDIV: return "vsdiv"; - case ISD::VUDIV: return "vudiv"; - case ISD::VAND: return "vand"; - case ISD::VOR: return "vor"; - case ISD::VXOR: return "vxor"; case ISD::SETCC: return "setcc"; case ISD::SELECT: return "select"; case ISD::SELECT_CC: return "select_cc"; - case ISD::VSELECT: return "vselect"; case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; - case ISD::VINSERT_VECTOR_ELT: return "vinsert_vector_elt"; case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; - case ISD::VEXTRACT_VECTOR_ELT: return "vextract_vector_elt"; - case ISD::VCONCAT_VECTORS: return "vconcat_vectors"; - case ISD::VEXTRACT_SUBVECTOR: return "vextract_subvector"; + case ISD::CONCAT_VECTORS: return "concat_vectors"; + case ISD::EXTRACT_SUBVECTOR: return "extract_subvector"; case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; - case ISD::VBUILD_VECTOR: return "vbuild_vector"; case ISD::VECTOR_SHUFFLE: return "vector_shuffle"; - case ISD::VVECTOR_SHUFFLE: return "vvector_shuffle"; - case ISD::VBIT_CONVERT: return "vbit_convert"; case ISD::CARRY_FALSE: return "carry_false"; case ISD::ADDC: return "addc"; case ISD::ADDE: return "adde"; @@ -3456,7 +3466,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { // Other operators case ISD::LOAD: return "load"; case ISD::STORE: return "store"; - case ISD::VLOAD: return "vload"; case ISD::VAARG: return "vaarg"; case ISD::VACOPY: return "vacopy"; case ISD::VAEND: return "vaend"; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index cce0a3254a..472ca49637 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -290,15 +290,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, if (PN->use_empty()) continue; MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters; - if (VT != MVT::Vector) - NumRegisters = TLI.getNumRegisters(VT); - else { - MVT::ValueType VT1,VT2; - NumRegisters = - TLI.getVectorTypeBreakdown(cast<VectorType>(PN->getType()), - VT1, VT2); - } + unsigned NumRegisters = TLI.getNumRegisters(VT); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); @@ -320,7 +312,7 @@ unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { // If this is a vector type, figure out what type it will decompose into // and how many of the elements it will use. - if (VT == MVT::Vector) { + if (MVT::isVector(VT)) { const VectorType *PTy = cast<VectorType>(V->getType()); unsigned NumElts = PTy->getNumElements(); MVT::ValueType EltTy = TLI.getValueType(PTy->getElementType()); @@ -582,36 +574,30 @@ public: void visitInvoke(InvokeInst &I); void visitUnwind(UnwindInst &I); - void visitScalarBinary(User &I, unsigned OpCode); - void visitVectorBinary(User &I, unsigned OpCode); - void visitEitherBinary(User &I, unsigned ScalarOp, unsigned VectorOp); + void visitBinary(User &I, unsigned OpCode); void visitShift(User &I, unsigned Opcode); void visitAdd(User &I) { - if (isa<VectorType>(I.getType())) - visitVectorBinary(I, ISD::VADD); - else if (I.getType()->isFloatingPoint()) - visitScalarBinary(I, ISD::FADD); + if (I.getType()->isFPOrFPVector()) + visitBinary(I, ISD::FADD); else - visitScalarBinary(I, ISD::ADD); + visitBinary(I, ISD::ADD); } void visitSub(User &I); void visitMul(User &I) { - if (isa<VectorType>(I.getType())) - visitVectorBinary(I, ISD::VMUL); - else if (I.getType()->isFloatingPoint()) - visitScalarBinary(I, ISD::FMUL); + if (I.getType()->isFPOrFPVector()) + visitBinary(I, ISD::FMUL); else - visitScalarBinary(I, ISD::MUL); + visitBinary(I, ISD::MUL); } - void visitURem(User &I) { visitScalarBinary(I, ISD::UREM); } - void visitSRem(User &I) { visitScalarBinary(I, ISD::SREM); } - void visitFRem(User &I) { visitScalarBinary(I, ISD::FREM); } - void visitUDiv(User &I) { visitEitherBinary(I, ISD::UDIV, ISD::VUDIV); } - void visitSDiv(User &I) { visitEitherBinary(I, ISD::SDIV, ISD::VSDIV); } - void visitFDiv(User &I) { visitEitherBinary(I, ISD::FDIV, ISD::VSDIV); } - void visitAnd (User &I) { visitEitherBinary(I, ISD::AND, ISD::VAND ); } - void visitOr (User &I) { visitEitherBinary(I, ISD::OR, ISD::VOR ); } - void visitXor (User &I) { visitEitherBinary(I, ISD::XOR, ISD::VXOR ); } + void visitURem(User &I) { visitBinary(I, ISD::UREM); } + void visitSRem(User &I) { visitBinary(I, ISD::SREM); } + void visitFRem(User &I) { visitBinary(I, ISD::FREM); } + void visitUDiv(User &I) { visitBinary(I, ISD::UDIV); } + void visitSDiv(User &I) { visitBinary(I, ISD::SDIV); } + void visitFDiv(User &I) { visitBinary(I, ISD::FDIV); } + void visitAnd (User &I) { visitBinary(I, ISD::AND); } + void visitOr (User &I) { visitBinary(I, ISD::OR); } + void visitXor (User &I) { visitBinary(I, ISD::XOR); } void visitShl (User &I) { visitShift(I, ISD::SHL); } void visitLShr(User &I) { visitShift(I, ISD::SRL); } void visitAShr(User &I) { visitShift(I, ISD::SRA); } @@ -687,7 +673,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { if (!isa<VectorType>(VTy)) return N = DAG.getNode(ISD::UNDEF, VT); - // Create a VBUILD_VECTOR of undef nodes. + // Create a BUILD_VECTOR of undef nodes. const VectorType *PTy = cast<VectorType>(VTy); unsigned NumElements = PTy->getNumElements(); MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); @@ -696,9 +682,8 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); // Create a VConstant node with generic Vector type. - Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); - Ops.push_back(DAG.getValueType(PVT)); - return N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); + return N = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { return N = DAG.getConstantFP(CFP->getValue(), VT); @@ -723,10 +708,9 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { Ops.assign(NumElements, Op); } - // Create a VBUILD_VECTOR node with generic Vector type. - Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); - Ops.push_back(DAG.getValueType(PVT)); - return NodeMap[V] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], + // Create a BUILD_VECTOR node. + MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); + return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } else { // Canonicalize all constant ints to be unsigned. @@ -745,7 +729,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { assert(InReg && "Value not in map!"); // If this type is not legal, make it so now. - if (VT != MVT::Vector) { + if (!MVT::isVector(VT)) { if (TLI.getTypeAction(VT) == TargetLowering::Expand) { // Source must be expanded. This input value is actually coming from the // register pair InReg and InReg+1. @@ -757,7 +741,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { else { assert(NumVals == 2 && "1 to 4 (and more) expansion not implemented!"); N = DAG.getNode(ISD::BUILD_PAIR, VT, N, - DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT)); + DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT)); } } else { MVT::ValueType DestVT = TLI.getTypeToTransformTo(VT); @@ -768,29 +752,28 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { : DAG.getNode(ISD::TRUNCATE, VT, N); } } else { - // Otherwise, if this is a vector, make it available as a generic vector + // Otherwise, if this is a vector, make it available as a vector // here. - MVT::ValueType PTyElementVT, PTyLegalElementVT; - const VectorType *PTy = cast<VectorType>(VTy); - unsigned NE = TLI.getVectorTypeBreakdown(PTy, PTyElementVT, - PTyLegalElementVT); + MVT::ValueType ElementVT, LegalElementVT; + unsigned NE = TLI.getVectorTypeBreakdown(VT, ElementVT, + LegalElementVT); - // Build a VBUILD_VECTOR or VCONCAT_VECTORS with the input registers. + // Build a BUILD_VECTOR or CONCAT_VECTORS with the input registers. SmallVector<SDOperand, 8> Ops; - if (PTyElementVT == PTyLegalElementVT) { - // If the value types are legal, just VBUILD the CopyFromReg nodes. + if (ElementVT == LegalElementVT) { + // If the value types are legal, just BUILD the CopyFromReg nodes. for (unsigned i = 0; i != NE; ++i) Ops.push_back(DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyElementVT)); - } else if (PTyElementVT < PTyLegalElementVT) { + ElementVT)); + } else if (ElementVT < LegalElementVT) { // If the register was promoted, use TRUNCATE or FP_ROUND as appropriate. for (unsigned i = 0; i != NE; ++i) { SDOperand Op = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); - if (MVT::isFloatingPoint(PTyElementVT)) - Op = DAG.getNode(ISD::FP_ROUND, PTyElementVT, Op); + LegalElementVT); + if (MVT::isFloatingPoint(ElementVT)) + Op = DAG.getNode(ISD::FP_ROUND, ElementVT, Op); else - Op = DAG.getNode(ISD::TRUNCATE, PTyElementVT, Op); + Op = DAG.getNode(ISD::TRUNCATE, ElementVT, Op); Ops.push_back(Op); } } else { @@ -798,21 +781,22 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { assert((NE & 1) == 0 && "Must expand into a multiple of 2 elements!"); for (unsigned i = 0; i != NE; ++i) { SDOperand Op0 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); + LegalElementVT); SDOperand Op1 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); - Ops.push_back(DAG.getNode(ISD::BUILD_PAIR, PTyElementVT, Op0, Op1)); + LegalElementVT); + Ops.push_back(DAG.getNode(ISD::BUILD_PAIR, ElementVT, Op0, Op1)); } } - if (MVT::isVector(PTyElementVT)) { - Ops.push_back(DAG.getConstant(NE * MVT::getVectorNumElements(PTyElementVT), MVT::i32)); - Ops.push_back(DAG.getValueType(MVT::getVectorElementType(PTyElementVT))); - N = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &Ops[0], Ops.size()); + if (MVT::isVector(ElementVT)) { + N = DAG.getNode(ISD::CONCAT_VECTORS, + MVT::getVectorType(MVT::getVectorElementType(ElementVT), + NE * MVT::getVectorNumElements(ElementVT)), + &Ops[0], Ops.size()); } else { - Ops.push_back(DAG.getConstant(NE, MVT::i32)); - Ops.push_back(DAG.getValueType(PTyElementVT)); - N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); + N = DAG.getNode(ISD::BUILD_VECTOR, + MVT::getVectorType(ElementVT, NE), + &Ops[0], Ops.size()); } } @@ -1220,7 +1204,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // register so it can be used as an index into the jump table in a // subsequent basic block. This value may be smaller or larger than the // target's pointer type, and therefore require extension or truncating. - if (VT > TLI.getPointerTy()) + if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB); else SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB); @@ -1270,7 +1254,7 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) ISD::SETUGT); SDOperand ShiftOp; - if (VT > TLI.getShiftAmountTy()) + if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy())) ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB); else ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB); @@ -1910,52 +1894,44 @@ void SelectionDAGLowering::visitSub(User &I) { // -0.0 - X --> fneg const Type *Ty = I.getType(); if (isa<VectorType>(Ty)) { - visitVectorBinary(I, ISD::VSUB); - } else if (Ty->isFloatingPoint()) { + if (ConstantVector *CV = dyn_cast<ConstantVector>(I.getOperand(0))) { + const VectorType *DestTy = cast<VectorType>(I.getType()); + const Type *ElTy = DestTy->getElementType(); + unsigned VL = DestTy->getNumElements(); + std::vector<Constant*> NZ(VL, ConstantFP::get(ElTy, -0.0)); + Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size()); + if (CV == CNZ) { + SDOperand Op2 = getValue(I.getOperand(1)); + setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); + return; + } + } + } + if (Ty->isFloatingPoint()) { if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0))) if (CFP->isExactlyValue(-0.0)) { SDOperand Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } - visitScalarBinary(I, ISD::FSUB); - } else - visitScalarBinary(I, ISD::SUB); + } + + visitBinary(I, Ty->isFPOrFPVector() ? ISD::FSUB : ISD::SUB); } -void SelectionDAGLowering::visitScalarBinary(User &I, unsigned OpCode) { +void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) { SDOperand Op1 = getValue(I.getOperand(0)); SDOperand Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2)); } -void -SelectionDAGLowering::visitVectorBinary(User &I, unsigned OpCode) { - assert(isa<VectorType>(I.getType())); - const VectorType *Ty = cast<VectorType>(I.getType()); - SDOperand Typ = DAG.getValueType(TLI.getValueType(Ty->getElementType())); - - setValue(&I, DAG.getNode(OpCode, MVT::Vector, - getValue(I.getOperand(0)), - getValue(I.getOperand(1)), - DAG.getConstant(Ty->getNumElements(), MVT::i32), - Typ)); -} - -void SelectionDAGLowering::visitEitherBinary(User &I, unsigned ScalarOp, - unsigned VectorOp) { - if (isa<VectorType>(I.getType())) - visitVectorBinary(I, VectorOp); - else - visitScalarBinary(I, ScalarOp); -} - void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { SDOperand Op1 = getValue(I.getOperand(0)); SDOperand Op2 = getValue(I.getOperand(1)); - if (TLI.getShiftAmountTy() < Op2.getValueType()) + if (MVT::getSizeInBits(TLI.getShiftAmountTy()) < + MVT::getSizeInBits(Op2.getValueType())) Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); else if (TLI.getShiftAmountTy() > Op2.getValueType()) Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); @@ -2033,14 +2009,8 @@ void SelectionDAGLowering::visitSelect(User &I) { SDOperand Cond = getValue(I.getOperand(0)); SDOperand TrueVal = getValue(I.getOperand(1)); SDOperand FalseVal = getValue(I.getOperand(2)); - if (!isa<VectorType>(I.getType())) { - setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, - TrueVal, FalseVal)); - } else { - setValue(&I, DAG.getNode(ISD::VSELECT, MVT::Vector, Cond, TrueVal, FalseVal, - *(TrueVal.Val->op_end()-2), - *(TrueVal.Val->op_end()-1))); - } + setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, + TrueVal, FalseVal)); } @@ -2140,23 +2110,6 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { void SelectionDAGLowering::visitBitCast(User &I) { SDOperand N = getValue(I.getOperand(0)); MVT::ValueType DestVT = TLI.getValueType(I.getType()); - if (DestVT == MVT::Vector) { - // This is a cast to a vector from something else. - // Get information about the output vector. - const VectorType *DestTy = cast<VectorType>(I.getType()); - MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); - setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N, - DAG.getConstant(DestTy->getNumElements(),MVT::i32), - DAG.getValueType(EltVT))); - return; - } - MVT::ValueType SrcVT = N.getValueType(); - if (SrcVT == MVT::Vector) { - // This is a cast from a vctor to something else. - // Get information about the input vector. - setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N)); - return; - } // BitCast assures us that source and destination are the same size so this // is either a BIT_CONVERT or a no-op. @@ -2172,18 +2125,16 @@ void SelectionDAGLowering::visitInsertElement(User &I) { SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(2))); - SDOperand Num = *(InVec.Val->op_end()-2); - SDOperand Typ = *(InVec.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VINSERT_VECTOR_ELT, MVT::Vector, - InVec, InVal, InIdx, Num, Typ)); + setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, + TLI.getValueType(I.getType()), + InVec, InVal, InIdx)); } void SelectionDAGLowering::visitExtractElement(User &I) { SDOperand InVec = getValue(I.getOperand(0)); SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(1))); - SDOperand Typ = *(InVec.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VEXTRACT_VECTOR_ELT, + setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TLI.getValueType(I.getType()), InVec, InIdx)); } @@ -2192,10 +2143,9 @@ void SelectionDAGLowering::visitShuffleVector(User &I) { SDOperand V2 = getValue(I.getOperand(1)); SDOperand Mask = getValue(I.getOperand(2)); - SDOperand Num = *(V1.Val->op_end()-2); - SDOperand Typ = *(V2.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, - V1, V2, Mask, Num, Typ)); + setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, + TLI.getValueType(I.getType()), + V1, V2, Mask)); } @@ -2325,15 +2275,9 @@ SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, const Value *SV, SDOperand Root, bool isVolatile, unsigned Alignment) { - SDOperand L; - if (const VectorType *PTy = dyn_cast<VectorType>(Ty)) { - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); - L = DAG.getVecLoad(PTy->getNumElements(), PVT, Root, Ptr, - DAG.getSrcValue(SV)); - } else { - L = DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, - isVolatile, Alignment); - } + SDOperand L = + DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, + isVolatile, Alignment); if (isVolatile) DAG.setRoot(L.getValue(1)); @@ -2394,17 +2338,6 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, // Add all operands of the call to the operand list. for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) { SDOperand Op = getValue(I.getOperand(i)); - - // If this is a vector type, force it to the right vector type. - if (Op.getValueType() == MVT::Vector) { - const VectorType *OpTy = cast<VectorType>(I.getOperand(i)->getType()); - MVT::ValueType EltVT = TLI.getValueType(OpTy->getElementType()); - - MVT::ValueType VVT = MVT::getVectorType(EltVT, OpTy->getNumElements()); - assert(VVT != MVT::Other && "Intrinsic uses a non-legal type?"); - Op = DAG.getNode(ISD::VBIT_CONVERT, VVT, Op); - } - assert(TLI.isTypeLegal(Op.getValueType()) && "Intrinsic uses a non-legal type?"); Ops.push_back(Op); @@ -2413,7 +2346,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, std::vector<MVT::ValueType> VTs; if (I.getType() != Type::VoidTy) { MVT::ValueType VT = TLI.getValueType(I.getType()); - if (VT == MVT::Vector) { + if (MVT::isVector(VT)) { const VectorType *DestTy = cast<VectorType>(I.getType()); MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); @@ -2450,10 +2383,8 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, } if (I.getType() != Type::VoidTy) { if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) { - MVT::ValueType EVT = TLI.getValueType(PTy->getElementType()); - Result = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Result, - DAG.getConstant(PTy->getNumElements(), MVT::i32), - DAG.getValueType(EVT)); + MVT::ValueType VT = TLI.getValueType(PTy); + Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result); } setValue(&I, Result); } @@ -2931,11 +2862,8 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, return Val; if (MVT::isVector(RegVT)) { - assert(ValueVT == MVT::Vector && "Unknown vector conversion!"); - return DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, - DAG.getConstant(MVT::getVectorNumElements(RegVT), - MVT::i32), - DAG.getValueType(MVT::getVectorElementType(RegVT))); + assert(MVT::isVector(ValueVT) && "Unknown vector conversion!"); + return DAG.getNode(ISD::BIT_CONVERT, RegVT, Val); } if (MVT::isInteger(RegVT)) { @@ -2960,8 +2888,9 @@ void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, // a promotion. if (RegVT != ValueVT) { if (MVT::isVector(RegVT)) { - assert(Val.getValueType() == MVT::Vector &&"Not a vector-vector cast?"); - Val = DAG.getNode(ISD::VBIT_CONVERT, RegVT, Val); + assert(MVT::isVector(Val.getValueType()) && + "Not a vector-vector cast?"); + Val = DAG.getNode(ISD::BIT_CONVERT, RegVT, Val); } else if (MVT::isInteger(RegVT) && MVT::isInteger(Val.getValueType())) { if (RegVT < ValueVT) Val = DAG.getNode(ISD::TRUNCATE, RegVT, Val); @@ -3631,15 +3560,12 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { // If the result of the inline asm is a vector, it may have the wrong // width/num elts. Make sure to convert it to the right type with - // vbit_convert. - if (Val.getValueType() == MVT::Vector) { + // bit_convert. + if (MVT::isVector(Val.getValueType())) { const VectorType *VTy = cast<VectorType>(I.getType()); - unsigned DesiredNumElts = VTy->getNumElements(); - MVT::ValueType DesiredEltVT = TLI.getValueType(VTy->getElementType()); + MVT::ValueType DesiredVT = TLI.getValueType(VTy); - Val = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, - DAG.getConstant(DesiredNumElts, MVT::i32), - DAG.getValueType(DesiredEltVT)); + Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val); } setValue(&I, Val); @@ -3826,7 +3752,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { Ops.push_back(DAG.getConstant(Flags, MVT::i32)); break; case Expand: - if (VT != MVT::Vector) { + if (!MVT::isVector(VT)) { // If this is a large integer, it needs to be broken up into small // integers. Figure out what the destination type is and how many small // integers it turns into. @@ -3848,7 +3774,8 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { // Figure out if there is a Packed type corresponding to this Vector // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); + MVT::ValueType TVT = + MVT::getVectorType(getValueType(EltTy), NumElems); if (TVT != MVT::Other && isTypeLegal(TVT)) { RetVals.push_back(TVT); Ops.push_back(DAG.getConstant(Flags, MVT::i32)); @@ -3900,7 +3827,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { break; } case Expand: - if (VT != MVT::Vector) { + if (!MVT::isVector(VT)) { // If this is a large integer or a floating point node that needs to be // expanded, it needs to be reassembled from small integers. Figure out // what the source elt type is and how many small integers it is. @@ -3914,13 +3841,12 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { // Figure out if there is a Packed type corresponding to this Vector // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); + MVT::ValueType TVT = + MVT::getVectorType(getValueType(EltTy), NumElems); if (TVT != MVT::Other && isTypeLegal(TVT)) { SDOperand N = SDOperand(Result, i++); - // Handle copies from generic vectors to registers. - N = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, N, - DAG.getConstant(NumElems, MVT::i32), - DAG.getValueType(getValueType(EltTy))); + // Handle copies from vectors to registers. + N = DAG.getNode(ISD::BIT_CONVERT, TVT, N); Ops.push_back(N); } else { assert(0 && "Don't support illegal by-val vector arguments yet!"); @@ -4040,7 +3966,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, Ops.push_back(DAG.getConstant(Flags, MVT::i32)); break; case Expand: - if (VT != MVT::Vector) { + if (!MVT::isVector(VT)) { // If this is a large integer, it needs to be broken down into small // integers. Figure out what the source elt type is and how many small // integers it is. @@ -4054,10 +3980,11 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // Figure out if there is a Packed type corresponding to this Vector // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); + MVT::ValueType TVT = + MVT::getVectorType(getValueType(EltTy), NumElems); if (TVT != MVT::Other && isTypeLegal(TVT)) { - // Insert a VBIT_CONVERT of the MVT::Vector type to the vector type. - Op = DAG.getNode(ISD::VBIT_CONVERT, TVT, Op); + // Insert a BIT_CONVERT of the original type to the vector type. + Op = DAG.getNode(ISD::BIT_CONVERT, TVT, Op); Ops.push_back(Op); Ops.push_back(DAG.getConstant(Flags, MVT::i32)); } else { @@ -4083,7 +4010,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, RetTys.push_back(getTypeToTransformTo(VT)); break; case Expand: - if (VT != MVT::Vector) { + if (!MVT::isVector(VT)) { // If this is a large integer, it needs to be reassembled from small // integers. Figure out what the source elt type is and how many small // integers it is. @@ -4100,7 +4027,8 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // Figure out if there is a Packed type corresponding to this Vector // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); + MVT::ValueType TVT = + MVT::getVectorType(getValueType(EltTy), NumElems); if (TVT != MVT::Other && isTypeLegal(TVT)) { RetTys.push_back(TVT); } else { @@ -4129,21 +4057,20 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // If this value was promoted, truncate it down. if (ResVal.getValueType() != VT) { - if (VT == MVT::Vector) { - // Insert a VBIT_CONVERT to convert from the packed result type to the - // MVT::Vector type. + if (MVT::isVector(VT)) { + // Insert a BIT_CONVERT to convert from the packed result type to the + // new vector type. unsigned NumElems = cast<VectorType>(RetTy)->getNumElements(); const Type *EltTy = cast<VectorType>(RetTy)->getElementType(); // Figure out if there is a Packed type corresponding to this Vector // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy),NumElems); + MVT::ValueType TVT = + MVT::getVectorType(getValueType(EltTy),NumElems); if (TVT != MVT::Other && isTypeLegal(TVT)) { - // Insert a VBIT_CONVERT of the FORMAL_ARGUMENTS to a - // "N x PTyElementVT" MVT::Vector type. - ResVal = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, ResVal, - DAG.getConstant(NumElems, MVT::i32), - DAG.getValueType(getValueType(EltTy))); + // Insert a BIT_CONVERT of the FORMAL_ARGUMENTS to a + // "N x PTyElementVT" vector type. + ResVal = DAG.getNode(ISD::BIT_CONVERT, TVT, ResVal); } else { abort(); } @@ -4457,40 +4384,40 @@ SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, MVT::ValueType DestVT = TLI.getTypeToTransformTo(SrcVT); if (SrcVT == DestVT) { return DAG.getCopyToReg(getRoot(), Reg, Op); - } else if (SrcVT == MVT::Vector) { + } else if (MVT::isVector(SrcVT)) { // Handle copies from generic vectors to registers. - MVT::ValueType PTyElementVT, PTyLegalElementVT; - unsigned NE = TLI.getVectorTypeBreakdown(cast<VectorType>(V->getType()), - PTyElementVT, PTyLegalElementVT); - uint64_t SrcVL = cast<ConstantSDNode>(*(Op.Val->op_end()-2))->getValue(); + MVT::ValueType ElementVT, LegalElementVT; + unsigned NE = TLI.getVectorTypeBreakdown(SrcVT, + ElementVT, LegalElementVT); + uint64_t SrcVL = MVT::getVectorNumElements(SrcVT); // Loop over all of the elements of the resultant vector, - // VEXTRACT_VECTOR_ELT'ing or VEXTRACT_SUBVECTOR'ing them, converting them - // to PTyLegalElementVT, then copying them into output registers. + // EXTRACT_VECTOR_ELT'ing or EXTRACT_SUBVECTOR'ing them, converting them + // to LegalElementVT, then copying them into output registers. SmallVector<SDOperand, 8> OutChains; SDOperand Root = getRoot(); for (unsigned i = 0; i != NE; ++i) { - SDOperand Elt = MVT::isVector(PTyElementVT) ? - DAG.getNode(ISD::VEXTRACT_SUBVECTOR, PTyElementVT, + SDOperand Elt = MVT::isVector(ElementVT) ? + DAG.getNode(ISD::EXTRACT_SUBVECTOR, ElementVT, Op, DAG.getConstant(i * (SrcVL / NE), TLI.getPointerTy())) : - DAG.getNode(ISD::VEXTRACT_VECTOR_ELT, PTyElementVT, + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, ElementVT, Op, DAG.getConstant(i, TLI.getPointerTy())); - if (PTyElementVT == PTyLegalElementVT) { + if (ElementVT == LegalElementVT) { // Elements are legal. OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt)); - } else if (PTyLegalElementVT > PTyElementVT) { + } else if (LegalElementVT > ElementVT) { // Elements are promoted. - if (MVT::isFloatingPoint(PTyLegalElementVT)) - Elt = DAG.getNode(ISD::FP_EXTEND, PTyLegalElementVT, Elt); + if (MVT::isFloatingPoint(LegalElementVT)) + Elt = DAG.getNode(ISD::FP_EXTEND, LegalElementVT, Elt); else - Elt = DAG.getNode(ISD::ANY_EXTEND, PTyLegalElementVT, Elt); + Elt = DAG.getNode(ISD::ANY_EXTEND, LegalElementVT, Elt); OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt)); } else { // Elements are expanded. // The src value is expanded into multiple registers. - SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT, + SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, LegalElementVT, Elt, DAG.getConstant(0, TLI.getPointerTy())); - SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT, + SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, LegalElementVT, Elt, DAG.getConstant(1, TLI.getPointerTy())); OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Lo)); OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Hi)); @@ -4695,15 +4622,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, // Remember that this register needs to added to the machine PHI node as // the input for this MBB. MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters; - if (VT != MVT::Vector) - NumRegisters = TLI.getNumRegisters(VT); - else { - MVT::ValueType VT1,VT2; - NumRegisters = - TLI.getVectorTypeBreakdown(cast<VectorType>(PN->getType()), - VT1, VT2); - } + unsigned NumRegisters = TLI.getNumRegisters(VT); for (unsigned i = 0, e = NumRegisters; i != e; ++i) PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); } diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index b2c9016639..7ab552839c 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -199,7 +199,7 @@ static void SetValueTypeAction(MVT::ValueType VT, else if (VT == MVT::f64) TransformToType[VT] = MVT::i64; else { - assert((VT == MVT::Vector || MVT::isInteger(VT)) && VT > MVT::i8 && + assert((MVT::isVector(VT) || MVT::isInteger(VT)) && VT > MVT::i8 && "Cannot expand this type: target must support SOME integer reg!"); // Expand to the next smaller integer type! TransformToType[VT] = (MVT::ValueType)(VT-1); @@ -265,16 +265,18 @@ void TargetLowering::computeRegisterProperties() { ValueTypeActions); } - // Set MVT::Vector to always be Expanded - SetValueTypeAction(MVT::Vector, Expand, *this, TransformToType, - ValueTypeActions); - // Loop over all of the legal vector value types, specifying an identity type // transformation. for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; i <= MVT::LAST_VECTOR_VALUETYPE; ++i) { if (isTypeLegal((MVT::ValueType)i)) TransformToType[i] = (MVT::ValueType)i; + else { + MVT::ValueType VT1, VT2; + NumRegistersForVT[i] = getVectorTypeBreakdown(i, VT1, VT2); + SetValueTypeAction(i, Expand, *this, TransformToType, + ValueTypeActions); + } } } @@ -282,38 +284,42 @@ const char *TargetLowering::getTargetNodeName(unsigned Opcode) const { return NULL; } -/// getVectorTypeBreakdown - Packed types are broken down into some number of -/// legal first class types. For example, <8 x float> maps to 2 MVT::v4f32 +/// getVectorTypeBreakdown - Vector types are broken down into some number of +/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. +/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. /// -/// This method returns the number and type of the resultant breakdown. +/// This method returns the number of registers needed, and the VT for each +/// register. It also returns the VT of the VectorType elements before they +/// are promoted/expanded. /// -unsigned TargetLowering::getVectorTypeBreakdown(const VectorType *PTy, - MVT::ValueType &PTyElementVT, - MVT::ValueType &PTyLegalElementVT) const { +unsigned TargetLowering::getVectorTypeBreakdown(MVT::ValueType VT, + MVT::ValueType &ElementVT, + MVT::ValueType &LegalElementVT) const { // Figure out the right, legal destination reg to copy into. - unsigned NumElts = PTy->getNumElements(); - MVT::ValueType EltTy = getValueType(PTy->getElementType()); + unsigned NumElts = MVT::getVectorNumElements(VT); + MVT::ValueType EltTy = MVT::getVectorElementType(VT); unsigned NumVectorRegs = 1; // Divide the input until we get to a supported size. This will always // end with a scalar if the target doesn't support vectors. - while (NumElts > 1 && !isTypeLegal(MVT::getVectorType(EltTy, NumElts))) { + while (NumElts > 1 && + !isTypeLegal(MVT::getVectorType(EltTy, NumElts))) { NumElts >>= 1; NumVectorRegs <<= 1; } - MVT::ValueType VT = MVT::getVectorType(EltTy, NumElts); - if (!isTypeLegal(VT)) - VT = EltTy; - PTyElementVT = VT; - - MVT::ValueType DestVT = getTypeToTransformTo(VT); - PTyLegalElementVT = DestVT; - if (DestVT < VT) { + MVT::ValueType NewVT = MVT::getVectorType(EltTy, NumElts); + if (!isTypeLegal(NewVT)) + NewVT = EltTy; + ElementVT = NewVT; + + MVT::ValueType DestVT = getTypeToTransformTo(NewVT); + LegalElementVT = DestVT; + if (DestVT < NewVT) { // Value is expanded, e.g. i64 -> i16. - return NumVectorRegs*(MVT::getSizeInBits(VT)/MVT::getSizeInBits(DestVT)); + return NumVectorRegs*(MVT::getSizeInBits(NewVT)/MVT::getSizeInBits(DestVT)); } else { // Otherwise, promotion or legal types use the same number of registers as // the vector decimated to the appropriate level. diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index d048caee7d..4c1d6ea406 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -2493,9 +2493,14 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) { } } - if (NumNonZero == 0) - // Must be a mix of zero and undef. Return a zero vector. - return getZeroVector(VT, DAG); + if (NumNonZero == 0) { + if (NumZero == 0) + // All undef vector. Return an UNDEF. + return DAG.getNode(ISD::UNDEF, VT); + else + // A mix of zero and undef. Return a zero vector. + return getZeroVector(VT, DAG); + } // Splat is obviously ok. Let legalizer expand it to a shuffle. if (Values.size() == 1) @@ -4919,7 +4924,6 @@ X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, case MVT::i64: return std::make_pair(0U, X86::FR64RegisterClass); // Vector types. - case MVT::Vector: case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: diff --git a/lib/VMCore/ValueTypes.cpp b/lib/VMCore/ValueTypes.cpp index e7881610e9..751bd588f6 100644 --- a/lib/VMCore/ValueTypes.cpp +++ b/lib/VMCore/ValueTypes.cpp @@ -14,13 +14,21 @@ #include "llvm/CodeGen/ValueTypes.h" #include "llvm/Type.h" #include "llvm/DerivedTypes.h" +#include <sstream> using namespace llvm; /// MVT::getValueTypeString - This function returns value type as a string, /// e.g. "i32". -const char *MVT::getValueTypeString(MVT::ValueType VT) { +std::string MVT::getValueTypeString(MVT::ValueType VT) { switch (VT) { - default: assert(0 && "Invalid ValueType!"); + default: + if (isExtendedValueType(VT)) { + std::ostringstream OS; + OS << "v" << getVectorNumElements(VT) + << getValueTypeString(getVectorElementType(VT)); + return OS.str(); + } + assert(0 && "Invalid ValueType!"); case MVT::i1: return "i1"; case MVT::i8: return "i8"; case MVT::i16: return "i16"; @@ -34,7 +42,6 @@ const char *MVT::getValueTypeString(MVT::ValueType VT) { case MVT::isVoid:return "isVoid"; case MVT::Other: return "ch"; case MVT::Flag: return "flag"; - case MVT::Vector:return "vec"; case MVT::v8i8: return "v8i8"; case MVT::v4i16: return "v4i16"; case MVT::v2i32: return "v2i32"; @@ -49,47 +56,16 @@ const char *MVT::getValueTypeString(MVT::ValueType VT) { } } -/// MVT::getVectorType - Returns the ValueType that represents a vector -/// NumElements in length, where each element is of type VT. If there is no -/// ValueType that represents this vector, a ValueType of Other is returned. -/// -MVT::ValueType MVT::getVectorType(ValueType VT, unsigned NumElements) { - switch (VT) { - default: - break; - case MVT::i8: - if (NumElements == 8) return MVT::v8i8; - if (NumElements == 16) return MVT::v16i8; - break; - case MVT::i16: - if (NumElements == 4) return MVT::v4i16; - if (NumElements == 8) return MVT::v8i16; - break; - case MVT::i32: - if (NumElements == 2) return MVT::v2i32; - if (NumElements == 4) return MVT::v4i32; - break; - case MVT::i64: - if (NumElements == 1) return MVT::v1i64; - if (NumElements == 2) return MVT::v2i64; - break; - case MVT::f32: - if (NumElements == 2) return MVT::v2f32; - if (NumElements == 4) return MVT::v4f32; - break; - case MVT::f64: - if (NumElements == 2) return MVT::v2f64; - break; - } - return MVT::Other; -} - /// MVT::getTypeForValueType - This method returns an LLVM type corresponding /// to the specified ValueType. Note that this will abort for types that cannot /// be represented. const Type *MVT::getTypeForValueType(MVT::ValueType VT) { switch (VT) { - default: assert(0 && "ValueType does not correspond to LLVM type!"); + default: + if (isExtendedValueType(VT)) + return VectorType::get(getTypeForValueType(getVectorElementType(VT)), + getVectorNumElements(VT)); + assert(0 && "ValueType does not correspond to LLVM type!"); case MVT::isVoid:return Type::VoidTy; case MVT::i1: return Type::Int1Ty; case MVT::i8: return Type::Int8Ty; @@ -114,9 +90,8 @@ const Type *MVT::getTypeForValueType(MVT::ValueType VT) { } /// MVT::getValueType - Return the value type corresponding to the specified -/// type. This returns all vectors as MVT::Vector and all pointers as -/// MVT::iPTR. If HandleUnknown is true, unknown types are returned as Other, -/// otherwise they are invalid. +/// type. This returns all pointers as MVT::iPTR. If HandleUnknown is true, +/// unknown types are returned as Other, otherwise they are invalid. MVT::ValueType MVT::getValueType(const Type *Ty, bool HandleUnknown) { switch (Ty->getTypeID()) { default: @@ -141,6 +116,10 @@ MVT::ValueType MVT::getValueType(const Type *Ty, bool HandleUnknown) { case Type::FloatTyID: return MVT::f32; case Type::DoubleTyID: return MVT::f64; case Type::PointerTyID: return MVT::iPTR; - case Type::VectorTyID: return MVT::Vector; + case Type::VectorTyID: { + const VectorType *VTy = cast<VectorType>(Ty); + return getVectorType(getValueType(VTy->getElementType(), false), + VTy->getNumElements()); + } } } |