llvm.org GIT mirror llvm / c76909a
Improve MachineMemOperand handling. - Allocate MachineMemOperands and MachineMemOperand lists in MachineFunctions. This eliminates MachineInstr's std::list member and allows the data to be created by isel and live for the remainder of codegen, avoiding a lot of copying and unnecessary translation. This also shrinks MemSDNode. - Delete MemOperandSDNode. Introduce MachineSDNode which has dedicated fields for MachineMemOperands. - Change MemSDNode to have a MachineMemOperand member instead of its own fields with the same information. This introduces some redundancy, but it's more consistent with what MachineInstr will eventually want. - Ignore alignment when searching for redundant loads for CSE, but remember the greatest alignment. Target-specific code which previously used MemOperandSDNodes with generic SDNodes now use MemIntrinsicSDNodes, with opcodes in a designated range so that the SelectionDAG framework knows that MachineMemOperand information is available. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@82794 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 10 years ago
30 changed file(s) with 752 addition(s) and 552 deletion(s). Raw diff Collapse all Expand all
2626
2727 namespace llvm {
2828
29 class Value;
2930 class Function;
3031 class MachineRegisterInfo;
3132 class MachineFrameInfo;
319320 ///
320321 void DeleteMachineBasicBlock(MachineBasicBlock *MBB);
321322
323 /// getMachineMemOperand - Allocate a new MachineMemOperand.
324 /// MachineMemOperands are owned by the MachineFunction and need not be
325 /// explicitly deallocated.
326 MachineMemOperand *getMachineMemOperand(const Value *v, unsigned f,
327 int64_t o, uint64_t s,
328 unsigned base_alignment);
329
330 /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
331 /// an existing one, adjusting by an offset and using the given EVT.
332 /// MachineMemOperands are owned by the MachineFunction and need not be
333 /// explicitly deallocated.
334 MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
335 int64_t Offset, uint64_t Size);
336
337 /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand
338 /// pointers. This array is owned by the MachineFunction.
339 MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num);
340
322341 //===--------------------------------------------------------------------===//
323342 // Debug location.
324343 //
1919 #include "llvm/ADT/ilist_node.h"
2020 #include "llvm/ADT/STLExtras.h"
2121 #include "llvm/CodeGen/MachineOperand.h"
22 #include "llvm/CodeGen/MachineMemOperand.h"
2322 #include "llvm/Target/TargetInstrDesc.h"
2423 #include "llvm/Support/DebugLoc.h"
25 #include
2624 #include
2725
2826 namespace llvm {
3129 class TargetInstrInfo;
3230 class TargetRegisterInfo;
3331 class MachineFunction;
32 class MachineMemOperand;
3433
3534 //===----------------------------------------------------------------------===//
3635 /// MachineInstr - Representation of each machine instruction.
3736 ///
3837 class MachineInstr : public ilist_node {
38 public:
39 typedef MachineMemOperand **mmo_iterator;
40
41 private:
3942 const TargetInstrDesc *TID; // Instruction descriptor.
4043 unsigned short NumImplicitOps; // Number of implicit operands (which
4144 // are determined at construction time).
4245
4346 std::vector Operands; // the operands
44 std::list MemOperands; // information on memory references
47 mmo_iterator MemRefs; // information on memory references
48 mmo_iterator MemRefsEnd;
4549 MachineBasicBlock *Parent; // Pointer to the owning basic block.
4650 DebugLoc debugLoc; // Source line information.
4751
131135 unsigned getNumExplicitOperands() const;
132136
133137 /// Access to memory operands of the instruction
134 std::list::iterator memoperands_begin()
135 { return MemOperands.begin(); }
136 std::list::iterator memoperands_end()
137 { return MemOperands.end(); }
138 std::list::const_iterator memoperands_begin() const
139 { return MemOperands.begin(); }
140 std::list::const_iterator memoperands_end() const
141 { return MemOperands.end(); }
142 bool memoperands_empty() const { return MemOperands.empty(); }
138 mmo_iterator memoperands_begin() const { return MemRefs; }
139 mmo_iterator memoperands_end() const { return MemRefsEnd; }
140 bool memoperands_empty() const { return MemRefsEnd == MemRefs; }
143141
144142 /// hasOneMemOperand - Return true if this instruction has exactly one
145143 /// MachineMemOperand.
146144 bool hasOneMemOperand() const {
147 return !memoperands_empty() &&
148 next(memoperands_begin()) == memoperands_end();
145 return MemRefsEnd - MemRefs == 1;
149146 }
150147
151148 /// isIdenticalTo - Return true if this instruction is identical to (same
318315 ///
319316 void RemoveOperand(unsigned i);
320317
321 /// addMemOperand - Add a MachineMemOperand to the machine instruction,
322 /// referencing arbitrary storage.
323 void addMemOperand(MachineFunction &MF,
324 const MachineMemOperand &MO);
325
326 /// clearMemOperands - Erase all of this MachineInstr's MachineMemOperands.
327 void clearMemOperands(MachineFunction &MF);
318 /// addMemOperand - Add a MachineMemOperand to the machine instruction.
319 /// This function should be used only occasionally. The setMemRefs function
320 /// is the primary method for setting up a MachineInstr's MemRefs list.
321 void addMemOperand(MachineFunction &MF, MachineMemOperand *MO);
322
323 /// setMemRefs - Assign this MachineInstr's memory reference descriptor
324 /// list. This does not transfer ownership.
325 void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
326 MemRefs = NewMemRefs;
327 MemRefsEnd = NewMemRefsEnd;
328 }
328329
329330 private:
330331 /// getRegInfo - If this instruction is embedded into a MachineFunction,
120120 return *this;
121121 }
122122
123 const MachineInstrBuilder &addMemOperand(const MachineMemOperand &MMO) const {
123 const MachineInstrBuilder &addMemOperand(MachineMemOperand *MMO) const {
124124 MI->addMemOperand(*MI->getParent()->getParent(), MMO);
125125 return *this;
126126 }
8787 bool isStore() const { return Flags & MOStore; }
8888 bool isVolatile() const { return Flags & MOVolatile; }
8989
90 /// refineAlignment - Update this MachineMemOperand to reflect the alignment
91 /// of MMO, if it has a greater alignment. This must only be used when the
92 /// new alignment applies to all users of this MachineMemOperand.
93 void refineAlignment(const MachineMemOperand *MMO);
94
95 /// setValue - Change the SourceValue for this MachineMemOperand. This
96 /// should only be used when an object is being relocated and all references
97 /// to it are being updated.
98 void setValue(const Value *NewSV) { V = NewSV; }
99
90100 /// Profile - Gather unique data for the object.
91101 ///
92102 void Profile(FoldingSetNodeID &ID) const;
515515 ///
516516 void EmitNoop();
517517
518 void AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO);
519
520518 void EmitPhysRegCopy(SUnit *SU, DenseMap &VRBaseMap);
521519
522520 private:
1818 #include "llvm/ADT/DenseSet.h"
1919 #include "llvm/ADT/StringMap.h"
2020 #include "llvm/CodeGen/SelectionDAGNodes.h"
21 #include "llvm/Support/RecyclingAllocator.h"
2122 #include "llvm/Target/TargetMachine.h"
2223 #include
2324 #include
513514 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
514515 SDValue Ptr, SDValue Cmp, SDValue Swp, const Value* PtrVal,
515516 unsigned Alignment=0);
517 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
518 SDValue Ptr, SDValue Cmp, SDValue Swp,
519 MachineMemOperand *MMO);
516520
517521 /// getAtomic - Gets a node for an atomic op, produces result and chain and
518522 /// takes 2 operands.
519523 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
520524 SDValue Ptr, SDValue Val, const Value* PtrVal,
521525 unsigned Alignment = 0);
526 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
527 SDValue Ptr, SDValue Val,
528 MachineMemOperand *MMO);
522529
523530 /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
524 /// result and takes a list of operands.
531 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
532 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
533 /// less than FIRST_TARGET_MEMORY_OPCODE.
525534 SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl,
526535 const EVT *VTs, unsigned NumVTs,
527536 const SDValue *Ops, unsigned NumOps,
534543 EVT MemVT, const Value *srcValue, int SVOff,
535544 unsigned Align = 0, bool Vol = false,
536545 bool ReadMem = true, bool WriteMem = true);
546
547 SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
548 const SDValue *Ops, unsigned NumOps,
549 EVT MemVT, MachineMemOperand *MMO);
537550
538551 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
539552 SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl);
554567 EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset,
555568 const Value *SV, int SVOffset, EVT MemVT,
556569 bool isVolatile=false, unsigned Alignment=0);
570 SDValue getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType,
571 EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset,
572 EVT MemVT, MachineMemOperand *MMO);
557573
558574 /// getStore - Helper function to build ISD::STORE nodes.
559575 ///
560576 SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
561577 const Value *SV, int SVOffset, bool isVolatile=false,
562578 unsigned Alignment=0);
579 SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
580 MachineMemOperand *MMO);
563581 SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
564582 const Value *SV, int SVOffset, EVT TVT,
565583 bool isVolatile=false, unsigned Alignment=0);
584 SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
585 EVT TVT, MachineMemOperand *MMO);
566586 SDValue getIndexedStore(SDValue OrigStoe, DebugLoc dl, SDValue Base,
567587 SDValue Offset, ISD::MemIndexedMode AM);
568588
569589 /// getSrcValue - Construct a node to track a Value* through the backend.
570590 SDValue getSrcValue(const Value *v);
571
572 /// getMemOperand - Construct a node to track a memory reference
573 /// through the backend.
574 SDValue getMemOperand(const MachineMemOperand &MO);
575591
576592 /// getShiftAmountOperand - Return the specified value casted to
577593 /// the target's desired shift amount type.
681697 SDNode *getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2,
682698 EVT VT3, EVT VT4, const SDValue *Ops, unsigned NumOps);
683699 SDNode *getMachineNode(unsigned Opcode, DebugLoc dl,
684 const std::vector &ResultTys, const SDValue *Ops,
685 unsigned NumOps);
700 const std::vector &ResultTys, const SDValue *Ops,
701 unsigned NumOps);
702 SDNode *getMachineNode(unsigned Opcode, DebugLoc dl, SDVTList VTs,
703 const SDValue *Ops, unsigned NumOps);
686704
687705 /// getTargetExtractSubreg - A convenience function for creating
688706 /// TargetInstrInfo::EXTRACT_SUBREG nodes.
2626 #include "llvm/ADT/STLExtras.h"
2727 #include "llvm/CodeGen/ValueTypes.h"
2828 #include "llvm/CodeGen/MachineMemOperand.h"
29 #include "llvm/Support/Allocator.h"
3029 #include "llvm/Support/MathExtras.h"
31 #include "llvm/Support/RecyclingAllocator.h"
3230 #include "llvm/Support/DataTypes.h"
3331 #include "llvm/Support/DebugLoc.h"
3432 #include
35 #include
3633
3734 namespace llvm {
3835
534531 // SRCVALUE - This is a node type that holds a Value* that is used to
535532 // make reference to a value in the LLVM IR.
536533 SRCVALUE,
537
538 // MEMOPERAND - This is a node that contains a MachineMemOperand which
539 // records information about a memory reference. This is used to make
540 // AliasAnalysis queries from the backend.
541 MEMOPERAND,
542534
543535 // PCMARKER - This corresponds to the pcmarker intrinsic.
544536 PCMARKER,
616608 ATOMIC_LOAD_UMIN,
617609 ATOMIC_LOAD_UMAX,
618610
619 // BUILTIN_OP_END - This must be the last enum value in this list.
611 /// BUILTIN_OP_END - This must be the last enum value in this list.
612 /// The target-specific pre-isel opcode values start here.
620613 BUILTIN_OP_END
621614 };
615
616 /// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations
617 /// which do not reference a specific memory location should be less than
618 /// this value. Those that do must not be less than this value, and can
619 /// be used with SelectionDAG::getMemIntrinsicNode.
620 static const int FIRST_TARGET_MEMORY_OPCODE = 1 << 14;
622621
623622 /// Node predicates
624623
866865 inline unsigned getNumOperands() const;
867866 inline const SDValue &getOperand(unsigned i) const;
868867 inline uint64_t getConstantOperandVal(unsigned i) const;
868 inline bool isTargetMemoryOpcode() const;
869869 inline bool isTargetOpcode() const;
870870 inline bool isMachineOpcode() const;
871871 inline unsigned getMachineOpcode() const;
10301030 private:
10311031 /// NodeType - The operation that this node performs.
10321032 ///
1033 short NodeType;
1033 int16_t NodeType;
10341034
10351035 /// OperandsNeedDelete - This is true if OperandList was new[]'d. If true,
10361036 /// then they will be delete[]'d when the node is destroyed.
1037 unsigned short OperandsNeedDelete : 1;
1037 uint16_t OperandsNeedDelete : 1;
10381038
10391039 protected:
10401040 /// SubclassData - This member is defined by this class, but is not used for
10411041 /// anything. Subclasses can use it to hold whatever state they find useful.
10421042 /// This field is initialized to zero by the ctor.
1043 unsigned short SubclassData : 15;
1043 uint16_t SubclassData : 15;
10441044
10451045 private:
10461046 /// NodeId - Unique id per SDNode in the DAG.
10831083 /// isTargetOpcode - Test if this node has a target-specific opcode (in the
10841084 /// \ISD namespace).
10851085 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
1086
1087 /// isTargetMemoryOpcode - Test if this node has a target-specific
1088 /// memory-referencing opcode (in the \ISD namespace and
1089 /// greater than FIRST_TARGET_MEMORY_OPCODE).
1090 bool isTargetMemoryOpcode() const {
1091 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
1092 }
10861093
10871094 /// isMachineOpcode - Test if this node has a post-isel opcode, directly
10881095 /// corresponding to a MachineInstr opcode.
14161423 inline bool SDValue::isTargetOpcode() const {
14171424 return Node->isTargetOpcode();
14181425 }
1426 inline bool SDValue::isTargetMemoryOpcode() const {
1427 return Node->isTargetMemoryOpcode();
1428 }
14191429 inline bool SDValue::isMachineOpcode() const {
14201430 return Node->isMachineOpcode();
14211431 }
15141524 // MemoryVT - VT of in-memory value.
15151525 EVT MemoryVT;
15161526
1517 //! SrcValue - Memory location for alias analysis.
1518 const Value *SrcValue;
1519
1520 //! SVOffset - Memory location offset. Note that base is defined in MemSDNode
1521 int SVOffset;
1527 protected:
1528 /// MMO - Memory reference information.
1529 MachineMemOperand *MMO;
1530
15221531 public:
15231532 MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT,
1524 const Value *srcValue, int SVOff, unsigned alignment,
1525 bool isvolatile);
1533 MachineMemOperand *MMO);
15261534
15271535 MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops,
1528 unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVOff,
1529 unsigned alignment, bool isvolatile);
1536 unsigned NumOps, EVT MemoryVT, MachineMemOperand *MMO);
1537
1538 bool readMem() const { return MMO->isLoad(); }
1539 bool writeMem() const { return MMO->isStore(); }
15301540
15311541 /// Returns alignment and volatility of the memory access
15321542 unsigned getOriginalAlignment() const {
1533 return (1u << (SubclassData >> 6)) >> 1;
1543 return MMO->getBaseAlignment();
15341544 }
15351545 unsigned getAlignment() const {
1536 return MinAlign(getOriginalAlignment(), SVOffset);
1537 }
1538 bool isVolatile() const { return (SubclassData >> 5) & 1; }
1546 return MMO->getAlignment();
1547 }
15391548
15401549 /// getRawSubclassData - Return the SubclassData value, which contains an
1541 /// encoding of the alignment and volatile information, as well as bits
1542 /// used by subclasses. This function should only be used to compute a
1543 /// FoldingSetNodeID value.
1550 /// encoding of the volatile flag, as well as bits used by subclasses. This
1551 /// function should only be used to compute a FoldingSetNodeID value.
15441552 unsigned getRawSubclassData() const {
15451553 return SubclassData;
15461554 }
15471555
1556 bool isVolatile() const { return (SubclassData >> 5) & 1; }
1557
15481558 /// Returns the SrcValue and offset that describes the location of the access
1549 const Value *getSrcValue() const { return SrcValue; }
1550 int getSrcValueOffset() const { return SVOffset; }
1559 const Value *getSrcValue() const { return MMO->getValue(); }
1560 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
15511561
15521562 /// getMemoryVT - Return the type of the in-memory value.
15531563 EVT getMemoryVT() const { return MemoryVT; }
15541564
15551565 /// getMemOperand - Return a MachineMemOperand object describing the memory
15561566 /// reference performed by operation.
1557 MachineMemOperand getMemOperand() const;
1567 MachineMemOperand *getMemOperand() const { return MMO; }
1568
1569 /// refineAlignment - Update this MemSDNode's MachineMemOperand information
1570 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1571 /// This must only be used when the new alignment applies to all users of
1572 /// this MachineMemOperand.
1573 void refineAlignment(const MachineMemOperand *NewMMO) {
1574 MMO->refineAlignment(NewMMO);
1575 }
15581576
15591577 const SDValue &getChain() const { return getOperand(0); }
15601578 const SDValue &getBasePtr() const {
15821600 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
15831601 N->getOpcode() == ISD::INTRINSIC_W_CHAIN ||
15841602 N->getOpcode() == ISD::INTRINSIC_VOID ||
1585 N->isTargetOpcode();
1603 N->isTargetMemoryOpcode();
15861604 }
15871605 };
15881606
16021620 // Align: alignment of memory
16031621 AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
16041622 SDValue Chain, SDValue Ptr,
1605 SDValue Cmp, SDValue Swp, const Value* SrcVal,
1606 unsigned Align=0)
1607 : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0,
1608 Align, /*isVolatile=*/true) {
1623 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
1624 : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
1625 assert(readMem() && "Atomic MachineMemOperand is not a load!");
1626 assert(writeMem() && "Atomic MachineMemOperand is not a store!");
16091627 InitOperands(Ops, Chain, Ptr, Cmp, Swp);
16101628 }
16111629 AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
16121630 SDValue Chain, SDValue Ptr,
1613 SDValue Val, const Value* SrcVal, unsigned Align=0)
1614 : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0,
1615 Align, /*isVolatile=*/true) {
1631 SDValue Val, MachineMemOperand *MMO)
1632 : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
1633 assert(readMem() && "Atomic MachineMemOperand is not a load!");
1634 assert(writeMem() && "Atomic MachineMemOperand is not a store!");
16161635 InitOperands(Ops, Chain, Ptr, Val);
16171636 }
16181637
16421661 }
16431662 };
16441663
1645 /// MemIntrinsicSDNode - This SDNode is used for target intrinsic that touches
1646 /// memory and need an associated memory operand.
1647 ///
1664 /// MemIntrinsicSDNode - This SDNode is used for target intrinsics that touch
1665 /// memory and need an associated MachineMemOperand. Its opcode may be
1666 /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, or a target-specific opcode with a
1667 /// value not less than FIRST_TARGET_MEMORY_OPCODE.
16481668 class MemIntrinsicSDNode : public MemSDNode {
1649 bool ReadMem; // Intrinsic reads memory
1650 bool WriteMem; // Intrinsic writes memory
16511669 public:
16521670 MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs,
16531671 const SDValue *Ops, unsigned NumOps,
1654 EVT MemoryVT, const Value *srcValue, int SVO,
1655 unsigned Align, bool Vol, bool ReadMem, bool WriteMem)
1656 : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol),
1657 ReadMem(ReadMem), WriteMem(WriteMem) {
1658 }
1659
1660 bool readMem() const { return ReadMem; }
1661 bool writeMem() const { return WriteMem; }
1672 EVT MemoryVT, MachineMemOperand *MMO)
1673 : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) {
1674 }
16621675
16631676 // Methods to support isa and dyn_cast
16641677 static bool classof(const MemIntrinsicSDNode *) { return true; }
16671680 // early a node with a target opcode can be of this class
16681681 return N->getOpcode() == ISD::INTRINSIC_W_CHAIN ||
16691682 N->getOpcode() == ISD::INTRINSIC_VOID ||
1670 N->isTargetOpcode();
1683 N->isTargetMemoryOpcode();
16711684 }
16721685 };
16731686
19551968 /// used when the SelectionDAG needs to make a simple reference to something
19561969 /// in the LLVM IR representation.
19571970 ///
1958 /// Note that this is not used for carrying alias information; that is done
1959 /// with MemOperandSDNode, which includes a Value which is required to be a
1960 /// pointer, and several other fields specific to memory references.
1961 ///
19621971 class SrcValueSDNode : public SDNode {
19631972 const Value *V;
19641973 friend class SelectionDAG;
19741983 static bool classof(const SrcValueSDNode *) { return true; }
19751984 static bool classof(const SDNode *N) {
19761985 return N->getOpcode() == ISD::SRCVALUE;
1977 }
1978 };
1979
1980
1981 /// MemOperandSDNode - An SDNode that holds a MachineMemOperand. This is
1982 /// used to represent a reference to memory after ISD::LOAD
1983 /// and ISD::STORE have been lowered.
1984 ///
1985 class MemOperandSDNode : public SDNode {
1986 friend class SelectionDAG;
1987 /// Create a MachineMemOperand node
1988 explicit MemOperandSDNode(const MachineMemOperand &mo)
1989 : SDNode(ISD::MEMOPERAND, DebugLoc::getUnknownLoc(),
1990 getSDVTList(MVT::Other)), MO(mo) {}
1991
1992 public:
1993 /// MO - The contained MachineMemOperand.
1994 const MachineMemOperand MO;
1995
1996 static bool classof(const MemOperandSDNode *) { return true; }
1997 static bool classof(const SDNode *N) {
1998 return N->getOpcode() == ISD::MEMOPERAND;
19991986 }
20001987 };
20011988
22682255 public:
22692256 LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands,
22702257 unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM,
2271 EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol)
2272 : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol) {
2273 assert(Align != 0 && "Loads and stores should have non-zero aligment");
2258 EVT MemVT, MachineMemOperand *MMO)
2259 : MemSDNode(NodeTy, dl, VTs, MemVT, MMO) {
22742260 SubclassData |= AM << 2;
22752261 assert(getAddressingMode() == AM && "MemIndexedMode encoding error!");
22762262 InitOperands(Ops, Operands, numOperands);
23062292 class LoadSDNode : public LSBaseSDNode {
23072293 friend class SelectionDAG;
23082294 LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs,
2309 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT LVT,
2310 const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
2295 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2296 MachineMemOperand *MMO)
23112297 : LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3,
2312 VTs, AM, LVT, SV, O, Align, Vol) {
2298 VTs, AM, MemVT, MMO) {
23132299 SubclassData |= (unsigned short)ETy;
23142300 assert(getExtensionType() == ETy && "LoadExtType encoding error!");
2301 assert(readMem() && "Load MachineMemOperand is not a load!");
2302 assert(!writeMem() && "Load MachineMemOperand is a store!");
23152303 }
23162304 public:
23172305
23352323 class StoreSDNode : public LSBaseSDNode {
23362324 friend class SelectionDAG;
23372325 StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs,
2338 ISD::MemIndexedMode AM, bool isTrunc, EVT SVT,
2339 const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
2326 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2327 MachineMemOperand *MMO)
23402328 : LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4,
2341 VTs, AM, SVT, SV, O, Align, Vol) {
2329 VTs, AM, MemVT, MMO) {
23422330 SubclassData |= (unsigned short)isTrunc;
23432331 assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!");
2332 assert(!readMem() && "Store MachineMemOperand is a load!");
2333 assert(writeMem() && "Store MachineMemOperand is not a store!");
23442334 }
23452335 public:
23462336
23592349 }
23602350 };
23612351
2352 /// MachineSDNode - An SDNode that represents everything that will be needed
2353 /// to construct a MachineInstr. These nodes are created during the
2354 /// instruction selection proper phase.
2355 ///
2356 class MachineSDNode : public SDNode {
2357 public:
2358 typedef MachineMemOperand **mmo_iterator;
2359
2360 private:
2361 friend class SelectionDAG;
2362 MachineSDNode(unsigned Opc, const DebugLoc DL, SDVTList VTs)
2363 : SDNode(Opc, DL, VTs), MemRefs(0), MemRefsEnd(0) {}
2364
2365 /// LocalOperands - Operands for this instruction, if they fit here. If
2366 /// they don't, this field is unused.
2367 SDUse LocalOperands[4];
2368
2369 /// MemRefs - Memory reference descriptions for this instruction.
2370 mmo_iterator MemRefs;
2371 mmo_iterator MemRefsEnd;
2372
2373 public:
2374 mmo_iterator memoperands_begin() const { return MemRefs; }
2375 mmo_iterator memoperands_end() const { return MemRefsEnd; }
2376 bool memoperands_empty() const { return MemRefsEnd == MemRefs; }
2377
2378 /// setMemRefs - Assign this MachineSDNodes's memory reference descriptor
2379 /// list. This does not transfer ownership.
2380 void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
2381 MemRefs = NewMemRefs;
2382 MemRefsEnd = NewMemRefsEnd;
2383 }
2384
2385 static bool classof(const MachineSDNode *) { return true; }
2386 static bool classof(const SDNode *N) {
2387 return N->isMachineOpcode();
2388 }
2389 };
23622390
23632391 class SDNodeIterator : public std::iterator
23642392 SDNode, ptrdiff_t> {
2424 #include "llvm/CodeGen/MachineInstr.h"
2525 #include "llvm/CodeGen/MachineInstrBuilder.h"
2626 #include "llvm/CodeGen/MachineLoopInfo.h"
27 #include "llvm/CodeGen/MachineMemOperand.h"
2728 #include "llvm/CodeGen/MachineRegisterInfo.h"
2829 #include "llvm/CodeGen/Passes.h"
2930 #include "llvm/CodeGen/PseudoSourceValue.h"
14421443
14431444 // If the instruction accesses memory and the memory could be non-constant,
14441445 // assume the instruction is not rematerializable.
1445 for (std::list::const_iterator
1446 I = MI->memoperands_begin(), E = MI->memoperands_end(); I != E; ++I){
1447 const MachineMemOperand &MMO = *I;
1448 if (MMO.isVolatile() || MMO.isStore())
1446 for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
1447 E = MI->memoperands_end(); I != E; ++I){
1448 const MachineMemOperand *MMO = *I;
1449 if (MMO->isVolatile() || MMO->isStore())
14491450 return false;
1450 const Value *V = MMO.getValue();
1451 const Value *V = MMO->getValue();
14511452 if (!V)
14521453 return false;
14531454 if (const PseudoSourceValue *PSV = dyn_cast(V)) {
189189 ///
190190 void
191191 MachineFunction::DeleteMachineInstr(MachineInstr *MI) {
192 // Clear the instructions memoperands. This must be done manually because
193 // the instruction's parent pointer is now null, so it can't properly
194 // deallocate them on its own.
195 MI->clearMemOperands(*this);
196
197192 MI->~MachineInstr();
198193 InstructionRecycler.Deallocate(Allocator, MI);
199194 }
214209 assert(MBB->getParent() == this && "MBB parent mismatch!");
215210 MBB->~MachineBasicBlock();
216211 BasicBlockRecycler.Deallocate(Allocator, MBB);
212 }
213
214 MachineMemOperand *
215 MachineFunction::getMachineMemOperand(const Value *v, unsigned f,
216 int64_t o, uint64_t s,
217 unsigned base_alignment) {
218 return new (Allocator.Allocate())
219 MachineMemOperand(v, f, o, s, base_alignment);
220 }
221
222 MachineMemOperand *
223 MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
224 int64_t Offset, uint64_t Size) {
225 return new (Allocator.Allocate())
226 MachineMemOperand(MMO->getValue(), MMO->getFlags(),
227 int64_t(uint64_t(MMO->getOffset()) +
228 uint64_t(Offset)),
229 Size, MMO->getBaseAlignment());
230 }
231
232 MachineInstr::mmo_iterator
233 MachineFunction::allocateMemRefsArray(unsigned long Num) {
234 return Allocator.Allocate(Num);
217235 }
218236
219237 void MachineFunction::dump() const {
1616 #include "llvm/Value.h"
1717 #include "llvm/Assembly/Writer.h"
1818 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineMemOperand.h"
1920 #include "llvm/CodeGen/MachineRegisterInfo.h"
2021 #include "llvm/CodeGen/PseudoSourceValue.h"
2122 #include "llvm/Target/TargetMachine.h"
297298 ID.AddInteger(Flags);
298299 }
299300
300 raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MRO) {
301 assert((MRO.isLoad() || MRO.isStore()) &&
301 void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
302 // The Value and Offset may differ due to CSE. But the flags and size
303 // should be the same.
304 assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
305 assert(MMO->getSize() == getSize() && "Size mismatch!");
306
307 if (MMO->getBaseAlignment() >= getBaseAlignment()) {
308 // Update the alignment value.
309 Flags = (Flags & 7) | ((Log2_32(MMO->getBaseAlignment()) + 1) << 3);
310 // Also update the base and offset, because the new alignment may
311 // not be applicable with the old ones.
312 V = MMO->getValue();
313 Offset = MMO->getOffset();
314 }
315 }
316
317 raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
318 assert((MMO.isLoad() || MMO.isStore()) &&
302319 "SV has to be a load, store or both.");
303320
304 if (MRO.isVolatile())
321 if (MMO.isVolatile())
305322 OS << "Volatile ";
306323
307 if (MRO.isLoad())
324 if (MMO.isLoad())
308325 OS << "LD";
309 if (MRO.isStore())
326 if (MMO.isStore())
310327 OS << "ST";
311 OS << MRO.getSize();
328 OS << MMO.getSize();
312329
313330 // Print the address information.
314331 OS << "[";
315 if (!MRO.getValue())
332 if (!MMO.getValue())
316333 OS << "";
317334 else
318 WriteAsOperand(OS, MRO.getValue(), /*PrintType=*/false);
335 WriteAsOperand(OS, MMO.getValue(), /*PrintType=*/false);
319336
320337 // If the alignment of the memory reference itself differs from the alignment
321338 // of the base pointer, print the base alignment explicitly, next to the base
322339 // pointer.
323 if (MRO.getBaseAlignment() != MRO.getAlignment())
324 OS << "(align=" << MRO.getBaseAlignment() << ")";
325
326 if (MRO.getOffset() != 0)
327 OS << "+" << MRO.getOffset();
340 if (MMO.getBaseAlignment() != MMO.getAlignment())
341 OS << "(align=" << MMO.getBaseAlignment() << ")";
342
343 if (MMO.getOffset() != 0)
344 OS << "+" << MMO.getOffset();
328345 OS << "]";
329346
330347 // Print the alignment of the reference.
331 if (MRO.getBaseAlignment() != MRO.getAlignment() ||
332 MRO.getBaseAlignment() != MRO.getSize())
333 OS << "(align=" << MRO.getAlignment() << ")";
348 if (MMO.getBaseAlignment() != MMO.getAlignment() ||
349 MMO.getBaseAlignment() != MMO.getSize())
350 OS << "(align=" << MMO.getAlignment() << ")";
334351
335352 return OS;
336353 }
342359 /// MachineInstr ctor - This constructor creates a dummy MachineInstr with
343360 /// TID NULL and no operands.
344361 MachineInstr::MachineInstr()
345 : TID(0), NumImplicitOps(0), Parent(0), debugLoc(DebugLoc::getUnknownLoc()) {
362 : TID(0), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
363 Parent(0), debugLoc(DebugLoc::getUnknownLoc()) {
346364 // Make sure that we get added to a machine basicblock
347365 LeakDetector::addGarbageObject(this);
348366 }
361379 /// TargetInstrDesc or the numOperands if it is not zero. (for
362380 /// instructions with variable number of operands).
363381 MachineInstr::MachineInstr(const TargetInstrDesc &tid, bool NoImp)
364 : TID(&tid), NumImplicitOps(0), Parent(0),
382 : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0), Parent(0),
365383 debugLoc(DebugLoc::getUnknownLoc()) {
366384 if (!NoImp && TID->getImplicitDefs())
367385 for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
379397 /// MachineInstr ctor - As above, but with a DebugLoc.
380398 MachineInstr::MachineInstr(const TargetInstrDesc &tid, const DebugLoc dl,
381399 bool NoImp)
382 : TID(&tid), NumImplicitOps(0), Parent(0), debugLoc(dl) {
400 : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
401 Parent(0), debugLoc(dl) {
383402 if (!NoImp && TID->getImplicitDefs())
384403 for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
385404 NumImplicitOps++;
398417 /// basic block.
399418 ///
400419 MachineInstr::MachineInstr(MachineBasicBlock *MBB, const TargetInstrDesc &tid)
401 : TID(&tid), NumImplicitOps(0), Parent(0),
420 : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0), Parent(0),
402421 debugLoc(DebugLoc::getUnknownLoc()) {
403422 assert(MBB && "Cannot use inserting ctor with null basic block!");
404423 if (TID->ImplicitDefs)
418437 ///
419438 MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl,
420439 const TargetInstrDesc &tid)
421 : TID(&tid), NumImplicitOps(0), Parent(0), debugLoc(dl) {
440 : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
441 Parent(0), debugLoc(dl) {
422442 assert(MBB && "Cannot use inserting ctor with null basic block!");
423443 if (TID->ImplicitDefs)
424444 for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
436456 /// MachineInstr ctor - Copies MachineInstr arg exactly
437457 ///
438458 MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
439 : TID(&MI.getDesc()), NumImplicitOps(0), Parent(0),
440 debugLoc(MI.getDebugLoc()) {
459 : TID(&MI.getDesc()), NumImplicitOps(0),
460 MemRefs(MI.MemRefs), MemRefsEnd(MI.MemRefsEnd),
461 Parent(0), debugLoc(MI.getDebugLoc()) {
441462 Operands.reserve(MI.getNumOperands());
442463
443464 // Add operands
445466 addOperand(MI.getOperand(i));
446467 NumImplicitOps = MI.NumImplicitOps;
447468
448 // Add memory operands.
449 for (std::list::const_iterator i = MI.memoperands_begin(),
450 j = MI.memoperands_end(); i != j; ++i)
451 addMemOperand(MF, *i);
452
453469 // Set parent to null.
454470 Parent = 0;
455471
458474
459475 MachineInstr::~MachineInstr() {
460476 LeakDetector::removeGarbageObject(this);
461 assert(MemOperands.empty() &&
462 "MachineInstr being deleted with live memoperands!");
463477 #ifndef NDEBUG
464478 for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
465479 assert(Operands[i].ParentMI == this && "ParentMI mismatch!");
620634 }
621635 }
622636
623 /// addMemOperand - Add a MachineMemOperand to the machine instruction,
624 /// referencing arbitrary storage.
637 /// addMemOperand - Add a MachineMemOperand to the machine instruction.
638 /// This function should be used only occasionally. The setMemRefs function
639 /// is the primary method for setting up a MachineInstr's MemRefs list.
625640 void MachineInstr::addMemOperand(MachineFunction &MF,
626 const MachineMemOperand &MO) {
627 MemOperands.push_back(MO);
628 }
629
630 /// clearMemOperands - Erase all of this MachineInstr's MachineMemOperands.
631 void MachineInstr::clearMemOperands(MachineFunction &MF) {
632 MemOperands.clear();
633 }
634
641 MachineMemOperand *MO) {
642 mmo_iterator OldMemRefs = MemRefs;
643 mmo_iterator OldMemRefsEnd = MemRefsEnd;
644
645 size_t NewNum = (MemRefsEnd - MemRefs) + 1;
646 mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NewNum);
647 mmo_iterator NewMemRefsEnd = NewMemRefs + NewNum;
648
649 std::copy(OldMemRefs, OldMemRefsEnd, NewMemRefs);
650 NewMemRefs[NewNum - 1] = MO;
651
652 MemRefs = NewMemRefs;
653 MemRefsEnd = NewMemRefsEnd;
654 }
635655
636656 /// removeFromParent - This method unlinks 'this' from the containing basic
637657 /// block, and returns it, but does not delete it.
971991 return true;
972992
973993 // Check the memory reference information for volatile references.
974 for (std::list::const_iterator I = memoperands_begin(),
975 E = memoperands_end(); I != E; ++I)
976 if (I->isVolatile())
994 for (mmo_iterator I = memoperands_begin(), E = memoperands_end(); I != E; ++I)
995 if ((*I)->isVolatile())
977996 return true;
978997
979998 return false;
10031022
10041023 if (!memoperands_empty()) {
10051024 OS << ", Mem:";
1006 for (std::list::const_iterator i = memoperands_begin(),
1007 e = memoperands_end(); i != e; ++i) {
1008 OS << *i;
1025 for (mmo_iterator i = memoperands_begin(), e = memoperands_end();
1026 i != e; ++i) {
1027 OS << **i;
10091028 if (next(i) != e)
10101029 OS << " ";
10111030 }
2626 #include "llvm/Support/Debug.h"
2727 #include "llvm/Support/MathExtras.h"
2828 using namespace llvm;
29
30 void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) {
31 MI->addMemOperand(MF, MO);
32 }
3329
3430 void ScheduleDAG::EmitNoop() {
3531 TII->insertNoop(*BB, InsertPos);
1616 #include "llvm/Operator.h"
1717 #include "llvm/Analysis/AliasAnalysis.h"
1818 #include "llvm/CodeGen/MachineFunctionPass.h"
19 #include "llvm/CodeGen/MachineMemOperand.h"
1920 #include "llvm/CodeGen/MachineRegisterInfo.h"
2021 #include "llvm/CodeGen/PseudoSourceValue.h"
2122 #include "llvm/Target/TargetMachine.h"
9596 /// object, return the Value for that object. Otherwise return null.
9697 static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI) {
9798 if (!MI->hasOneMemOperand() ||
98 !MI->memoperands_begin()->getValue() ||
99 MI->memoperands_begin()->isVolatile())
99 !(*MI->memoperands_begin())->getValue() ||
100 (*MI->memoperands_begin())->isVolatile())
100101 return 0;
101102
102 const Value *V = MI->memoperands_begin()->getValue();
103 const Value *V = (*MI->memoperands_begin())->getValue();
103104 if (!V)
104105 return 0;
105106
334335 if (!ChainTID.isCall() &&
335336 !ChainTID.hasUnmodeledSideEffects() &&
336337 ChainMI->hasOneMemOperand() &&
337 !ChainMI->memoperands_begin()->isVolatile() &&
338 ChainMI->memoperands_begin()->getValue())
338 !(*ChainMI->memoperands_begin())->isVolatile() &&
339 (*ChainMI->memoperands_begin())->getValue())
339340 // We know that the Chain accesses one specific memory location.
340 ChainMMO = &*ChainMI->memoperands_begin();
341 ChainMMO = *ChainMI->memoperands_begin();
341342 else
342343 // Unknown memory accesses. Assume the worst.
343344 ChainMMO = 0;
262262 }
263263
264264 /// CountOperands - The inputs to target nodes have any actual inputs first,
265 /// followed by special operands that describe memory references, then an
266 /// optional chain operand, then an optional flag operand. Compute the number
267 /// of actual operands that will go into the resulting MachineInstr.
265 /// followed by an optional chain operand, then an optional flag operand.
266 /// Compute the number of actual operands that will go into the resulting
267 /// MachineInstr.
268268 unsigned ScheduleDAGSDNodes::CountOperands(SDNode *Node) {
269 unsigned N = ComputeMemOperandsEnd(Node);
270 while (N && isa(Node->getOperand(N - 1).getNode()))
271 --N; // Ignore MEMOPERAND nodes
272 return N;
273 }
274
275 /// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode
276 /// operand
277 unsigned ScheduleDAGSDNodes::ComputeMemOperandsEnd(SDNode *Node) {
278269 unsigned N = Node->getNumOperands();
279270 while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag)
280271 --N;
282273 --N; // Ignore chain if it exists.
283274 return N;
284275 }
285
286276
287277 void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const {
288278 if (!SU->getNode()) {
5757 if (isa(Node)) return true;
5858 if (isa(Node)) return true;
5959 if (isa(Node)) return true;
60 if (isa(Node)) return true;
6160 if (Node->getOpcode() == ISD::EntryToken) return true;
6261 return false;
6362 }
9897 static unsigned CountResults(SDNode *Node);
9998
10099 /// CountOperands - The inputs to target nodes have any actual inputs first,
101 /// followed by special operands that describe memory references, then an
102 /// optional chain operand, then flag operands. Compute the number of
103 /// actual operands that will go into the resulting MachineInstr.
100 /// followed by an optional chain operand, then flag operands. Compute
101 /// the number of actual operands that will go into the resulting
102 /// MachineInstr.
104103 static unsigned CountOperands(SDNode *Node);
105
106 /// ComputeMemOperandsEnd - Find the index one past the last
107 /// MemOperandSDNode operand
108 static unsigned ComputeMemOperandsEnd(SDNode *Node);
109104
110105 /// EmitNode - Generate machine code for an node and needed dependencies.
111106 /// VRBaseMap contains, for each already emitted node, the first virtual
496496 const TargetInstrDesc &II = TII->get(Opc);
497497 unsigned NumResults = CountResults(Node);
498498 unsigned NodeOperands = CountOperands(Node);
499 unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node);
500499 bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
501500 II.getImplicitDefs() != 0;
502501 #ifndef NDEBUG
524523 AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
525524 VRBaseMap);
526525
527 // Emit all of the memory operands of this instruction
528 for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i)
529 AddMemOperand(MI,cast(Node->getOperand(i+NumSkip))->MO);
526 // Transfer all of the memory reference descriptions of this instruction.
527 MI->setMemRefs(cast(Node)->memoperands_begin(),
528 cast(Node)->memoperands_end());
530529
531530 if (II.usesCustomDAGSchedInsertionHook()) {
532531 // Insert this instruction into the basic block using a target
401401 case ISD::SRCVALUE:
402402 ID.AddPointer(cast(N)->getValue());
403403 break;
404 case ISD::MEMOPERAND: {
405 const MachineMemOperand &MO = cast(N)->MO;
406 MO.Profile(ID);
407 break;
408 }
409404 case ISD::FrameIndex:
410405 case ISD::TargetFrameIndex:
411406 ID.AddInteger(cast(N)->getIndex());
480475 }
481476
482477 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
483 /// the CSE map that carries alignment, volatility, indexing mode, and
478 /// the CSE map that carries volatility, indexing mode, and
484479 /// extension/truncation information.
485480 ///
486481 static inline unsigned
487 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM,
488 bool isVolatile, unsigned Alignment) {
482 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile) {
489483 assert((ConvType & 3) == ConvType &&
490484 "ConvType may not require more than 2 bits!");
491485 assert((AM & 7) == AM &&
492486 "AM may not require more than 3 bits!");
493487 return ConvType |
494488 (AM << 2) |
495 (isVolatile << 5) |
496 ((Log2_32(Alignment) + 1) << 6);
489 (isVolatile << 5);
497490 }
498491
499492 //===----------------------------------------------------------------------===//
13241317
13251318 SDNode *N = NodeAllocator.Allocate();
13261319 new (N) SrcValueSDNode(V);
1327 CSEMap.InsertNode(N, IP);
1328 AllNodes.push_back(N);
1329 return SDValue(N, 0);
1330 }
1331
1332 SDValue SelectionDAG::getMemOperand(const MachineMemOperand &MO) {
1333 #ifndef NDEBUG
1334 const Value *v = MO.getValue();
1335 assert((!v || isa(v->getType())) &&
1336 "SrcValue is not a pointer?");
1337 #endif
1338
1339 FoldingSetNodeID ID;
1340 AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), 0, 0);
1341 MO.Profile(ID);
1342
1343 void *IP = 0;
1344 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1345 return SDValue(E, 0);
1346
1347 SDNode *N = NodeAllocator.Allocate();
1348 new (N) MemOperandSDNode(MO);
13491320 CSEMap.InsertNode(N, IP);
13501321 AllNodes.push_back(N);
13511322 return SDValue(N, 0);
35223493 SDValue Ptr, SDValue Cmp,
35233494 SDValue Swp, const Value* PtrVal,
35243495 unsigned Alignment) {
3496 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3497 Alignment = getEVTAlignment(MemVT);
3498
3499 // Check if the memory reference references a frame index
3500 if (!PtrVal)
3501 if (const FrameIndexSDNode *FI =
3502 dyn_cast(Ptr.getNode()))
3503 PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex());
3504
3505 MachineFunction &MF = getMachineFunction();
3506 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
3507
3508 // For now, atomics are considered to be volatile always.
3509 Flags |= MachineMemOperand::MOVolatile;
3510
3511 MachineMemOperand *MMO =
3512 MF.getMachineMemOperand(PtrVal, Flags, 0,
3513 MemVT.getStoreSize(), Alignment);
3514
3515 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO);
3516 }
3517
3518 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
3519 SDValue Chain,
3520 SDValue Ptr, SDValue Cmp,
3521 SDValue Swp, MachineMemOperand *MMO) {
35253522 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
35263523 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
35273524
35283525 EVT VT = Cmp.getValueType();
3529
3530 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3531 Alignment = getEVTAlignment(MemVT);
35323526
35333527 SDVTList VTs = getVTList(VT, MVT::Other);
35343528 FoldingSetNodeID ID;
35363530 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
35373531 AddNodeIDNode(ID, Opcode, VTs, Ops, 4);
35383532 void* IP = 0;
3539 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3533 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3534 cast(E)->refineAlignment(MMO);
35403535 return SDValue(E, 0);
3536 }
35413537 SDNode* N = NodeAllocator.Allocate();
3542 new (N) AtomicSDNode(Opcode, dl, VTs, MemVT,
3543 Chain, Ptr, Cmp, Swp, PtrVal, Alignment);
3538 new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO);
35443539 CSEMap.InsertNode(N, IP);
35453540 AllNodes.push_back(N);
35463541 return SDValue(N, 0);
35513546 SDValue Ptr, SDValue Val,
35523547 const Value* PtrVal,
35533548 unsigned Alignment) {
3549 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3550 Alignment = getEVTAlignment(MemVT);
3551
3552 // Check if the memory reference references a frame index
3553 if (!PtrVal)
3554 if (const FrameIndexSDNode *FI =
3555 dyn_cast(Ptr.getNode()))
3556 PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex());
3557
3558 MachineFunction &MF = getMachineFunction();
3559 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
3560
3561 // For now, atomics are considered to be volatile always.
3562 Flags |= MachineMemOperand::MOVolatile;
3563
3564 MachineMemOperand *MMO =
3565 MF.getMachineMemOperand(PtrVal, Flags, 0,
3566 MemVT.getStoreSize(), Alignment);
3567
3568 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
3569 }
3570
3571 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
3572 SDValue Chain,
3573 SDValue Ptr, SDValue Val,
3574 MachineMemOperand *MMO) {
35543575 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
35553576 Opcode == ISD::ATOMIC_LOAD_SUB ||
35563577 Opcode == ISD::ATOMIC_LOAD_AND ||
35663587
35673588 EVT VT = Val.getValueType();
35683589
3569 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3570 Alignment = getEVTAlignment(MemVT);
3571
35723590 SDVTList VTs = getVTList(VT, MVT::Other);
35733591 FoldingSetNodeID ID;
35743592 ID.AddInteger(MemVT.getRawBits());
35753593 SDValue Ops[] = {Chain, Ptr, Val};
35763594 AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
35773595 void* IP = 0;
3578 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3596 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3597 cast(E)->refineAlignment(MMO);
35793598 return SDValue(E, 0);
3599 }
35803600 SDNode* N = NodeAllocator.Allocate();
3581 new (N) AtomicSDNode(Opcode, dl, VTs, MemVT,
3582 Chain, Ptr, Val, PtrVal, Alignment);
3601 new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Val, MMO);
35833602 CSEMap.InsertNode(N, IP);
35843603 AllNodes.push_back(N);
35853604 return SDValue(N, 0);
36183637 EVT MemVT, const Value *srcValue, int SVOff,
36193638 unsigned Align, bool Vol,
36203639 bool ReadMem, bool WriteMem) {
3640 if (Align == 0) // Ensure that codegen never sees alignment 0
3641 Align = getEVTAlignment(MemVT);
3642
3643 MachineFunction &MF = getMachineFunction();
3644 unsigned Flags = 0;
3645 if (WriteMem)
3646 Flags |= MachineMemOperand::MOStore;
3647 if (ReadMem)
3648 Flags |= MachineMemOperand::MOLoad;
3649 if (Vol)
3650 Flags |= MachineMemOperand::MOVolatile;
3651 MachineMemOperand *MMO =
3652 MF.getMachineMemOperand(srcValue, Flags, SVOff,
3653 MemVT.getStoreSize(), Align);
3654
3655 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
3656 }
3657
3658 SDValue
3659 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
3660 const SDValue *Ops, unsigned NumOps,
3661 EVT MemVT, MachineMemOperand *MMO) {
3662 assert((Opcode == ISD::INTRINSIC_VOID ||
3663 Opcode == ISD::INTRINSIC_W_CHAIN ||
3664 (Opcode <= INT_MAX &&
3665 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
3666 "Opcode is not a memory-accessing opcode!");
3667
36213668 // Memoize the node unless it returns a flag.
36223669 MemIntrinsicSDNode *N;
36233670 if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) {
36243671 FoldingSetNodeID ID;
36253672 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
36263673 void *IP = 0;
3627 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3674 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3675 cast(E)->refineAlignment(MMO);
36283676 return SDValue(E, 0);
3677 }
36293678
36303679 N = NodeAllocator.Allocate();
3631 new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT,
3632 srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
3680 new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
36333681 CSEMap.InsertNode(N, IP);
36343682 } else {
36353683 N = NodeAllocator.Allocate();
3636 new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT,
3637 srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
3684 new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
36383685 }
36393686 AllNodes.push_back(N);
36403687 return SDValue(N, 0);
36493696 if (Alignment == 0) // Ensure that codegen never sees alignment 0
36503697 Alignment = getEVTAlignment(VT);
36513698
3699 // Check if the memory reference references a frame index
3700 if (!SV)
3701 if (const FrameIndexSDNode *FI =
3702 dyn_cast(Ptr.getNode()))
3703 SV = PseudoSourceValue::getFixedStack(FI->getIndex());
3704
3705 MachineFunction &MF = getMachineFunction();
3706 unsigned Flags = MachineMemOperand::MOLoad;
3707 if (isVolatile)
3708 Flags |= MachineMemOperand::MOVolatile;
3709 MachineMemOperand *MMO =
3710 MF.getMachineMemOperand(SV, Flags, SVOffset,
3711 MemVT.getStoreSize(), Alignment);
3712 return getLoad(AM, dl, ExtType, VT, Chain, Ptr, Offset, MemVT, MMO);
3713 }
3714
3715 SDValue
3716 SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl,
3717 ISD::LoadExtType ExtType, EVT VT, SDValue Chain,
3718 SDValue Ptr, SDValue Offset, EVT MemVT,
3719 MachineMemOperand *MMO) {
36523720 if (VT == MemVT) {
36533721 ExtType = ISD::NON_EXTLOAD;
36543722 } else if (ExtType == ISD::NON_EXTLOAD) {
36773745 FoldingSetNodeID ID;
36783746 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
36793747 ID.AddInteger(MemVT.getRawBits());
3680 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment));
3748 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile()));
36813749 void *IP = 0;
3682 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3750 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3751 cast(E)->refineAlignment(MMO);
36833752 return SDValue(E, 0);
3753 }
36843754 SDNode *N = NodeAllocator.Allocate();
3685 new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, SV, SVOffset,
3686 Alignment, isVolatile);
3755 new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, MMO);
36873756 CSEMap.InsertNode(N, IP);
36883757 AllNodes.push_back(N);
36893758 return SDValue(N, 0);
37233792 SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
37243793 SDValue Ptr, const Value *SV, int SVOffset,
37253794 bool isVolatile, unsigned Alignment) {
3795 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3796 Alignment = getEVTAlignment(Val.getValueType());
3797
3798 // Check if the memory reference references a frame index
3799 if (!SV)
3800 if (const FrameIndexSDNode *FI =
3801 dyn_cast(Ptr.getNode()))
3802 SV = PseudoSourceValue::getFixedStack(FI->getIndex());
3803
3804 MachineFunction &MF = getMachineFunction();
3805 unsigned Flags = MachineMemOperand::MOStore;
3806 if (isVolatile)
3807 Flags |= MachineMemOperand::MOVolatile;
3808 MachineMemOperand *MMO =
3809 MF.getMachineMemOperand(SV, Flags, SVOffset,
3810 Val.getValueType().getStoreSize(), Alignment);
3811
3812 return getStore(Chain, dl, Val, Ptr, MMO);
3813 }
3814
3815 SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
3816 SDValue Ptr, MachineMemOperand *MMO) {
37263817 EVT VT = Val.getValueType();
3727
3728 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3729 Alignment = getEVTAlignment(VT);
3730
37313818 SDVTList VTs = getVTList(MVT::Other);
37323819 SDValue Undef = getUNDEF(Ptr.getValueType());
37333820 SDValue Ops[] = { Chain, Val, Ptr, Undef };
37343821 FoldingSetNodeID ID;
37353822 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
37363823 ID.AddInteger(VT.getRawBits());
3737 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED,
3738 isVolatile, Alignment));
3824 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile()));
37393825 void *IP = 0;
3740 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3826 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3827 cast(E)->refineAlignment(MMO);
37413828 return SDValue(E, 0);
3829 }
37423830 SDNode *N = NodeAllocator.Allocate();
3743 new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false,
3744 VT, SV, SVOffset, Alignment, isVolatile);
3831 new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, VT, MMO);
37453832 CSEMap.InsertNode(N, IP);
37463833 AllNodes.push_back(N);
37473834 return SDValue(N, 0);
37513838 SDValue Ptr, const Value *SV,
37523839 int SVOffset, EVT SVT,
37533840 bool isVolatile, unsigned Alignment) {
3841 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3842 Alignment = getEVTAlignment(SVT);
3843
3844 // Check if the memory reference references a frame index
3845 if (!SV)
3846 if (const FrameIndexSDNode *FI =
3847 dyn_cast(Ptr.getNode()))
3848 SV = PseudoSourceValue::getFixedStack(FI->getIndex());
3849
3850 MachineFunction &MF = getMachineFunction();
3851 unsigned Flags = MachineMemOperand::MOStore;
3852 if (isVolatile)
3853 Flags |= MachineMemOperand::MOVolatile;
3854 MachineMemOperand *MMO =
3855 MF.getMachineMemOperand(SV, Flags, SVOffset, SVT.getStoreSize(), Alignment);
3856
3857 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
3858 }
3859
3860 SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
3861 SDValue Ptr, EVT SVT,
3862 MachineMemOperand *MMO) {
37543863 EVT VT = Val.getValueType();
37553864
37563865 if (VT == SVT)
3757 return getStore(Chain, dl, Val, Ptr, SV, SVOffset, isVolatile, Alignment);
3866 return getStore(Chain, dl, Val, Ptr, MMO);
37583867
37593868 assert(VT.bitsGT(SVT) && "Not a truncation?");
37603869 assert(VT.isInteger() == SVT.isInteger() &&
37613870 "Can't do FP-INT conversion!");
37623871
3763 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3764 Alignment = getEVTAlignment(VT);
37653872
37663873 SDVTList VTs = getVTList(MVT::Other);
37673874 SDValue Undef = getUNDEF(Ptr.getValueType());
37693876 FoldingSetNodeID ID;
37703877 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
37713878 ID.AddInteger(SVT.getRawBits());
3772 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED,
3773 isVolatile, Alignment));
3879 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile()));
37743880 void *IP = 0;
3775 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3881 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3882 cast(E)->refineAlignment(MMO);
37763883 return SDValue(E, 0);
3884 }
37773885 SDNode *N = NodeAllocator.Allocate();
3778 new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true,
3779 SVT, SV, SVOffset, Alignment, isVolatile);
3886 new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true, SVT, MMO);
37803887 CSEMap.InsertNode(N, IP);
37813888 AllNodes.push_back(N);
37823889 return SDValue(N, 0);
38003907 SDNode *N = NodeAllocator.Allocate();
38013908 new (N) StoreSDNode(Ops, dl, VTs, AM,
38023909 ST->isTruncatingStore(), ST->getMemoryVT(),
3803 ST->getSrcValue(), ST->getSrcValueOffset(),
3804 ST->getAlignment(), ST->isVolatile());
3910 ST->getMemOperand());
38053911 CSEMap.InsertNode(N, IP);
38063912 AllNodes.push_back(N);
38073913 return SDValue(N, 0);
44534559 DeadNodeSet.insert(Used);
44544560 }
44554561
4456 // If NumOps is larger than the # of operands we currently have, reallocate
4457 // the operand list.
4458 if (NumOps > N->NumOperands) {
4459 if (N->OperandsNeedDelete)
4460 delete[] N->OperandList;
4461
4462 if (N->isMachineOpcode()) {
4463 // We're creating a final node that will live unmorphed for the
4464 // remainder of the current SelectionDAG iteration, so we can allocate
4465 // the operands directly out of a pool with no recycling metadata.
4466 N->OperandList = OperandAllocator.Allocate(NumOps);
4467 N->OperandsNeedDelete = false;
4468 } else {
4469 N->OperandList = new SDUse[NumOps];
4562 if (MachineSDNode *MN = dyn_cast(N)) {
4563 // Initialize the memory references information.
4564 MN->setMemRefs(0, 0);
4565 // If NumOps is larger than the # of operands we can have in a
4566 // MachineSDNode, reallocate the operand list.
4567 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
4568 if (MN->OperandsNeedDelete)
4569 delete[] MN->OperandList;
4570 if (NumOps > array_lengthof(MN->LocalOperands))
4571 // We're creating a final node that will live unmorphed for the
4572 // remainder of the current SelectionDAG iteration, so we can allocate
4573 // the operands directly out of a pool with no recycling metadata.
4574 MN->InitOperands(OperandAllocator.Allocate(NumOps),
4575 Ops, NumOps);
4576 else
4577 MN->InitOperands(MN->LocalOperands, Ops, NumOps);
4578 MN->OperandsNeedDelete = false;
4579 } else
4580 MN->InitOperands(MN->OperandList, Ops, NumOps);
4581 } else {
4582 // If NumOps is larger than the # of operands we currently have, reallocate
4583 // the operand list.
4584 if (NumOps > N->NumOperands) {
4585 if (N->OperandsNeedDelete)
4586 delete[] N->OperandList;
4587 N->InitOperands(new SDUse[NumOps], Ops, NumOps);
44704588 N->OperandsNeedDelete = true;
4471 }
4472 }
4473
4474 // Assign the new operands.
4475 N->NumOperands = NumOps;
4476 for (unsigned i = 0, e = NumOps; i != e; ++i) {
4477 N->OperandList[i].setUser(N);
4478 N->OperandList[i].setInitial(Ops[i]);
4589 } else
4590 MN->InitOperands(MN->OperandList, Ops, NumOps);
44794591 }
44804592
44814593 // Delete any nodes that are still dead after adding the uses for the
45004612 /// node of the specified opcode and operands, it returns that node instead of
45014613 /// the current one.
45024614 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) {
4503 return getNode(~Opcode, dl, VT).getNode();
4615 SDVTList VTs = getVTList(VT);
4616 return getMachineNode(Opcode, dl, VTs, 0, 0);
45044617 }
45054618
45064619 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
45074620 SDValue Op1) {
4508 return getNode(~Opcode, dl, VT, Op1).getNode();
4621 SDVTList VTs = getVTList(VT);
4622 SDValue Ops[] = { Op1 };
4623 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45094624 }
45104625
45114626 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
45124627 SDValue Op1, SDValue Op2) {
4513 return getNode(~Opcode, dl, VT, Op1, Op2).getNode();
4628 SDVTList VTs = getVTList(VT);
4629 SDValue Ops[] = { Op1, Op2 };
4630 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45144631 }
45154632
45164633 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
45174634 SDValue Op1, SDValue Op2,
45184635 SDValue Op3) {
4519 return getNode(~Opcode, dl, VT, Op1, Op2, Op3).getNode();
4636 SDVTList VTs = getVTList(VT);
4637 SDValue Ops[] = { Op1, Op2, Op3 };
4638 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45204639 }
45214640
45224641 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
45234642 const SDValue *Ops, unsigned NumOps) {
4524 return getNode(~Opcode, dl, VT, Ops, NumOps).getNode();
4643 SDVTList VTs = getVTList(VT);
4644 return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
45254645 }
45264646
45274647 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45284648 EVT VT1, EVT VT2) {
45294649 SDVTList VTs = getVTList(VT1, VT2);
4530 SDValue Op;
4531 return getNode(~Opcode, dl, VTs, &Op, 0).getNode();
4650 return getMachineNode(Opcode, dl, VTs, 0, 0);
45324651 }
45334652
45344653 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
45354654 EVT VT2, SDValue Op1) {
45364655 SDVTList VTs = getVTList(VT1, VT2);
4537 return getNode(~Opcode, dl, VTs, &Op1, 1).getNode();
4656 SDValue Ops[] = { Op1 };
4657 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45384658 }
45394659
45404660 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
45424662 SDValue Op2) {
45434663 SDVTList VTs = getVTList(VT1, VT2);
45444664 SDValue Ops[] = { Op1, Op2 };
4545 return getNode(~Opcode, dl, VTs, Ops, 2).getNode();
4665 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45464666 }
45474667
45484668 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
45504670 SDValue Op2, SDValue Op3) {
45514671 SDVTList VTs = getVTList(VT1, VT2);
45524672 SDValue Ops[] = { Op1, Op2, Op3 };
4553 return getNode(~Opcode, dl, VTs, Ops, 3).getNode();
4673 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45544674 }
45554675
45564676 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45574677 EVT VT1, EVT VT2,
45584678 const SDValue *Ops, unsigned NumOps) {
45594679 SDVTList VTs = getVTList(VT1, VT2);
4560 return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
4680 return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
45614681 }
45624682
45634683 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45654685 SDValue Op1, SDValue Op2) {
45664686 SDVTList VTs = getVTList(VT1, VT2, VT3);
45674687 SDValue Ops[] = { Op1, Op2 };
4568 return getNode(~Opcode, dl, VTs, Ops, 2).getNode();
4688 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45694689 }
45704690
45714691 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45744694 SDValue Op3) {
45754695 SDVTList VTs = getVTList(VT1, VT2, VT3);
45764696 SDValue Ops[] = { Op1, Op2, Op3 };
4577 return getNode(~Opcode, dl, VTs, Ops, 3).getNode();
4697 return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
45784698 }
45794699
45804700 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45814701 EVT VT1, EVT VT2, EVT VT3,
45824702 const SDValue *Ops, unsigned NumOps) {
45834703 SDVTList VTs = getVTList(VT1, VT2, VT3);
4584 return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
4704 return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
45854705 }
45864706
45874707 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
45884708 EVT VT2, EVT VT3, EVT VT4,
45894709 const SDValue *Ops, unsigned NumOps) {
45904710 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
4591 return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
4711 return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
45924712 }
45934713
45944714 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
45954715 const std::vector &ResultTys,
45964716 const SDValue *Ops, unsigned NumOps) {
4597 return getNode(~Opcode, dl, ResultTys, Ops, NumOps).getNode();
4717 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size());
4718 return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
4719 }
4720
4721 SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs,
4722 const SDValue *Ops, unsigned NumOps) {
4723 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Flag;
4724 MachineSDNode *N;
4725 void *IP;
4726
4727 if (DoCSE) {
4728 FoldingSetNodeID ID;
4729 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps);
4730 IP = 0;
4731 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
4732 return E;
4733 }
4734
4735 // Allocate a new MachineSDNode.
4736 N = NodeAllocator.Allocate();
4737 new (N) MachineSDNode(~Opcode, DL, VTs);
4738
4739 // Initialize the operands list.
4740 if (NumOps > array_lengthof(N->LocalOperands))
4741 // We're creating a final node that will live unmorphed for the
4742 // remainder of the current SelectionDAG iteration, so we can allocate
4743 // the operands directly out of a pool with no recycling metadata.
4744 N->InitOperands(OperandAllocator.Allocate(NumOps),
4745 Ops, NumOps);
4746 else
4747 N->InitOperands(N->LocalOperands, Ops, NumOps);
4748 N->OperandsNeedDelete = false;
4749
4750 if (DoCSE)
4751 CSEMap.InsertNode(N, IP);
4752
4753 AllNodes.push_back(N);
4754 #ifndef NDEBUG
4755 VerifyNode(N);
4756 #endif
4757 return N;
45984758 }
45994759
46004760 /// getTargetExtractSubreg - A convenience function for creating
49675127 }
49685128
49695129 MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt,
4970 const Value *srcValue, int SVO, unsigned alignment,
4971 bool vol)
4972 : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
4973 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
4974 assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
4975 assert(getOriginalAlignment() == alignment && "Alignment encoding error!");
4976 assert(isVolatile() == vol && "Volatile encoding error!");
5130 MachineMemOperand *mmo)
5131 : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) {
5132 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile());
5133 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
5134 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
49775135 }
49785136
49795137 MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs,
49805138 const SDValue *Ops, unsigned NumOps, EVT memvt,
4981 const Value *srcValue, int SVO, unsigned alignment,
4982 bool vol)
5139 MachineMemOperand *mmo)
49835140 : SDNode(Opc, dl, VTs, Ops, NumOps),
4984 MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
4985 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
4986 assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
4987 assert(getOriginalAlignment() == alignment && "Alignment encoding error!");
4988 assert(isVolatile() == vol && "Volatile encoding error!");
4989 }
4990
4991 /// getMemOperand - Return a MachineMemOperand object describing the memory
4992 /// reference performed by this memory reference.
4993 MachineMemOperand MemSDNode::getMemOperand() const {
4994 int Flags = 0;
4995 if (isa(this))
4996 Flags = MachineMemOperand::MOLoad;
4997 else if (isa(this))
4998 Flags = MachineMemOperand::MOStore;
4999 else if (isa(this)) {
5000 Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5001 }
5002 else {
5003 const MemIntrinsicSDNode* MemIntrinNode = dyn_cast(this);
5004 assert(MemIntrinNode && "Unknown MemSDNode opcode!");
5005 if (MemIntrinNode->readMem()) Flags |= MachineMemOperand::MOLoad;
5006 if (MemIntrinNode->writeMem()) Flags |= MachineMemOperand::MOStore;
5007 }
5008
5009 int Size = (getMemoryVT().getSizeInBits() + 7) >> 3;
5010 if (isVolatile()) Flags |= MachineMemOperand::MOVolatile;
5011
5012 // Check if the memory reference references a frame index
5013 const FrameIndexSDNode *FI =
5014 dyn_cast(getBasePtr().getNode());
5015 if (!getSrcValue() && FI)
5016 return MachineMemOperand(PseudoSourceValue::getFixedStack(FI->getIndex()),
5017 Flags, 0, Size, getOriginalAlignment());
5018 else
5019 return MachineMemOperand(getSrcValue(), Flags, getSrcValueOffset(),
5020 Size, getOriginalAlignment());
5141 MemoryVT(memvt), MMO(mmo) {
5142 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile());
5143 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
5144 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
50215145 }
50225146
50235147 /// Profile - Gather unique data for the node.
52205344 case ISD::PCMARKER: return "PCMarker";
52215345 case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
52225346 case ISD::SRCVALUE: return "SrcValue";
5223 case ISD::MEMOPERAND: return "MemOperand";
52245347 case ISD::EntryToken: return "EntryToken";
52255348 case ISD::TokenFactor: return "TokenFactor";
52265349 case ISD::AssertSext: return "AssertSext";
54995622 }
55005623
55015624 void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
5502 if (!isTargetOpcode() && getOpcode() == ISD::VECTOR_SHUFFLE) {
5503 const ShuffleVectorSDNode *SVN = cast(this);
5625 if (const MachineSDNode *MN = dyn_cast(this)) {
5626 if (!MN->memoperands_empty()) {
5627 OS << "<";
5628 OS << "Mem:";
5629 for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(),
5630 e = MN->memoperands_end(); i != e; ++i) {
5631 OS << **i;
5632 if (next(i) != e)
5633 OS << " ";
5634 }
5635 OS << ">";
5636 }
5637 } else if (const ShuffleVectorSDNode *SVN =
5638 dyn_cast(this)) {
55045639 OS << "<";
55055640 for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) {
55065641 int Idx = SVN->getMaskElt(i);
55115646 OS << Idx;
55125647 }
55135648 OS << ">";
5514 }
5515
5516 if (const ConstantSDNode *CSDN = dyn_cast(this)) {
5649 } else if (const ConstantSDNode *CSDN = dyn_cast(this)) {
55175650 OS << '<' << CSDN->getAPIntValue() << '>';
55185651 } else if (const ConstantFPSDNode *CSDN = dyn_cast(this)) {
55195652 if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle)
55785711 OS << "<" << M->getValue() << ">";
55795712 else
55805713 OS << "";
5581 } else if (const MemOperandSDNode *M = dyn_cast(this)) {
5582 OS << ": " << M->MO;
55835714 } else if (const VTSDNode *N = dyn_cast(this)) {
55845715 OS << ":" << N->getVT().getEVTString();
55855716 }
55865717 else if (const LoadSDNode *LD = dyn_cast(this)) {
5587 const Value *SrcValue = LD->getSrcValue();
5588 int SrcOffset = LD->getSrcValueOffset();
5589 OS << " <";
5590 if (SrcValue)
5591 OS << SrcValue;
5592 else
5593 OS << "null";
5594 OS << ":" << SrcOffset << ">";
5718 OS << " <" << *LD->getMemOperand();
55955719
55965720 bool doExt = true;
55975721 switch (LD->getExtensionType()) {
55985722 default: doExt = false; break;
5599 case ISD::EXTLOAD: OS << "
5600 case ISD::SEXTLOAD: OS << "
5601 case ISD::ZEXTLOAD: OS << " "; break;
5723 case ISD::EXTLOAD: OS << ", anyext"; break;
5724 case ISD::SEXTLOAD: OS << ", sext"; break;
5725 case ISD::ZEXTLOAD: OS << ", zext"; break;
56025726 }
56035727 if (doExt)
5604 OS << LD->getMemoryVT().getEVTString() << ">";
5728 OS << " from " << LD->getMemoryVT().getEVTString();
56055729
56065730 const char *AM = getIndexedModeName(LD->getAddressingMode());
56075731 if (*AM)
5608 OS << " " << AM;
5609 if (LD->isVolatile())
5610 OS << " ";
5611 OS << " alignment=" << LD->getAlignment();
5732 OS << ", " << AM;
5733
5734 OS << ">";
56125735 } else if (const StoreSDNode *ST = dyn_cast(this)) {
5613 const Value *SrcValue = ST->getSrcValue();
5614 int SrcOffset = ST->getSrcValueOffset();
5615 OS << " <";
5616 if (SrcValue)
5617 OS << SrcValue;
5618 else
5619 OS << "null";
5620 OS << ":" << SrcOffset << ">";
5736 OS << " <" << *ST->getMemOperand();
56215737
56225738 if (ST->isTruncatingStore())
5623 OS << " getMemoryVT().getEVTString() << ">";
5739 OS << ", trunc to " << ST->getMemoryVT().getEVTString();
56245740
56255741 const char *AM = getIndexedModeName(ST->getAddressingMode());
56265742 if (*AM)
5627 OS << " " << AM;
5628 if (ST->isVolatile())
5629 OS << " ";
5630 OS << " alignment=" << ST->getAlignment();
5631 } else if (const AtomicSDNode* AT = dyn_cast(this)) {
5632 const Value *SrcValue = AT->getSrcValue();
5633 int SrcOffset = AT->getSrcValueOffset();
5634 OS << " <";
5635 if (SrcValue)
5636 OS << SrcValue;
5637 else
5638 OS << "null";
5639 OS << ":" << SrcOffset << ">";
5640 if (AT->isVolatile())
5641 OS << " ";
5642 OS << " alignment=" << AT->getAlignment();
5743 OS << ", " << AM;
5744
5745 OS << ">";
5746 } else if (const MemSDNode* M = dyn_cast(this)) {
5747 OS << " <" << *M->getMemOperand() << ">";
56435748 }
56445749 }
56455750
1717 #include "llvm/CodeGen/LiveStackAnalysis.h"
1818 #include "llvm/CodeGen/MachineFrameInfo.h"
1919 #include "llvm/CodeGen/MachineLoopInfo.h"
20 #include "llvm/CodeGen/MachineMemOperand.h"
2021 #include "llvm/CodeGen/MachineRegisterInfo.h"
2122 #include "llvm/CodeGen/PseudoSourceValue.h"
2223 #include "llvm/Support/CommandLine.h"
450451 /// to old frame index with new one.
451452 void StackSlotColoring::RewriteInstruction(MachineInstr *MI, int OldFI,
452453 int NewFI, MachineFunction &MF) {
454 // Update the operands.
453455 for (unsigned i = 0, ee = MI->getNumOperands(); i != ee; ++i) {
454456 MachineOperand &MO = MI->getOperand(i);
455457 if (!MO.isFI())
460462 MO.setIndex(NewFI);
461463 }
462464
463 // Update the MachineMemOperand for the new memory location.
464 // FIXME: We need a better method of managing these too.
465 SmallVector MMOs(MI->memoperands_begin(),
466 MI->memoperands_end());
467 MI->clearMemOperands(MF);
465 // Update the memory references. This changes the MachineMemOperands
466 // directly. They may be in use by multiple instructions, however all
467 // instructions using OldFI are being rewritten to use NewFI.
468468 const Value *OldSV = PseudoSourceValue::getFixedStack(OldFI);
469 for (unsigned i = 0, ee = MMOs.size(); i != ee; ++i) {
470 if (MMOs[i].getValue() != OldSV)
471 MI->addMemOperand(MF, MMOs[i]);
472 else {
473 MachineMemOperand MMO(PseudoSourceValue::getFixedStack(NewFI),
474 MMOs[i].getFlags(), MMOs[i].getOffset(),
475 MMOs[i].getSize(), MMOs[i].getBaseAlignment());
476 MI->addMemOperand(MF, MMO);
477 }
478 }
469 const Value *NewSV = PseudoSourceValue::getFixedStack(NewFI);
470 for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
471 E = MI->memoperands_end(); I != E; ++I)
472 if ((*I)->getValue() == OldSV)
473 (*I)->setValue(NewSV);
479474 }
480475
481476 /// PropagateBackward - Traverse backward and look for the definition of
1616 #include "llvm/CodeGen/MachineFrameInfo.h"
1717 #include "llvm/CodeGen/MachineInstr.h"
1818 #include "llvm/CodeGen/MachineInstrBuilder.h"
19 #include "llvm/CodeGen/MachineMemOperand.h"
1920 #include "llvm/CodeGen/PseudoSourceValue.h"
2021 #include "llvm/Support/ErrorHandling.h"
2122 #include "llvm/Support/raw_ostream.h"
202203 "Folded a use to a non-load!");
203204 const MachineFrameInfo &MFI = *MF.getFrameInfo();
204205 assert(MFI.getObjectOffset(FrameIndex) != -1);
205 MachineMemOperand MMO(PseudoSourceValue::getFixedStack(FrameIndex),
206 Flags,
207 /*Offset=*/0,
208 MFI.getObjectSize(FrameIndex),
209 MFI.getObjectAlignment(FrameIndex));
206 MachineMemOperand *MMO =
207 MF.getMachineMemOperand(PseudoSourceValue::getFixedStack(FrameIndex),
208 Flags, /*Offset=*/0,
209 MFI.getObjectSize(FrameIndex),
210 MFI.getObjectAlignment(FrameIndex));
210211 NewMI->addMemOperand(MF, MMO);
211212
212213 return NewMI;
231232 if (!NewMI) return 0;
232233
233234 // Copy the memoperands from the load to the folded instruction.
234 for (std::list::iterator I = LoadMI->memoperands_begin(),
235 E = LoadMI->memoperands_end(); I != E; ++I)
236 NewMI->addMemOperand(MF, *I);
235 NewMI->setMemRefs(LoadMI->memoperands_begin(),
236 LoadMI->memoperands_end());
237237
238238 return NewMI;
239239 }
11731173
11741174 // Must sure the base address satisfies i64 ld / st alignment requirement.
11751175 if (!Op0->hasOneMemOperand() ||
1176 !Op0->memoperands_begin()->getValue() ||
1177 Op0->memoperands_begin()->isVolatile())
1176 !(*Op0->memoperands_begin())->getValue() ||
1177 (*Op0->memoperands_begin())->isVolatile())
11781178 return false;
11791179
1180 unsigned Align = Op0->memoperands_begin()->getAlignment();
1180 unsigned Align = (*Op0->memoperands_begin())->getAlignment();
11811181 unsigned ReqAlign = STI->hasV6Ops()
11821182 ? TD->getPrefTypeAlignment(
11831183 Type::getInt64Ty(Op0->getParent()->getParent()->getFunction()->getContext()))
33763376 // 64-bit registers. In particular, sign extend the input value into the
33773377 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
33783378 // then lfd it and fcfid it.
3379 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
3379 MachineFunction &MF = DAG.getMachineFunction();
3380 MachineFrameInfo *FrameInfo = MF.getFrameInfo();
33803381 int FrameIdx = FrameInfo->CreateStackObject(8, 8);
33813382 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
33823383 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
33853386 Op.getOperand(0));
33863387
33873388 // STD the extended value into the stack slot.
3388 MachineMemOperand MO(PseudoSourceValue::getFixedStack(FrameIdx),
3389 MachineMemOperand::MOStore, 0, 8, 8);
3390 SDValue Store = DAG.getNode(PPCISD::STD_32, dl, MVT::Other,
3391 DAG.getEntryNode(), Ext64, FIdx,
3392 DAG.getMemOperand(MO));
3389 MachineMemOperand *MMO =
3390 MF.getMachineMemOperand(PseudoSourceValue::getFixedStack(FrameIdx),
3391 MachineMemOperand::MOStore, 0, 8, 8);
3392 SDValue Ops[] = { DAG.getEntryNode(), Ext64, FIdx };
3393 SDValue Store =
3394 DAG.getMemIntrinsicNode(PPCISD::STD_32, dl, DAG.getVTList(MVT::Other),
3395 Ops, 4, MVT::i64, MMO);
33933396 // Load the value as a double.
33943397 SDValue Ld = DAG.getLoad(MVT::f64, dl, Store, FIdx, NULL, 0);
33953398
49304933 if (BSwapOp.getValueType() == MVT::i16)
49314934 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp);
49324935
4933 return DAG.getNode(PPCISD::STBRX, dl, MVT::Other, N->getOperand(0),
4934 BSwapOp, N->getOperand(2), N->getOperand(3),
4935 DAG.getValueType(N->getOperand(1).getValueType()));
4936 SDValue Ops[] = {
4937 N->getOperand(0), BSwapOp, N->getOperand(2),
4938 DAG.getValueType(N->getOperand(1).getValueType())
4939 };
4940 return
4941 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other),
4942 Ops, array_lengthof(Ops),
4943 cast(N)->getMemoryVT(),
4944 cast(N)->getMemOperand());
49364945 }
49374946 break;
49384947 case ISD::BSWAP:
49434952 SDValue Load = N->getOperand(0);
49444953 LoadSDNode *LD = cast(Load);
49454954 // Create the byte-swapping load.
4946 std::vector VTs;
4947 VTs.push_back(MVT::i32);
4948 VTs.push_back(MVT::Other);
4949 SDValue MO = DAG.getMemOperand(LD->getMemOperand());
49504955 SDValue Ops[] = {
49514956 LD->getChain(), // Chain
49524957 LD->getBasePtr(), // Ptr
4953 MO, // MemOperand
49544958 DAG.getValueType(N->getValueType(0)) // VT
49554959 };
4956 SDValue BSLoad = DAG.getNode(PPCISD::LBRX, dl, VTs, Ops, 4);
4960 SDValue BSLoad =
4961 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl,
4962 DAG.getVTList(MVT::i32, MVT::Other), Ops, 3,
4963 LD->getMemoryVT(), LD->getMemOperand());
49574964
49584965 // If this is an i16 load, insert the truncate.
49594966 SDValue ResVal = BSLoad;
4040 FCTIDZ, FCTIWZ,
4141
4242 /// STFIWX - The STFIWX instruction. The first operand is an input token
43 /// chain, then an f64 value to store, then an address to store it to,
44 /// then a SRCVALUE for the address.
43 /// chain, then an f64 value to store, then an address to store it to.
4544 STFIWX,
4645
4746 // VMADDFP, VNMSUBFP - The VMADDFP and VNMSUBFP instructions, taking
7978 /// registers.
8079 EXTSW_32,
8180
82 /// STD_32 - This is the STD instruction for use with "32-bit" registers.
83 STD_32,
84
8581 /// CALL - A direct function call.
8682 CALL_Darwin, CALL_SVR4,
8783
123119 /// an optional input flag argument.
124120 COND_BRANCH,
125121
126 /// CHAIN = STBRX CHAIN, GPRC, Ptr, SRCVALUE, Type - This is a
127 /// byte-swapping store instruction. It byte-swaps the low "Type" bits of
128 /// the GPRC input, then stores it through Ptr. Type can be either i16 or
129 /// i32.
130 STBRX,
131
132 /// GPRC, CHAIN = LBRX CHAIN, Ptr, SRCVALUE, Type - This is a
133 /// byte-swapping load instruction. It loads "Type" bits, byte swaps it,
134 /// then puts it in the bottom bits of the GPRC. TYPE can be either i16
135 /// or i32.
136 LBRX,
137
138122 // The following 5 instructions are used only as part of the
139123 // long double-to-int conversion sequence.
140124
169153 /// operand #1 callee (register or absolute)
170154 /// operand #2 stack adjustment
171155 /// operand #3 optional in flag
172 TC_RETURN
156 TC_RETURN,
157
158 /// STD_32 - This is the STD instruction for use with "32-bit" registers.
159 STD_32 = ISD::FIRST_TARGET_MEMORY_OPCODE,
160
161 /// CHAIN = STBRX CHAIN, GPRC, Ptr, Type - This is a
162 /// byte-swapping store instruction. It byte-swaps the low "Type" bits of
163 /// the GPRC input, then stores it through Ptr. Type can be either i16 or
164 /// i32.
165 STBRX,
166
167 /// GPRC, CHAIN = LBRX CHAIN, Ptr, Type - This is a
168 /// byte-swapping load instruction. It loads "Type" bits, byte swaps it,
169 /// then puts it in the bottom bits of the GPRC. TYPE can be either i16
170 /// or i32.
171 LBRX
173172 };
174173 }
175174
3434 SDTCisVT<0, i32>, SDTCisVT<2, OtherVT>
3535 ]>;
3636
37 def SDT_PPClbrx : SDTypeProfile<1, 3, [
38 SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>
37 def SDT_PPClbrx : SDTypeProfile<1, 2, [
38 SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>
3939 ]>;
40 def SDT_PPCstbrx : SDTypeProfile<0, 4, [
41 SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>
40 def SDT_PPCstbrx : SDTypeProfile<0, 3, [
41 SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>
4242 ]>;
4343
4444 def SDT_PPClarx : SDTypeProfile<1, 1, [
740740
741741 def LHBRX : XForm_1<31, 790, (outs GPRC:$rD), (ins memrr:$src),
742742 "lhbrx $rD, $src", LdStGeneral,
743 [(set GPRC:$rD, (PPClbrx xoaddr:$src, srcvalue:$sv, i16))]>;
743 [(set GPRC:$rD, (PPClbrx xoaddr:$src, i16))]>;
744744 def LWBRX : XForm_1<31, 534, (outs GPRC:$rD), (ins memrr:$src),
745745 "lwbrx $rD, $src", LdStGeneral,
746 [(set GPRC:$rD, (PPClbrx xoaddr:$src, srcvalue:$sv, i32))]>;
746 [(set GPRC:$rD, (PPClbrx xoaddr:$src, i32))]>;
747747
748748 def LFSX : XForm_25<31, 535, (outs F4RC:$frD), (ins memrr:$src),
749749 "lfsx $frD, $src", LdStLFDU,
836836 }
837837 def STHBRX: XForm_8<31, 918, (outs), (ins GPRC:$rS, memrr:$dst),
838838 "sthbrx $rS, $dst", LdStGeneral,
839 [(PPCstbrx GPRC:$rS, xoaddr:$dst, srcvalue:$dummy, i16)]>,
839 [(PPCstbrx GPRC:$rS, xoaddr:$dst, i16)]>,
840840 PPC970_DGroup_Cracked;
841841 def STWBRX: XForm_8<31, 662, (outs), (ins GPRC:$rS, memrr:$dst),
842842 "stwbrx $rS, $dst", LdStGeneral,
843 [(PPCstbrx GPRC:$rS, xoaddr:$dst, srcvalue:$dummy, i32)]>,
843 [(PPCstbrx GPRC:$rS, xoaddr:$dst, i32)]>,
844844 PPC970_DGroup_Cracked;
845845
846846 def STFIWX: XForm_28<31, 983, (outs), (ins F8RC:$frS, memrr:$dst),
2222
2323 #include "llvm/CodeGen/MachineFrameInfo.h"
2424 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineMemOperand.h"
2526 #include "llvm/CodeGen/PseudoSourceValue.h"
2627
2728 namespace llvm {
112113 Flags |= MachineMemOperand::MOLoad;
113114 if (TID.mayStore())
114115 Flags |= MachineMemOperand::MOStore;
115 MachineMemOperand MMO(PseudoSourceValue::getFixedStack(FI),
116 Flags,
117 Offset,
118 MFI.getObjectSize(FI),
119 MFI.getObjectAlignment(FI));
116 MachineMemOperand *MMO =
117 MF.getMachineMemOperand(PseudoSourceValue::getFixedStack(FI),
118 Flags, Offset,
119 MFI.getObjectSize(FI),
120 MFI.getObjectAlignment(FI));
120121 return addOffset(MIB.addFrameIndex(FI), Offset)
121122 .addMemOperand(MMO);
122123 }
14641464 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
14651465 if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4))
14661466 return NULL;
1467 SDValue LSI = Node->getOperand(4); // MemOperand
1468 const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, LSI, Chain};
1469 return CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
1470 MVT::i32, MVT::i32, MVT::Other, Ops,
1471 array_lengthof(Ops));
1467 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1468 MemOp[0] = cast(Node)->getMemOperand();
1469 const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, Chain};
1470 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
1471 MVT::i32, MVT::i32, MVT::Other, Ops,
1472 array_lengthof(Ops));
1473 cast(ResNode)->setMemRefs(MemOp, MemOp + 1);
1474 return ResNode;
14721475 }
14731476
14741477 SDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) {
16041607 DebugLoc dl = Node->getDebugLoc();
16051608 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF,
16061609 dl, NVT), 0);
1607 SDValue MemOp = CurDAG->getMemOperand(cast(Node)->getMemOperand());
1610 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1611 MemOp[0] = cast(Node)->getMemOperand();
16081612 if (isInc || isDec) {
1609 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, MemOp, Chain };
1610 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0);
1613 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain };
1614 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0);
1615 cast(Ret)->setMemRefs(MemOp, MemOp + 1);
16111616 SDValue RetVals[] = { Undef, Ret };
16121617 return CurDAG->getMergeValues(RetVals, 2, dl).getNode();
16131618 } else {
1614 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, MemOp, Chain };
1615 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8), 0);
1619 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain };
1620 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0);
1621 cast(Ret)->setMemRefs(MemOp, MemOp + 1);
16161622 SDValue RetVals[] = { Undef, Ret };
16171623 return CurDAG->getMergeValues(RetVals, 2, dl).getNode();
16181624 }
69826982 Node->getOperand(2), DAG.getIntPtrConstant(0));
69836983 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
69846984 Node->getOperand(2), DAG.getIntPtrConstant(1));
6985 // This is a generalized SDNode, not an AtomicSDNode, so it doesn't
6986 // have a MemOperand. Pass the info through as a normal operand.
6987 SDValue LSI = DAG.getMemOperand(cast(Node)->getMemOperand());
6988 SDValue Ops[] = { Chain, In1, In2L, In2H, LSI };
6985 SDValue Ops[] = { Chain, In1, In2L, In2H };
69896986 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6990 SDValue Result = DAG.getNode(NewOp, dl, Tys, Ops, 5);
6987 SDValue Result =
6988 DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64,
6989 cast(Node)->getMemOperand());
69916990 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
69926991 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
69936992 Results.push_back(Result.getValue(2));
73957394 (*MIB).addOperand(*argOpers[i]);
73967395 MIB.addReg(t2);
73977396 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7398 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin());
7397 (*MIB).setMemRefs(bInstr->memoperands_begin(),
7398 bInstr->memoperands_end());
73997399
74007400 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), destOper.getReg());
74017401 MIB.addReg(EAXreg);
75477547 (*MIB).addOperand(*argOpers[i]);
75487548
75497549 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7550 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin());
7550 (*MIB).setMemRefs(bInstr->memoperands_begin(),
7551 bInstr->memoperands_end());
75517552
75527553 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t3);
75537554 MIB.addReg(X86::EAX);
76517652 (*MIB).addOperand(*argOpers[i]);
76527653 MIB.addReg(t3);
76537654 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7654 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin());
7655 (*MIB).setMemRefs(mInstr->memoperands_begin(),
7656 mInstr->memoperands_end());
76557657
76567658 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), destOper.getReg());
76577659 MIB.addReg(X86::EAX);
77467748 // In the XMM save block, save all the XMM argument registers.
77477749 for (int i = 3, e = MI->getNumOperands(); i != e; ++i) {
77487750 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
7751 MachineMemOperand *MMO =
7752 F->getMachineMemOperand(
7753 PseudoSourceValue::getFixedStack(RegSaveFrameIndex),
7754 MachineMemOperand::MOStore, Offset,
7755 /*Size=*/16, /*Align=*/16);
77497756 BuildMI(XMMSaveMBB, DL, TII->get(X86::MOVAPSmr))
77507757 .addFrameIndex(RegSaveFrameIndex)
77517758 .addImm(/*Scale=*/1)
77537760 .addImm(/*Disp=*/Offset)
77547761 .addReg(/*Segment=*/0)
77557762 .addReg(MI->getOperand(i).getReg())
7756 .addMemOperand(MachineMemOperand(
7757 PseudoSourceValue::getFixedStack(RegSaveFrameIndex),
7758 MachineMemOperand::MOStore, Offset,
7759 /*Size=*/16, /*Align=*/16));
7763 .addMemOperand(MMO);
77607764 }
77617765
77627766 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now.
203203 LCMPXCHG_DAG,
204204 LCMPXCHG8_DAG,
205205
206 // ATOMADD64_DAG, ATOMSUB64_DAG, ATOMOR64_DAG, ATOMAND64_DAG,
207 // ATOMXOR64_DAG, ATOMNAND64_DAG, ATOMSWAP64_DAG -
208 // Atomic 64-bit binary operations.
209 ATOMADD64_DAG,
210 ATOMSUB64_DAG,
211 ATOMOR64_DAG,
212 ATOMXOR64_DAG,
213 ATOMAND64_DAG,
214 ATOMNAND64_DAG,
215 ATOMSWAP64_DAG,
216
217206 // FNSTCW16m - Store FP control world into i16 memory.
218207 FNSTCW16m,
219208
247236 // VASTART_SAVE_XMM_REGS - Save xmm argument registers to the stack,
248237 // according to %al. An operator is needed so that this can be expanded
249238 // with control flow.
250 VASTART_SAVE_XMM_REGS
239 VASTART_SAVE_XMM_REGS,
240
241 // ATOMADD64_DAG, ATOMSUB64_DAG, ATOMOR64_DAG, ATOMAND64_DAG,
242 // ATOMXOR64_DAG, ATOMNAND64_DAG, ATOMSWAP64_DAG -
243 // Atomic 64-bit binary operations.
244 ATOMADD64_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE,
245 ATOMSUB64_DAG,
246 ATOMOR64_DAG,
247 ATOMXOR64_DAG,
248 ATOMAND64_DAG,
249 ATOMNAND64_DAG,
250 ATOMSWAP64_DAG
251251 };
252252 }
253253
2525
2626 #include "llvm/CodeGen/MachineFrameInfo.h"
2727 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineMemOperand.h"
2829 #include "llvm/CodeGen/PseudoSourceValue.h"
2930
3031 namespace llvm {
141142 Flags |= MachineMemOperand::MOLoad;
142143 if (TID.mayStore())
143144 Flags |= MachineMemOperand::MOStore;
144 MachineMemOperand MMO(PseudoSourceValue::getFixedStack(FI),
145 Flags,
146 Offset,
147 MFI.getObjectSize(FI),
148 MFI.getObjectAlignment(FI));
145 MachineMemOperand *MMO =
146 MF.getMachineMemOperand(PseudoSourceValue::getFixedStack(FI),
147 Flags, Offset,
148 MFI.getObjectSize(FI),
149 MFI.getObjectAlignment(FI));
149150 return addOffset(MIB.addFrameIndex(FI), Offset)
150151 .addMemOperand(MMO);
151152 }
22952295 // Determine the alignment of the load.
22962296 unsigned Alignment = 0;
22972297 if (LoadMI->hasOneMemOperand())
2298 Alignment = LoadMI->memoperands_begin()->getAlignment();
2298 Alignment = (*LoadMI->memoperands_begin())->getAlignment();
22992299 else
23002300 switch (LoadMI->getOpcode()) {
23012301 case X86::V_SET0:
25662566 std::vector AfterOps;
25672567 DebugLoc dl = N->getDebugLoc();
25682568 unsigned NumOps = N->getNumOperands();
2569 for (unsigned i = 0; i != NumOps-2; ++i) {
2569 for (unsigned i = 0; i != NumOps-1; ++i) {
25702570 SDValue Op = N->getOperand(i);
25712571 if (i >= Index-NumDefs && i < Index-NumDefs + X86AddrNumOperands)
25722572 AddrOps.push_back(Op);
25752575 else if (i > Index-NumDefs)
25762576 AfterOps.push_back(Op);
25772577 }
2578 SDValue MemOp = N->getOperand(NumOps-2);
2579 AddrOps.push_back(MemOp);
25802578 SDValue Chain = N->getOperand(NumOps-1);
25812579 AddrOps.push_back(Chain);
25822580
26142612 // Emit the store instruction.
26152613 if (FoldedStore) {
26162614 AddrOps.pop_back();
2617 AddrOps.pop_back();
26182615 AddrOps.push_back(SDValue(NewNode, 0));
2619 AddrOps.push_back(MemOp);
26202616 AddrOps.push_back(Chain);
26212617 bool isAligned = (RI.getStackAlignment() >= 16) ||
26222618 RI.needsStackRealignment(MF);
None ; RUN: llc < %s -mtriple=arm-apple-darwin9 -stats |& grep asm-printer | grep 162
0 ; RUN: llc < %s -mtriple=arm-apple-darwin9 -stats |& grep asm-printer | grep 159
11
22 %"struct.Adv5::Ekin<3>" = type <{ i8 }>
33 %"struct.Adv5::X::Energyflux<3>" = type { double }
11341134 emitCode("}");
11351135 }
11361136
1137 // Generate MemOperandSDNodes nodes for each memory accesses covered by
1137 // Populate MemRefs with entries for each memory accesses covered by
11381138 // this pattern.
1139 if (II.mayLoad | II.mayStore) {
1140 std::vector::const_iterator mi, mie;
1141 for (mi = LSI.begin(), mie = LSI.end(); mi != mie; ++mi) {
1142 std::string LSIName = "LSI_" + *mi;
1143 emitCode("SDValue " + LSIName + " = "
1144 "CurDAG->getMemOperand(cast(" +
1145 *mi + ")->getMemOperand());");
1146 if (GenDebug) {
1147 emitCode("CurDAG->setSubgraphColor(" + LSIName +".getNode(), \"yellow\");");
1148 emitCode("CurDAG->setSubgraphColor(" + LSIName +".getNode(), \"black\");");
1149 }
1150 if (IsVariadic)
1151 emitCode("Ops" + utostr(OpsNo) + ".push_back(" + LSIName + ");");
1152 else
1153 AllOps.push_back(LSIName);
1154 }
1139 if (isRoot && !LSI.empty()) {
1140 std::string MemRefs = "MemRefs" + utostr(OpsNo);
1141 emitCode("MachineSDNode::mmo_iterator " + MemRefs + " = "
1142 "MF->allocateMemRefsArray(" + utostr(LSI.size()) + ");");
1143 for (unsigned i = 0, e = LSI.size(); i != e; ++i)
1144 emitCode(MemRefs + "[" + utostr(i) + "] = "
1145 "cast(" + LSI[i] + ")->getMemOperand();");
1146 After.push_back("cast(ResNode)->setMemRefs(" +
1147 MemRefs + ", " + MemRefs + " + " + utostr(LSI.size()) +
1148 ");");
11551149 }
11561150
11571151 if (NodeHasChain) {
19641958 << " assert(!N.isMachineOpcode() && \"Node already selected!\");\n"
19651959 << " break;\n"
19661960 << " case ISD::EntryToken: // These nodes remain the same.\n"
1967 << " case ISD::MEMOPERAND:\n"
19681961 << " case ISD::BasicBlock:\n"
19691962 << " case ISD::Register:\n"
19701963 << " case ISD::HANDLENODE:\n"