llvm.org GIT mirror llvm / e8be6c6
Add a new function, ReplaceAllUsesOfValuesWith, which handles bulk replacement of multiple values. This is slightly more efficient than doing multiple ReplaceAllUsesOfValueWith calls, and theoretically could be optimized even further. However, an important property of this new function is that it handles the case where the source value set and destination value set overlap. This makes it feasible for isel to use SelectNodeTo in many very common cases, which is advantageous because SelectNodeTo avoids a temporary node and it doesn't require CSEMap updates for users of values that don't change position. Revamp MorphNodeTo, which is what does all the work of SelectNodeTo, to handle operand lists more efficiently, and to correctly handle a number of corner cases to which its new wider use exposes it. This commit also includes a change to the encoding of post-isel opcodes in SDNodes; now instead of being sandwiched between the target-independent pre-isel opcodes and the target-dependent pre-isel opcodes, post-isel opcodes are now represented as negative values. This makes it possible to test if an opcode is pre-isel or post-isel without having to know the size of the current target's post-isel instruction set. These changes speed up llc overall by 3% and reduce memory usage by 10% on the InstructionCombining.cpp testcase with -fast and -regalloc=local. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@53728 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 11 years ago
21 changed file(s) with 548 addition(s) and 338 deletion(s). Raw diff Collapse all Expand all
120120 ISelQueueUpdater ISQU(ISelQueue);
121121 CurDAG->ReplaceAllUsesOfValueWith(F, T, &ISQU);
122122 setSelected(F.Val->getNodeId());
123 UpdateQueue(ISQU);
124 }
125
126 /// ReplaceUses - replace all uses of the old nodes F with the use
127 /// of the new nodes T.
128 void ReplaceUses(const SDOperand *F, const SDOperand *T,
129 unsigned Num) DISABLE_INLINE {
130 ISelQueueUpdater ISQU(ISelQueue);
131 CurDAG->ReplaceAllUsesOfValuesWith(F, T, Num, &ISQU);
132 for (unsigned i = 0; i != Num; ++i)
133 setSelected(F[i].Val->getNodeId());
123134 UpdateQueue(ISQU);
124135 }
125136
5858 /// CSEMap - This structure is used to memoize nodes, automatically performing
5959 /// CSE with existing nodes with a duplicate is requested.
6060 FoldingSet CSEMap;
61
62 /// Allocator - Pool allocation for misc. objects that are created once per
63 /// SelectionDAG.
64 BumpPtrAllocator Allocator;
6165
6266 public:
6367 SelectionDAG(TargetLowering &tli, MachineFunction &mf,
456460 /// SelectNodeTo - These are used for target selectors to *mutate* the
457461 /// specified node to have the specified return type, Target opcode, and
458462 /// operands. Note that target opcodes are stored as
459 /// ISD::BUILTIN_OP_END+TargetOpcode in the node opcode field. The 0th value
460 /// of the resultant node is returned.
463 /// ~TargetOpcode in the node opcode field. The resultant node is returned.
461464 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, MVT VT);
462465 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, MVT VT, SDOperand Op1);
463466 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, MVT VT,
480483 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, SDVTList VTs,
481484 const SDOperand *Ops, unsigned NumOps);
482485
486 /// MorphNodeTo - These *mutate* the specified node to have the specified
487 /// return type, opcode, and operands.
488 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT);
489 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT, SDOperand Op1);
490 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT,
491 SDOperand Op1, SDOperand Op2);
492 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT,
493 SDOperand Op1, SDOperand Op2, SDOperand Op3);
494 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT,
495 const SDOperand *Ops, unsigned NumOps);
496 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1, MVT VT2);
497 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1,
498 MVT VT2, const SDOperand *Ops, unsigned NumOps);
499 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1,
500 MVT VT2, MVT VT3, const SDOperand *Ops, unsigned NumOps);
501 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1,
502 MVT VT2, SDOperand Op1);
503 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1,
504 MVT VT2, SDOperand Op1, SDOperand Op2);
505 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, MVT VT1,
506 MVT VT2, SDOperand Op1, SDOperand Op2, SDOperand Op3);
507 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
508 const SDOperand *Ops, unsigned NumOps);
483509
484510 /// getTargetNode - These are used for target selectors to create a new node
485511 /// with specified return type(s), target opcode, and operands.
564590 void ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
565591 DAGUpdateListener *UpdateListener = 0);
566592
593 /// ReplaceAllUsesOfValuesWith - Like ReplaceAllUsesOfValueWith, but
594 /// for multiple values at once. This correctly handles the case where
595 /// there is an overlap between the From values and the To values.
596 void ReplaceAllUsesOfValuesWith(const SDOperand *From, const SDOperand *To,
597 unsigned Num,
598 DAGUpdateListener *UpdateListener = 0);
599
567600 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
568601 /// based on their topological order. It returns the maximum id and a vector
569602 /// of the SDNodes* in assigned order by reference.
653686 unsigned getMVTAlignment(MVT MemoryVT) const;
654687
655688 // List of non-single value types.
656 std::list > VTList;
689 std::vector> VTList;
657690
658691 // Maps to auto-CSE operations.
659692 std::vector CondCodeNodes;
845845 inline const SDOperand &getOperand(unsigned i) const;
846846 inline uint64_t getConstantOperandVal(unsigned i) const;
847847 inline bool isTargetOpcode() const;
848 inline unsigned getTargetOpcode() const;
848 inline bool isMachineOpcode() const;
849 inline unsigned getMachineOpcode() const;
849850
850851
851852 /// reachesChainWithoutSideEffects - Return true if this operand (which must
10271028 private:
10281029 /// NodeType - The operation that this node performs.
10291030 ///
1030 unsigned short NodeType;
1031 short NodeType;
10311032
10321033 /// OperandsNeedDelete - This is true if OperandList was new[]'d. If true,
10331034 /// then they will be delete[]'d when the node is destroyed.
10711072 //===--------------------------------------------------------------------===//
10721073 // Accessors
10731074 //
1074 unsigned getOpcode() const { return NodeType; }
1075
1076 /// getOpcode - Return the SelectionDAG opcode value for this node. For
1077 /// pre-isel nodes (those for which isMachineOpcode returns false), these
1078 /// are the opcode values in the ISD and ISD namespaces. For
1079 /// post-isel opcodes, see getMachineOpcode.
1080 unsigned getOpcode() const { return (unsigned short)NodeType; }
1081
1082 /// isTargetOpcode - Test if this node has a target-specific opcode (in the
1083 /// ISD namespace).
10751084 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
1076 unsigned getTargetOpcode() const {
1077 assert(isTargetOpcode() && "Not a target opcode!");
1078 return NodeType - ISD::BUILTIN_OP_END;
1085
1086 /// isMachineOpcode - Test if this node has a post-isel opcode, directly
1087 /// corresponding to a MachineInstr opcode.
1088 bool isMachineOpcode() const { return NodeType < 0; }
1089
1090 /// getMachineOpcode - This may only be called if isMachineOpcode returns
1091 /// true. It returns the MachineInstr opcode value that the node's opcode
1092 /// corresponds to.
1093 unsigned getMachineOpcode() const {
1094 assert(isMachineOpcode() && "Not a target opcode!");
1095 return ~NodeType;
10791096 }
10801097
10811098 size_t use_size() const { return std::distance(use_begin(), use_end()); }
13131330 }
13141331
13151332 /// DropOperands - Release the operands and set this node to have
1316 /// zero operands. This should only be used by HandleSDNode to clear
1317 /// its operand list.
1333 /// zero operands.
13181334 void DropOperands();
1319
1320 /// MorphNodeTo - This frees the operands of the current node, resets the
1321 /// opcode, types, and operands to the specified value. This should only be
1322 /// used by the SelectionDAG class.
1323 void MorphNodeTo(unsigned Opc, SDVTList L,
1324 const SDOperand *Ops, unsigned NumOps,
1325 SmallVectorImpl &DeadNodes);
13261335
13271336 void addUser(unsigned i, SDNode *User) {
13281337 assert(User->OperandList[i].getUser() && "Node without parent");
13571366 inline bool SDOperand::isTargetOpcode() const {
13581367 return Val->isTargetOpcode();
13591368 }
1360 inline unsigned SDOperand::getTargetOpcode() const {
1361 return Val->getTargetOpcode();
1369 inline bool SDOperand::isMachineOpcode() const {
1370 return Val->isMachineOpcode();
1371 }
1372 inline unsigned SDOperand::getMachineOpcode() const {
1373 return Val->getMachineOpcode();
13621374 }
13631375 inline bool SDOperand::hasOneUse() const {
13641376 return Val->hasNUsesOfValue(1, ResNo);
5757 return static_cast(Allocate(sizeof(T),AlignOf::Alignment));
5858 }
5959
60 template
61 T *Allocate(size_t Num) {
62 return static_cast(Allocate(Num * sizeof(T), AlignOf::Alignment));
63 }
64
6065 void Deallocate(void * /*Ptr*/) {}
6166
6267 void PrintStats() const;
6262 return;
6363
6464 unsigned ResNo = Use->getOperand(2).ResNo;
65 if (Def->isTargetOpcode()) {
66 const TargetInstrDesc &II = TII->get(Def->getTargetOpcode());
65 if (Def->isMachineOpcode()) {
66 const TargetInstrDesc &II = TII->get(Def->getMachineOpcode());
6767 if (ResNo >= II.getNumDefs() &&
6868 II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) {
6969 PhysReg = Reg;
166166 SUnit *SU = &SUnits[su];
167167 SDNode *MainNode = SU->Node;
168168
169 if (MainNode->isTargetOpcode()) {
170 unsigned Opc = MainNode->getTargetOpcode();
169 if (MainNode->isMachineOpcode()) {
170 unsigned Opc = MainNode->getMachineOpcode();
171171 const TargetInstrDesc &TID = TII->get(Opc);
172172 for (unsigned i = 0; i != TID.getNumOperands(); ++i) {
173173 if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) {
185185
186186 for (unsigned n = 0, e = SU->FlaggedNodes.size(); n != e; ++n) {
187187 SDNode *N = SU->FlaggedNodes[n];
188 if (N->isTargetOpcode() &&
189 TII->get(N->getTargetOpcode()).getImplicitDefs() &&
190 CountResults(N) > TII->get(N->getTargetOpcode()).getNumDefs())
188 if (N->isMachineOpcode() &&
189 TII->get(N->getMachineOpcode()).getImplicitDefs() &&
190 CountResults(N) > TII->get(N->getMachineOpcode()).getNumDefs())
191191 SU->hasPhysRegDefs = true;
192192
193193 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
226226 }
227227
228228 SU->Latency = 0;
229 if (SU->Node->isTargetOpcode()) {
230 unsigned SchedClass = TII->get(SU->Node->getTargetOpcode()).getSchedClass();
229 if (SU->Node->isMachineOpcode()) {
230 unsigned SchedClass = TII->get(SU->Node->getMachineOpcode()).getSchedClass();
231231 const InstrStage *S = InstrItins.begin(SchedClass);
232232 const InstrStage *E = InstrItins.end(SchedClass);
233233 for (; S != E; ++S)
235235 }
236236 for (unsigned i = 0, e = SU->FlaggedNodes.size(); i != e; ++i) {
237237 SDNode *FNode = SU->FlaggedNodes[i];
238 if (FNode->isTargetOpcode()) {
239 unsigned SchedClass = TII->get(FNode->getTargetOpcode()).getSchedClass();
238 if (FNode->isMachineOpcode()) {
239 unsigned SchedClass = TII->get(FNode->getMachineOpcode()).getSchedClass();
240240 const InstrStage *S = InstrItins.begin(SchedClass);
241241 const InstrStage *E = InstrItins.end(SchedClass);
242242 for (; S != E; ++S)
500500 void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI,
501501 const TargetInstrDesc &II,
502502 DenseMap &VRBaseMap) {
503 assert(Node->getTargetOpcode() != TargetInstrInfo::IMPLICIT_DEF &&
503 assert(Node->getMachineOpcode() != TargetInstrInfo::IMPLICIT_DEF &&
504504 "IMPLICIT_DEF should have been handled as a special case elsewhere!");
505505
506506 for (unsigned i = 0; i < II.getNumDefs(); ++i) {
543543 /// of the specified node.
544544 unsigned ScheduleDAG::getVR(SDOperand Op,
545545 DenseMap &VRBaseMap) {
546 if (Op.isTargetOpcode() &&
547 Op.getTargetOpcode() == TargetInstrInfo::IMPLICIT_DEF) {
546 if (Op.isMachineOpcode() &&
547 Op.getMachineOpcode() == TargetInstrInfo::IMPLICIT_DEF) {
548548 // Add an IMPLICIT_DEF instruction before every use.
549549 unsigned VReg = getDstOfOnlyCopyToRegUse(Op.Val, Op.ResNo);
550550 // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc
571571 unsigned IIOpNum,
572572 const TargetInstrDesc *II,
573573 DenseMap &VRBaseMap) {
574 if (Op.isTargetOpcode()) {
574 if (Op.isMachineOpcode()) {
575575 // Note that this case is redundant with the final else block, but we
576576 // include it because it is the most common and it makes the logic
577577 // simpler here.
703703 void ScheduleDAG::EmitSubregNode(SDNode *Node,
704704 DenseMap &VRBaseMap) {
705705 unsigned VRBase = 0;
706 unsigned Opc = Node->getTargetOpcode();
706 unsigned Opc = Node->getMachineOpcode();
707707
708708 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
709709 // the CopyToReg'd destination register instead of creating a new vreg.
798798 void ScheduleDAG::EmitNode(SDNode *Node, bool IsClone,
799799 DenseMap &VRBaseMap) {
800800 // If machine instruction
801 if (Node->isTargetOpcode()) {
802 unsigned Opc = Node->getTargetOpcode();
801 if (Node->isMachineOpcode()) {
802 unsigned Opc = Node->getMachineOpcode();
803803
804804 // Handle subreg insert/extract specially
805805 if (Opc == TargetInstrInfo::EXTRACT_SUBREG ||
205205 // If this is a pseudo op, like copyfromreg, look to see if there is a
206206 // real target node flagged to it. If so, use the target node.
207207 for (unsigned i = 0, e = CurSUnit->FlaggedNodes.size();
208 FoundNode->getOpcode() < ISD::BUILTIN_OP_END && i != e; ++i)
208 !FoundNode->isMachineOpcode() && i != e; ++i)
209209 FoundNode = CurSUnit->FlaggedNodes[i];
210210
211211 HazardRecognizer::HazardType HT = HazardRec->getHazardType(FoundNode);
215215 SUnit *SU = Sequence[i];
216216 if (!SU || !SU->Node) continue;
217217 if (SU->isCommutable) {
218 unsigned Opc = SU->Node->getTargetOpcode();
218 unsigned Opc = SU->Node->getMachineOpcode();
219219 const TargetInstrDesc &TID = TII->get(Opc);
220220 unsigned NumRes = TID.getNumDefs();
221221 unsigned NumOps = TID.getNumOperands() - NumRes;
677677 assert(N->getNodeId() == -1 && "Node already inserted!");
678678 N->setNodeId(NewSU->NodeNum);
679679
680 const TargetInstrDesc &TID = TII->get(N->getTargetOpcode());
680 const TargetInstrDesc &TID = TII->get(N->getMachineOpcode());
681681 for (unsigned i = 0; i != TID.getNumOperands(); ++i) {
682682 if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) {
683683 NewSU->isTwoAddress = true;
871871 /// FIXME: Move to SelectionDAG?
872872 static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
873873 const TargetInstrInfo *TII) {
874 const TargetInstrDesc &TID = TII->get(N->getTargetOpcode());
874 const TargetInstrDesc &TID = TII->get(N->getMachineOpcode());
875875 assert(TID.ImplicitDefs && "Physical reg def must be in implicit def list!");
876876 unsigned NumRes = TID.getNumDefs();
877877 for (const unsigned *ImpDef = TID.getImplicitDefs(); *ImpDef; ++ImpDef) {
912912
913913 for (unsigned i = 0, e = SU->FlaggedNodes.size()+1; i != e; ++i) {
914914 SDNode *Node = (i == 0) ? SU->Node : SU->FlaggedNodes[i-1];
915 if (!Node || !Node->isTargetOpcode())
915 if (!Node || !Node->isMachineOpcode())
916916 continue;
917 const TargetInstrDesc &TID = TII->get(Node->getTargetOpcode());
917 const TargetInstrDesc &TID = TII->get(Node->getMachineOpcode());
918918 if (!TID.ImplicitDefs)
919919 continue;
920920 for (const unsigned *Reg = TID.ImplicitDefs; *Reg; ++Reg) {
16721672 bool
16731673 BURegReductionPriorityQueue::canClobber(const SUnit *SU, const SUnit *Op) {
16741674 if (SU->isTwoAddress) {
1675 unsigned Opc = SU->Node->getTargetOpcode();
1675 unsigned Opc = SU->Node->getMachineOpcode();
16761676 const TargetInstrDesc &TID = TII->get(Opc);
16771677 unsigned NumRes = TID.getNumDefs();
16781678 unsigned NumOps = TID.getNumOperands() - NumRes;
17081708 const TargetInstrInfo *TII,
17091709 const TargetRegisterInfo *TRI) {
17101710 SDNode *N = SuccSU->Node;
1711 unsigned NumDefs = TII->get(N->getTargetOpcode()).getNumDefs();
1712 const unsigned *ImpDefs = TII->get(N->getTargetOpcode()).getImplicitDefs();
1711 unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs();
1712 const unsigned *ImpDefs = TII->get(N->getMachineOpcode()).getImplicitDefs();
17131713 assert(ImpDefs && "Caller should check hasPhysRegDefs");
17141714 const unsigned *SUImpDefs =
1715 TII->get(SU->Node->getTargetOpcode()).getImplicitDefs();
1715 TII->get(SU->Node->getMachineOpcode()).getImplicitDefs();
17161716 if (!SUImpDefs)
17171717 return false;
17181718 for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) {
17431743 continue;
17441744
17451745 SDNode *Node = SU->Node;
1746 if (!Node || !Node->isTargetOpcode() || SU->FlaggedNodes.size() > 0)
1746 if (!Node || !Node->isMachineOpcode() || SU->FlaggedNodes.size() > 0)
17471747 continue;
17481748
1749 unsigned Opc = Node->getTargetOpcode();
1749 unsigned Opc = Node->getMachineOpcode();
17501750 const TargetInstrDesc &TID = TII->get(Opc);
17511751 unsigned NumRes = TID.getNumDefs();
17521752 unsigned NumOps = TID.getNumOperands() - NumRes;
17671767 // depth and height.
17681768 if (SuccSU->Height < SU->Height && (SU->Height - SuccSU->Height) > 1)
17691769 continue;
1770 if (!SuccSU->Node || !SuccSU->Node->isTargetOpcode())
1770 if (!SuccSU->Node || !SuccSU->Node->isMachineOpcode())
17711771 continue;
17721772 // Don't constrain nodes with physical register defs if the
17731773 // predecessor can clobber them.
17771777 }
17781778 // Don't constraint extract_subreg / insert_subreg these may be
17791779 // coalesced away. We don't them close to their uses.
1780 unsigned SuccOpc = SuccSU->Node->getTargetOpcode();
1780 unsigned SuccOpc = SuccSU->Node->getMachineOpcode();
17811781 if (SuccOpc == TargetInstrInfo::EXTRACT_SUBREG ||
17821782 SuccOpc == TargetInstrInfo::INSERT_SUBREG)
17831783 continue;
18351835 bool td_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const {
18361836 unsigned LPriority = SPQ->getNodePriority(left);
18371837 unsigned RPriority = SPQ->getNodePriority(right);
1838 bool LIsTarget = left->Node && left->Node->isTargetOpcode();
1839 bool RIsTarget = right->Node && right->Node->isTargetOpcode();
1838 bool LIsTarget = left->Node && left->Node->isMachineOpcode();
1839 bool RIsTarget = right->Node && right->Node->isMachineOpcode();
18401840 bool LIsFloater = LIsTarget && left->NumPreds == 0;
18411841 bool RIsFloater = RIsTarget && right->NumPreds == 0;
18421842 unsigned LBonus = (LimitedSumOfUnscheduledPredsOfSuccs(left,1) == 1) ? 2 : 0;
196196 SDOperand Zero;
197197 if (N->getOpcode() == ISD::DBG_LABEL)
198198 return true;
199 if (N->isTargetOpcode() &&
200 N->getTargetOpcode() == TargetInstrInfo::DBG_LABEL)
199 if (N->isMachineOpcode() &&
200 N->getMachineOpcode() == TargetInstrInfo::DBG_LABEL)
201201 return true;
202202 return false;
203203 }
531531 }
532532
533533 void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){
534 SmallVector DeadNodes;
535 DeadNodes.push_back(N);
534 SmallVector DeadNodes(1, N);
536535 RemoveDeadNodes(DeadNodes, UpdateListener);
537536 }
538537
35223521 }
35233522
35243523 SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2) {
3525 for (std::list >::iterator I = VTList.begin(),
3526 E = VTList.end(); I != E; ++I) {
3527 if (I->size() == 2 && (*I)[0] == VT1 && (*I)[1] == VT2)
3528 return makeVTList(&(*I)[0], 2);
3529 }
3530 std::vector V;
3531 V.push_back(VT1);
3532 V.push_back(VT2);
3533 VTList.push_front(V);
3534 return makeVTList(&(*VTList.begin())[0], 2);
3535 }
3536 SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2,
3537 MVT VT3) {
3538 for (std::list >::iterator I = VTList.begin(),
3539 E = VTList.end(); I != E; ++I) {
3540 if (I->size() == 3 && (*I)[0] == VT1 && (*I)[1] == VT2 &&
3541 (*I)[2] == VT3)
3542 return makeVTList(&(*I)[0], 3);
3543 }
3544 std::vector V;
3545 V.push_back(VT1);
3546 V.push_back(VT2);
3547 V.push_back(VT3);
3548 VTList.push_front(V);
3549 return makeVTList(&(*VTList.begin())[0], 3);
3524 for (std::vector::reverse_iterator I = VTList.rbegin(),
3525 E = VTList.rend(); I != E; ++I)
3526 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2)
3527 return *I;
3528
3529 MVT *Array = Allocator.Allocate(2);
3530 Array[0] = VT1;
3531 Array[1] = VT2;
3532 SDVTList Result = makeVTList(Array, 2);
3533 VTList.push_back(Result);
3534 return Result;
3535 }
3536
3537 SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2, MVT VT3) {
3538 for (std::vector::reverse_iterator I = VTList.rbegin(),
3539 E = VTList.rend(); I != E; ++I)
3540 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 &&
3541 I->VTs[2] == VT3)
3542 return *I;
3543
3544 MVT *Array = Allocator.Allocate(3);
3545 Array[0] = VT1;
3546 Array[1] = VT2;
3547 Array[2] = VT3;
3548 SDVTList Result = makeVTList(Array, 3);
3549 VTList.push_back(Result);
3550 return Result;
35503551 }
35513552
35523553 SDVTList SelectionDAG::getVTList(const MVT *VTs, unsigned NumVTs) {
35583559 default: break;
35593560 }
35603561
3561 for (std::list >::iterator I = VTList.begin(),
3562 E = VTList.end(); I != E; ++I) {
3563 if (I->size() != NumVTs || VTs[0] != (*I)[0] || VTs[1] != (*I)[1]) continue;
3562 for (std::vector::reverse_iterator I = VTList.rbegin(),
3563 E = VTList.rend(); I != E; ++I) {
3564 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1])
3565 continue;
35643566
35653567 bool NoMatch = false;
35663568 for (unsigned i = 2; i != NumVTs; ++i)
3567 if (VTs[i] != (*I)[i]) {
3569 if (VTs[i] != I->VTs[i]) {
35683570 NoMatch = true;
35693571 break;
35703572 }
35713573 if (!NoMatch)
3572 return makeVTList(&*I->begin(), NumVTs);
3573 }
3574
3575 VTList.push_front(std::vector(VTs, VTs+NumVTs));
3576 return makeVTList(&*VTList.begin()->begin(), NumVTs);
3574 return *I;
3575 }
3576
3577 MVT *Array = Allocator.Allocate(NumVTs);
3578 std::copy(VTs, VTs+NumVTs, Array);
3579 SDVTList Result = makeVTList(Array, NumVTs);
3580 VTList.push_back(Result);
3581 return Result;
35773582 }
35783583
35793584
37103715 return InN;
37113716 }
37123717
3713 /// MorphNodeTo - This frees the operands of the current node, resets the
3714 /// opcode, types, and operands to the specified value. This should only be
3715 /// used by the SelectionDAG class.
3716 void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
3717 const SDOperand *Ops, unsigned NumOps,
3718 SmallVectorImpl &DeadNodes) {
3719 NodeType = Opc;
3720 ValueList = L.VTs;
3721 NumValues = L.NumVTs;
3722
3723 // Clear the operands list, updating used nodes to remove this from their
3724 // use list. Keep track of any operands that become dead as a result.
3725 SmallPtrSet DeadNodeSet;
3726 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) {
3727 SDNode *N = I->getVal();
3728 N->removeUser(std::distance(op_begin(), I), this);
3729 if (N->use_empty())
3730 DeadNodeSet.insert(N);
3731 }
3732
3733 // If NumOps is larger than the # of operands we currently have, reallocate
3734 // the operand list.
3735 if (NumOps > NumOperands) {
3736 if (OperandsNeedDelete) {
3737 delete [] OperandList;
3738 }
3739 OperandList = new SDUse[NumOps];
3740 OperandsNeedDelete = true;
3741 }
3742
3743 // Assign the new operands.
3744 NumOperands = NumOps;
3745
3746 for (unsigned i = 0, e = NumOps; i != e; ++i) {
3747 OperandList[i] = Ops[i];
3748 OperandList[i].setUser(this);
3749 SDNode *N = OperandList[i].getVal();
3750 N->addUser(i, this);
3751 DeadNodeSet.erase(N);
3752 }
3753
3754 // Clean up any nodes that are still dead after adding the uses for the
3755 // new operands.
3756 for (SmallPtrSet::iterator I = DeadNodeSet.begin(),
3757 E = DeadNodeSet.end(); I != E; ++I)
3758 DeadNodes.push_back(*I);
3759 }
3760
37613718 /// DropOperands - Release the operands and set this node to have
3762 /// zero operands. This should only be used by HandleSDNode to clear
3763 /// its operand list.
3719 /// zero operands.
37643720 void SDNode::DropOperands() {
3765 assert(NodeType == ISD::HANDLENODE &&
3766 "DropOperands is for HANDLENODE only!");
3767
37683721 // Unlike the code in MorphNodeTo that does this, we don't need to
37693722 // watch for dead nodes here.
37703723 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
37733726 NumOperands = 0;
37743727 }
37753728
3776 /// SelectNodeTo - These are used for target selectors to *mutate* the
3777 /// specified node to have the specified return type, Target opcode, and
3778 /// operands. Note that target opcodes are stored as
3779 /// ISD::BUILTIN_OP_END+TargetOpcode in the node opcode field.
3729 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
3730 /// machine opcode.
37803731 ///
3781 /// Note that SelectNodeTo returns the resultant node. If there is already a
3782 /// node of the specified opcode and operands, it returns that node instead of
3783 /// the current one.
3784 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3732 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
37853733 MVT VT) {
37863734 SDVTList VTs = getVTList(VT);
3787 return SelectNodeTo(N, TargetOpc, VTs, 0, 0);
3788 }
3789
3790 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3735 return SelectNodeTo(N, MachineOpc, VTs, 0, 0);
3736 }
3737
3738 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
37913739 MVT VT, SDOperand Op1) {
37923740 SDVTList VTs = getVTList(VT);
37933741 SDOperand Ops[] = { Op1 };
3794 return SelectNodeTo(N, TargetOpc, VTs, Ops, 1);
3795 }
3796
3797 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3742 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
3743 }
3744
3745 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
37983746 MVT VT, SDOperand Op1,
37993747 SDOperand Op2) {
38003748 SDVTList VTs = getVTList(VT);
38013749 SDOperand Ops[] = { Op1, Op2 };
3802 return SelectNodeTo(N, TargetOpc, VTs, Ops, 2);
3803 }
3804
3805 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3750 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
3751 }
3752
3753 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38063754 MVT VT, SDOperand Op1,
38073755 SDOperand Op2, SDOperand Op3) {
38083756 SDVTList VTs = getVTList(VT);
38093757 SDOperand Ops[] = { Op1, Op2, Op3 };
3810 return SelectNodeTo(N, TargetOpc, VTs, Ops, 3);
3811 }
3812
3813 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3758 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
3759 }
3760
3761 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38143762 MVT VT, const SDOperand *Ops,
38153763 unsigned NumOps) {
38163764 SDVTList VTs = getVTList(VT);
3817 return SelectNodeTo(N, TargetOpc, VTs, Ops, NumOps);
3818 }
3819
3820 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3765 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
3766 }
3767
3768 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38213769 MVT VT1, MVT VT2, const SDOperand *Ops,
38223770 unsigned NumOps) {
38233771 SDVTList VTs = getVTList(VT1, VT2);
3824 return SelectNodeTo(N, TargetOpc, VTs, Ops, NumOps);
3825 }
3826
3827 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3772 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
3773 }
3774
3775 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38283776 MVT VT1, MVT VT2) {
38293777 SDVTList VTs = getVTList(VT1, VT2);
3830 return SelectNodeTo(N, TargetOpc, VTs, (SDOperand *)0, 0);
3831 }
3832
3833 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3778 return SelectNodeTo(N, MachineOpc, VTs, (SDOperand *)0, 0);
3779 }
3780
3781 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38343782 MVT VT1, MVT VT2, MVT VT3,
38353783 const SDOperand *Ops, unsigned NumOps) {
38363784 SDVTList VTs = getVTList(VT1, VT2, VT3);
3837 return SelectNodeTo(N, TargetOpc, VTs, Ops, NumOps);
3838 }
3839
3840 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3785 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
3786 }
3787
3788 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38413789 MVT VT1, MVT VT2,
38423790 SDOperand Op1) {
38433791 SDVTList VTs = getVTList(VT1, VT2);
38443792 SDOperand Ops[] = { Op1 };
3845 return SelectNodeTo(N, TargetOpc, VTs, Ops, 1);
3846 }
3847
3848 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3793 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
3794 }
3795
3796 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38493797 MVT VT1, MVT VT2,
38503798 SDOperand Op1, SDOperand Op2) {
38513799 SDVTList VTs = getVTList(VT1, VT2);
38523800 SDOperand Ops[] = { Op1, Op2 };
3853 return SelectNodeTo(N, TargetOpc, VTs, Ops, 2);
3854 }
3855
3856 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3801 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
3802 }
3803
3804 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38573805 MVT VT1, MVT VT2,
38583806 SDOperand Op1, SDOperand Op2,
38593807 SDOperand Op3) {
38603808 SDVTList VTs = getVTList(VT1, VT2);
38613809 SDOperand Ops[] = { Op1, Op2, Op3 };
3862 return SelectNodeTo(N, TargetOpc, VTs, Ops, 3);
3863 }
3864
3865 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
3810 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
3811 }
3812
3813 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
38663814 SDVTList VTs, const SDOperand *Ops,
38673815 unsigned NumOps) {
3816 return MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps);
3817 }
3818
3819 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3820 MVT VT) {
3821 SDVTList VTs = getVTList(VT);
3822 return MorphNodeTo(N, Opc, VTs, 0, 0);
3823 }
3824
3825 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3826 MVT VT, SDOperand Op1) {
3827 SDVTList VTs = getVTList(VT);
3828 SDOperand Ops[] = { Op1 };
3829 return MorphNodeTo(N, Opc, VTs, Ops, 1);
3830 }
3831
3832 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3833 MVT VT, SDOperand Op1,
3834 SDOperand Op2) {
3835 SDVTList VTs = getVTList(VT);
3836 SDOperand Ops[] = { Op1, Op2 };
3837 return MorphNodeTo(N, Opc, VTs, Ops, 2);
3838 }
3839
3840 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3841 MVT VT, SDOperand Op1,
3842 SDOperand Op2, SDOperand Op3) {
3843 SDVTList VTs = getVTList(VT);
3844 SDOperand Ops[] = { Op1, Op2, Op3 };
3845 return MorphNodeTo(N, Opc, VTs, Ops, 3);
3846 }
3847
3848 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3849 MVT VT, const SDOperand *Ops,
3850 unsigned NumOps) {
3851 SDVTList VTs = getVTList(VT);
3852 return MorphNodeTo(N, Opc, VTs, Ops, NumOps);
3853 }
3854
3855 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3856 MVT VT1, MVT VT2, const SDOperand *Ops,
3857 unsigned NumOps) {
3858 SDVTList VTs = getVTList(VT1, VT2);
3859 return MorphNodeTo(N, Opc, VTs, Ops, NumOps);
3860 }
3861
3862 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3863 MVT VT1, MVT VT2) {
3864 SDVTList VTs = getVTList(VT1, VT2);
3865 return MorphNodeTo(N, Opc, VTs, (SDOperand *)0, 0);
3866 }
3867
3868 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3869 MVT VT1, MVT VT2, MVT VT3,
3870 const SDOperand *Ops, unsigned NumOps) {
3871 SDVTList VTs = getVTList(VT1, VT2, VT3);
3872 return MorphNodeTo(N, Opc, VTs, Ops, NumOps);
3873 }
3874
3875 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3876 MVT VT1, MVT VT2,
3877 SDOperand Op1) {
3878 SDVTList VTs = getVTList(VT1, VT2);
3879 SDOperand Ops[] = { Op1 };
3880 return MorphNodeTo(N, Opc, VTs, Ops, 1);
3881 }
3882
3883 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3884 MVT VT1, MVT VT2,
3885 SDOperand Op1, SDOperand Op2) {
3886 SDVTList VTs = getVTList(VT1, VT2);
3887 SDOperand Ops[] = { Op1, Op2 };
3888 return MorphNodeTo(N, Opc, VTs, Ops, 2);
3889 }
3890
3891 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3892 MVT VT1, MVT VT2,
3893 SDOperand Op1, SDOperand Op2,
3894 SDOperand Op3) {
3895 SDVTList VTs = getVTList(VT1, VT2);
3896 SDOperand Ops[] = { Op1, Op2, Op3 };
3897 return MorphNodeTo(N, Opc, VTs, Ops, 3);
3898 }
3899
3900 /// MorphNodeTo - These *mutate* the specified node to have the specified
3901 /// return type, opcode, and operands.
3902 ///
3903 /// Note that MorphNodeTo returns the resultant node. If there is already a
3904 /// node of the specified opcode and operands, it returns that node instead of
3905 /// the current one.
3906 ///
3907 /// Using MorphNodeTo is faster than creating a new node and swapping it in
3908 /// with ReplaceAllUsesWith both because it often avoids allocating a new
3909 /// node, and because it doesn't require CSE recalulation for any of
3910 /// the node's users.
3911 ///
3912 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
3913 SDVTList VTs, const SDOperand *Ops,
3914 unsigned NumOps) {
38683915 // If an identical node already exists, use it.
3869 FoldingSetNodeID ID;
3870 AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, NumOps);
38713916 void *IP = 0;
3872 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
3873 return ON;
3917 if (VTs.VTs[VTs.NumVTs-1] != MVT::Flag) {
3918 FoldingSetNodeID ID;
3919 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps);
3920 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
3921 return ON;
3922 }
38743923
38753924 RemoveNodeFromCSEMaps(N);
38763925
3926 // Start the morphing.
3927 N->NodeType = Opc;
3928 N->ValueList = VTs.VTs;
3929 N->NumValues = VTs.NumVTs;
3930
3931 // Clear the operands list, updating used nodes to remove this from their
3932 // use list. Keep track of any operands that become dead as a result.
3933 SmallPtrSet DeadNodeSet;
3934 for (SDNode::op_iterator B = N->op_begin(), I = B, E = N->op_end();
3935 I != E; ++I) {
3936 SDNode *Used = I->getVal();
3937 Used->removeUser(std::distance(B, I), N);
3938 if (Used->use_empty())
3939 DeadNodeSet.insert(Used);
3940 }
3941
3942 // If NumOps is larger than the # of operands we currently have, reallocate
3943 // the operand list.
3944 if (NumOps > N->NumOperands) {
3945 if (N->OperandsNeedDelete)
3946 delete[] N->OperandList;
3947 if (N->isMachineOpcode()) {
3948 // We're creating a final node that will live unmorphed for the
3949 // remainder of this SelectionDAG's duration, so we can allocate the
3950 // operands directly out of the pool with no recycling metadata.
3951 N->OperandList = Allocator.Allocate(NumOps);
3952 N->OperandsNeedDelete = false;
3953 } else {
3954 N->OperandList = new SDUse[NumOps];
3955 N->OperandsNeedDelete = true;
3956 }
3957 }
3958
3959 // Assign the new operands.
3960 N->NumOperands = NumOps;
3961 for (unsigned i = 0, e = NumOps; i != e; ++i) {
3962 N->OperandList[i] = Ops[i];
3963 N->OperandList[i].setUser(N);
3964 SDNode *ToUse = N->OperandList[i].getVal();
3965 ToUse->addUser(i, N);
3966 DeadNodeSet.erase(ToUse);
3967 }
3968
3969 // Delete any nodes that are still dead after adding the uses for the
3970 // new operands.
38773971 SmallVector DeadNodes;
3878 N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, NumOps, DeadNodes);
3972 for (SmallPtrSet::iterator I = DeadNodeSet.begin(),
3973 E = DeadNodeSet.end(); I != E; ++I)
3974 if ((*I)->use_empty())
3975 DeadNodes.push_back(*I);
38793976 RemoveDeadNodes(DeadNodes);
38803977
3881 CSEMap.InsertNode(N, IP); // Memoize the new node.
3978 if (IP)
3979 CSEMap.InsertNode(N, IP); // Memoize the new node.
38823980 return N;
38833981 }
38843982
38903988 /// node of the specified opcode and operands, it returns that node instead of
38913989 /// the current one.
38923990 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT) {
3893 return getNode(ISD::BUILTIN_OP_END+Opcode, VT).Val;
3991 return getNode(~Opcode, VT).Val;
38943992 }
38953993 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT, SDOperand Op1) {
3896 return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1).Val;
3994 return getNode(~Opcode, VT, Op1).Val;
38973995 }
38983996 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
38993997 SDOperand Op1, SDOperand Op2) {
3900 return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2).Val;
3998 return getNode(~Opcode, VT, Op1, Op2).Val;
39013999 }
39024000 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
39034001 SDOperand Op1, SDOperand Op2,
39044002 SDOperand Op3) {
3905 return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2, Op3).Val;
4003 return getNode(~Opcode, VT, Op1, Op2, Op3).Val;
39064004 }
39074005 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
39084006 const SDOperand *Ops, unsigned NumOps) {
3909 return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Ops, NumOps).Val;
4007 return getNode(~Opcode, VT, Ops, NumOps).Val;
39104008 }
39114009 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2) {
39124010 const MVT *VTs = getNodeValueTypes(VT1, VT2);
39134011 SDOperand Op;
3914 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, &Op, 0).Val;
4012 return getNode(~Opcode, VTs, 2, &Op, 0).Val;
39154013 }
39164014 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
39174015 MVT VT2, SDOperand Op1) {
39184016 const MVT *VTs = getNodeValueTypes(VT1, VT2);
3919 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, &Op1, 1).Val;
4017 return getNode(~Opcode, VTs, 2, &Op1, 1).Val;
39204018 }
39214019 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
39224020 MVT VT2, SDOperand Op1,
39234021 SDOperand Op2) {
39244022 const MVT *VTs = getNodeValueTypes(VT1, VT2);
39254023 SDOperand Ops[] = { Op1, Op2 };
3926 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 2).Val;
4024 return getNode(~Opcode, VTs, 2, Ops, 2).Val;
39274025 }
39284026 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
39294027 MVT VT2, SDOperand Op1,
39304028 SDOperand Op2, SDOperand Op3) {
39314029 const MVT *VTs = getNodeValueTypes(VT1, VT2);
39324030 SDOperand Ops[] = { Op1, Op2, Op3 };
3933 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 3).Val;
4031 return getNode(~Opcode, VTs, 2, Ops, 3).Val;
39344032 }
39354033 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2,
39364034 const SDOperand *Ops, unsigned NumOps) {
39374035 const MVT *VTs = getNodeValueTypes(VT1, VT2);
3938 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, NumOps).Val;
4036 return getNode(~Opcode, VTs, 2, Ops, NumOps).Val;
39394037 }
39404038 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
39414039 SDOperand Op1, SDOperand Op2) {
39424040 const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
39434041 SDOperand Ops[] = { Op1, Op2 };
3944 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 2).Val;
4042 return getNode(~Opcode, VTs, 3, Ops, 2).Val;
39454043 }
39464044 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
39474045 SDOperand Op1, SDOperand Op2,
39484046 SDOperand Op3) {
39494047 const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
39504048 SDOperand Ops[] = { Op1, Op2, Op3 };
3951 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 3).Val;
4049 return getNode(~Opcode, VTs, 3, Ops, 3).Val;
39524050 }
39534051 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
39544052 const SDOperand *Ops, unsigned NumOps) {
39554053 const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
3956 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, NumOps).Val;
4054 return getNode(~Opcode, VTs, 3, Ops, NumOps).Val;
39574055 }
39584056 SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
39594057 MVT VT2, MVT VT3, MVT VT4,
39644062 VTList.push_back(VT3);
39654063 VTList.push_back(VT4);
39664064 const MVT *VTs = getNodeValueTypes(VTList);
3967 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 4, Ops, NumOps).Val;
4065 return getNode(~Opcode, VTs, 4, Ops, NumOps).Val;
39684066 }
39694067 SDNode *SelectionDAG::getTargetNode(unsigned Opcode,
39704068 const std::vector &ResultTys,
39714069 const SDOperand *Ops, unsigned NumOps) {
39724070 const MVT *VTs = getNodeValueTypes(ResultTys);
3973 return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, ResultTys.size(),
4071 return getNode(~Opcode, VTs, ResultTys.size(),
39744072 Ops, NumOps).Val;
39754073 }
39764074
40424140 ///
40434141 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To,
40444142 DAGUpdateListener *UpdateListener) {
4045 assert(From != To && "Cannot replace uses of with self");
4046 assert(From->getNumValues() == To->getNumValues() &&
4143 assert(From->getVTList().VTs == To->getVTList().VTs &&
4144 From->getNumValues() == To->getNumValues() &&
40474145 "Cannot use this version of ReplaceAllUsesWith!");
4048 if (From->getNumValues() == 1) // If possible, use the faster version.
4049 return ReplaceAllUsesWith(SDOperand(From, 0), SDOperand(To, 0),
4050 UpdateListener);
4051
4146
4147 // Handle the trivial case.
4148 if (From == To)
4149 return;
4150
40524151 while (!From->use_empty()) {
40534152 SDNode::use_iterator UI = From->use_begin();
40544153 SDNode *U = UI->getUser();
41264225 }
41274226 }
41284227
4129 namespace {
4130 /// ChainedSetUpdaterListener - This class is a DAGUpdateListener that removes
4131 /// any deleted nodes from the set passed into its constructor and recursively
4132 /// notifies another update listener if specified.
4133 class ChainedSetUpdaterListener :
4134 public SelectionDAG::DAGUpdateListener {
4135 SmallSetVector &Set;
4136 SelectionDAG::DAGUpdateListener *Chain;
4137 public:
4138 ChainedSetUpdaterListener(SmallSetVector &set,
4139 SelectionDAG::DAGUpdateListener *chain)
4140 : Set(set), Chain(chain) {}
4141
4142 virtual void NodeDeleted(SDNode *N, SDNode *E) {
4143 Set.remove(N);
4144 if (Chain) Chain->NodeDeleted(N, E);
4145 }
4146 virtual void NodeUpdated(SDNode *N) {
4147 if (Chain) Chain->NodeUpdated(N);
4148 }
4149 };
4150 }
4151
41524228 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
41534229 /// uses of other values produced by From.Val alone. The Deleted vector is
41544230 /// handled the same way as for ReplaceAllUsesWith.
41554231 void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
41564232 DAGUpdateListener *UpdateListener){
4157 assert(From != To && "Cannot replace a value with itself");
4158
4233 // Handle the really simple, really trivial case efficiently.
4234 if (From == To) return;
4235
41594236 // Handle the simple, trivial, case efficiently.
41604237 if (From.Val->getNumValues() == 1) {
41614238 ReplaceAllUsesWith(From, To, UpdateListener);
41734250 Users.insert(User);
41744251 }
41754252
4176 // When one of the recursive merges deletes nodes from the graph, we need to
4177 // make sure that UpdateListener is notified *and* that the node is removed
4178 // from Users if present. CSUL does this.
4179 ChainedSetUpdaterListener CSUL(Users, UpdateListener);
4180
41814253 while (!Users.empty()) {
41824254 // We know that this user uses some value of From. If it is the right
41834255 // value, update it.
42164288
42174289 // If there was already an existing matching node, use ReplaceAllUsesWith
42184290 // to replace the dead one with the existing one. This can cause
4219 // recursive merging of other unrelated nodes down the line. The merging
4220 // can cause deletion of nodes that used the old value. To handle this, we
4221 // use CSUL to remove them from the Users set.
4222 ReplaceAllUsesWith(User, Existing, &CSUL);
4291 // recursive merging of other unrelated nodes down the line.
4292 ReplaceAllUsesWith(User, Existing, UpdateListener);
4293
4294 // User is now dead. Notify a listener if present.
4295 if (UpdateListener) UpdateListener->NodeDeleted(User, Existing);
4296 DeleteNodeNotInCSEMaps(User);
4297 }
4298 }
4299
4300 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
4301 /// uses of other values produced by From.Val alone. The same value may
4302 /// appear in both the From and To list. The Deleted vector is
4303 /// handled the same way as for ReplaceAllUsesWith.
4304 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDOperand *From,
4305 const SDOperand *To,
4306 unsigned Num,
4307 DAGUpdateListener *UpdateListener){
4308 // Handle the simple, trivial case efficiently.
4309 if (Num == 1)
4310 return ReplaceAllUsesOfValueWith(*From, *To, UpdateListener);
4311
4312 SmallVector, 16> Users;
4313 for (unsigned i = 0; i != Num; ++i)
4314 for (SDNode::use_iterator UI = From[i].Val->use_begin(),
4315 E = From[i].Val->use_end(); UI != E; ++UI)
4316 Users.push_back(std::make_pair(UI->getUser(), i));
4317
4318 while (!Users.empty()) {
4319 // We know that this user uses some value of From. If it is the right
4320 // value, update it.
4321 SDNode *User = Users.back().first;
4322 unsigned i = Users.back().second;
4323 Users.pop_back();
4324
4325 // Scan for an operand that matches From.
4326 SDNode::op_iterator Op = User->op_begin(), E = User->op_end();
4327 for (; Op != E; ++Op)
4328 if (*Op == From[i]) break;
4329
4330 // If there are no matches, the user must use some other result of From.
4331 if (Op == E) continue;
4332
4333 // Okay, we know this user needs to be updated. Remove its old self
4334 // from the CSE maps.
4335 RemoveNodeFromCSEMaps(User);
4336
4337 // Update all operands that match "From" in case there are multiple uses.
4338 for (; Op != E; ++Op) {
4339 if (*Op == From[i]) {
4340 From[i].Val->removeUser(Op-User->op_begin(), User);
4341 *Op = To[i];
4342 Op->setUser(User);
4343 To[i].Val->addUser(Op-User->op_begin(), User);
4344 }
4345 }
4346
4347 // Now that we have modified User, add it back to the CSE maps. If it
4348 // already exists there, recursively merge the results together.
4349 SDNode *Existing = AddNonLeafNodeToCSEMaps(User);
4350 if (!Existing) {
4351 if (UpdateListener) UpdateListener->NodeUpdated(User);
4352 continue; // Continue on to next user.
4353 }
4354
4355 // If there was already an existing matching node, use ReplaceAllUsesWith
4356 // to replace the dead one with the existing one. This can cause
4357 // recursive merging of other unrelated nodes down the line.
4358 ReplaceAllUsesWith(User, Existing, UpdateListener);
42234359
42244360 // User is now dead. Notify a listener if present.
42254361 if (UpdateListener) UpdateListener->NodeDeleted(User, Existing);
45064642 default:
45074643 if (getOpcode() < ISD::BUILTIN_OP_END)
45084644 return "<>";
4509 else {
4510 if (G) {
4645 if (isMachineOpcode()) {
4646 if (G)
45114647 if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo())
4512 if (getOpcode()-ISD::BUILTIN_OP_END < TII->getNumOpcodes())
4513 return TII->get(getOpcode()-ISD::BUILTIN_OP_END).getName();
4514
4515 TargetLowering &TLI = G->getTargetLoweringInfo();
4516 const char *Name =
4517 TLI.getTargetNodeName(getOpcode());
4518 if (Name) return Name;
4519 }
4520
4648 if (getMachineOpcode() < TII->getNumOpcodes())
4649 return TII->get(getMachineOpcode()).getName();
4650 return "<>";
4651 }
4652 if (G) {
4653 TargetLowering &TLI = G->getTargetLoweringInfo();
4654 const char *Name = TLI.getTargetNodeName(getOpcode());
4655 if (Name) return Name;
45214656 return "<>";
45224657 }
4658 return "<>";
45234659
4660 #ifndef NDEBUG
4661 case ISD::DELETED_NODE:
4662 return "<>";
4663 #endif
45244664 case ISD::PREFETCH: return "Prefetch";
45254665 case ISD::MEMBARRIER: return "MemBarrier";
45264666 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap";
529529
530530 SDNode *ARMDAGToDAGISel::Select(SDOperand Op) {
531531 SDNode *N = Op.Val;
532 unsigned Opcode = N->getOpcode();
533
534 if (Opcode >= ISD::BUILTIN_OP_END && Opcode < ARMISD::FIRST_NUMBER)
532
533 if (N->isMachineOpcode())
535534 return NULL; // Already selected.
536535
537536 switch (N->getOpcode()) {
243243 // target-specific node if it hasn't already been changed.
244244 SDNode *AlphaDAGToDAGISel::Select(SDOperand Op) {
245245 SDNode *N = Op.Val;
246 if (N->getOpcode() >= ISD::BUILTIN_OP_END &&
247 N->getOpcode() < AlphaISD::FIRST_NUMBER) {
246 if (N->isMachineOpcode()) {
248247 return NULL; // Already selected.
249248 }
250249
586586 MVT OpVT = Op.getValueType();
587587 SDOperand Ops[8];
588588
589 if (Opc >= ISD::BUILTIN_OP_END && Opc < SPUISD::FIRST_NUMBER) {
589 if (N->isMachineOpcode()) {
590590 return NULL; // Already selected.
591591 } else if (Opc == ISD::FrameIndex) {
592592 // Selects to (add $sp, FI * stackSlotSize)
304304 // target-specific node if it hasn't already been changed.
305305 SDNode *IA64DAGToDAGISel::Select(SDOperand Op) {
306306 SDNode *N = Op.Val;
307 if (N->getOpcode() >= ISD::BUILTIN_OP_END &&
308 N->getOpcode() < IA64ISD::FIRST_NUMBER)
307 if (N->isMachineOpcode())
309308 return NULL; // Already selected.
310309
311310 switch (N->getOpcode()) {
204204 #endif
205205
206206 // If we have a custom node, we already have selected!
207 if (Opcode >= ISD::BUILTIN_OP_END && Opcode < MipsISD::FIRST_NUMBER) {
207 if (Node->isMachineOpcode()) {
208208 #ifndef NDEBUG
209209 DOUT << std::string(Indent-2, ' ') << "== ";
210210 DEBUG(Node->dump(CurDAG));
238238 #endif
239239
240240 // If we have a custom node, we already have selected!
241 if (Opcode >= ISD::BUILTIN_OP_END && Opcode < PIC16ISD::FIRST_NUMBER) {
241 if (Node->isMachineOpcode()) {
242242 #ifndef NDEBUG
243243 DOUT << std::string(Indent-2, ' ') << "== ";
244244 DEBUG(Node->dump(CurDAG));
6363 bool &isFirst, bool &isSingle,
6464 bool &isCracked,
6565 bool &isLoad, bool &isStore) {
66 if (Opcode < ISD::BUILTIN_OP_END) {
66 if ((int)Opcode >= 0) {
6767 isFirst = isSingle = isCracked = isLoad = isStore = false;
6868 return PPCII::PPC970_Pseudo;
6969 }
70 Opcode -= ISD::BUILTIN_OP_END;
70 Opcode = ~Opcode;
7171
7272 const TargetInstrDesc &TID = TII.get(Opcode);
7373
124124 GetInstrType(Node->getOpcode(), isFirst, isSingle, isCracked,
125125 isLoad, isStore);
126126 if (InstrType == PPCII::PPC970_Pseudo) return NoHazard;
127 unsigned Opcode = Node->getOpcode()-ISD::BUILTIN_OP_END;
127 unsigned Opcode = Node->getMachineOpcode();
128128
129129 // We can only issue a PPC970_First/PPC970_Single instruction (such as
130130 // crand/mtspr/etc) if this is the first cycle of the dispatch group.
222222 GetInstrType(Node->getOpcode(), isFirst, isSingle, isCracked,
223223 isLoad, isStore);
224224 if (InstrType == PPCII::PPC970_Pseudo) return;
225 unsigned Opcode = Node->getOpcode()-ISD::BUILTIN_OP_END;
225 unsigned Opcode = Node->getMachineOpcode();
226226
227227 // Update structural hazard information.
228228 if (Opcode == PPC::MTCTR) HasCTRSet = true;
774774 // target-specific node if it hasn't already been changed.
775775 SDNode *PPCDAGToDAGISel::Select(SDOperand Op) {
776776 SDNode *N = Op.Val;
777 if (N->getOpcode() >= ISD::BUILTIN_OP_END &&
778 N->getOpcode() < PPCISD::FIRST_NUMBER)
777 if (N->isMachineOpcode())
779778 return NULL; // Already selected.
780779
781780 switch (N->getOpcode()) {
961960 AddToISelQueue(Offset);
962961 SDOperand Ops[] = { Offset, Base, Chain };
963962 // FIXME: PPC64
964 return CurDAG->getTargetNode(Opcode, MVT::i32, MVT::i32,
963 return CurDAG->getTargetNode(Opcode, LD->getValueType(0),
964 PPCLowering.getPointerTy(),
965965 MVT::Other, Ops, 3);
966966 } else {
967967 assert(0 && "R+R preindex loads not supported yet!");
136136
137137 SDNode *SparcDAGToDAGISel::Select(SDOperand Op) {
138138 SDNode *N = Op.Val;
139 if (N->getOpcode() >= ISD::BUILTIN_OP_END &&
140 N->getOpcode() < SPISD::FIRST_NUMBER)
139 if (N->isMachineOpcode())
141140 return NULL; // Already selected.
142141
143142 switch (N->getOpcode()) {
11581158 Indent += 2;
11591159 #endif
11601160
1161 if (Opcode >= ISD::BUILTIN_OP_END && Opcode < X86ISD::FIRST_NUMBER) {
1161 if (Node->isMachineOpcode()) {
11621162 #ifndef NDEBUG
11631163 DOUT << std::string(Indent-2, ' ') << "== ";
11641164 DEBUG(Node->dump(CurDAG));
22382238 bool
22392239 X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
22402240 SmallVectorImpl &NewNodes) const {
2241 if (!N->isTargetOpcode())
2241 if (!N->isMachineOpcode())
22422242 return false;
22432243
22442244 DenseMap >::iterator I =
2245 MemOp2RegOpTable.find((unsigned*)N->getTargetOpcode());
2245 MemOp2RegOpTable.find((unsigned*)N->getMachineOpcode());
22462246 if (I == MemOp2RegOpTable.end())
22472247 return false;
22482248 unsigned Opc = I->second.first;
0 ; RUN: llvm-as < %s | llc -march=ppc32 -mtriple=powerpc-apple-darwin | \
1 ; RUN: grep {stw r2, 32751}
1 ; RUN: grep {stw r3, 32751}
22 ; RUN: llvm-as < %s | llc -march=ppc64 -mtriple=powerpc-apple-darwin | \
3 ; RUN: grep {stw r2, 32751}
3 ; RUN: grep {stw r3, 32751}
44 ; RUN: llvm-as < %s | llc -march=ppc64 -mtriple=powerpc-apple-darwin | \
55 ; RUN: grep {std r2, 9024}
66
11561156 if (!isRoot)
11571157 Code += "), 0";
11581158
1159 bool NeedReplace = false;
1159 std::vector ReplaceFroms;
1160 std::vector ReplaceTos;
11601161 if (!isRoot) {
11611162 NodeOps.push_back("Tmp" + utostr(ResNo));
11621163 } else {
11751176
11761177 if (FoldedChains.size() > 0) {
11771178 std::string Code;
1178 for (unsigned j = 0, e = FoldedChains.size(); j < e; j++)
1179 After.push_back("ReplaceUses(SDOperand(" +
1180 FoldedChains[j].first + ".Val, " +
1181 utostr(FoldedChains[j].second) +
1182 "), SDOperand(ResNode, " +
1183 utostr(NumResults+NumDstRegs) + "));");
1184 NeedReplace = true;
1179 for (unsigned j = 0, e = FoldedChains.size(); j < e; j++) {
1180 ReplaceFroms.push_back("SDOperand(" +
1181 FoldedChains[j].first + ".Val, " +
1182 utostr(FoldedChains[j].second) +
1183 ")");
1184 ReplaceTos.push_back("SDOperand(ResNode, " +
1185 utostr(NumResults+NumDstRegs) + ")");
1186 }
11851187 }
11861188
11871189 if (NodeHasOutFlag) {
11881190 if (FoldedFlag.first != "") {
1189 After.push_back("ReplaceUses(SDOperand(" + FoldedFlag.first +
1190 ".Val, " +
1191 utostr(FoldedFlag.second) + "), InFlag);");
1191 ReplaceFroms.push_back("SDOperand(" + FoldedFlag.first + ".Val, " +
1192 utostr(FoldedFlag.second) + ")");
1193 ReplaceTos.push_back("InFlag");
11921194 } else {
11931195 assert(NodeHasProperty(Pattern, SDNPOutFlag, CGP));
1194 After.push_back("ReplaceUses(SDOperand(N.Val, " +
1195 utostr(NumPatResults + (unsigned)InputHasChain)
1196 +"), InFlag);");
1197 }
1198 NeedReplace = true;
1199 }
1200
1201 if (NeedReplace && InputHasChain) {
1202 After.push_back("ReplaceUses(SDOperand(N.Val, " +
1203 utostr(NumPatResults) + "), SDOperand(" + ChainName
1204 + ".Val, " + ChainName + ".ResNo" + "));");
1196 ReplaceFroms.push_back("SDOperand(N.Val, " +
1197 utostr(NumPatResults + (unsigned)InputHasChain)
1198 + ")");
1199 ReplaceTos.push_back("InFlag");
1200 }
1201 }
1202
1203 if (!ReplaceFroms.empty() && InputHasChain) {
1204 ReplaceFroms.push_back("SDOperand(N.Val, " +
1205 utostr(NumPatResults) + ")");
1206 ReplaceTos.push_back("SDOperand(" + ChainName + ".Val, " +
1207 ChainName + ".ResNo" + ")");
12051208 ChainAssignmentNeeded |= NodeHasChain;
12061209 }
12071210
12101213 ;
12111214 } else if (InputHasChain && !NodeHasChain) {
12121215 // One of the inner node produces a chain.
1213 if (NodeHasOutFlag)
1214 After.push_back("ReplaceUses(SDOperand(N.Val, " +
1215 utostr(NumPatResults+1) +
1216 "), SDOperand(ResNode, N.ResNo-1));");
1217 After.push_back("ReplaceUses(SDOperand(N.Val, " +
1218 utostr(NumPatResults) + "), " + ChainName + ");");
1219 NeedReplace = true;
1216 if (NodeHasOutFlag) {
1217 ReplaceFroms.push_back("SDOperand(N.Val, " +
1218 utostr(NumPatResults+1) +
1219 ")");
1220 ReplaceTos.push_back("SDOperand(ResNode, N.ResNo-1)");
1221 }
1222 ReplaceFroms.push_back("SDOperand(N.Val, " +
1223 utostr(NumPatResults) + ")");
1224 ReplaceTos.push_back(ChainName);
12201225 }
12211226 }
12221227
12331238 After.push_front(ChainAssign);
12341239 }
12351240
1236 // Use getTargetNode or SelectNodeTo? The safe choice is getTargetNode,
1237 // but SelectNodeTo can be faster.
1241 if (ReplaceFroms.size() == 1) {
1242 After.push_back("ReplaceUses(" + ReplaceFroms[0] + ", " +
1243 ReplaceTos[0] + ");");
1244 } else if (!ReplaceFroms.empty()) {
1245 After.push_back("const SDOperand Froms[] = {");
1246 for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i)
1247 After.push_back(" " + ReplaceFroms[i] + (i + 1 != e ? "," : ""));
1248 After.push_back("};");
1249 After.push_back("const SDOperand Tos[] = {");
1250 for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i)
1251 After.push_back(" " + ReplaceTos[i] + (i + 1 != e ? "," : ""));
1252 After.push_back("};");
1253 After.push_back("ReplaceUses(Froms, Tos, " +
1254 itostr(ReplaceFroms.size()) + ");");
1255 }
1256
1257 // We prefer to use SelectNodeTo since it avoids allocation when
1258 // possible and it avoids CSE map recalculation for the node's
1259 // users, however it's tricky to use in a non-root context.
12381260 //
1239 // SelectNodeTo is not safe in a non-root context, or if there is any
1240 // replacement of results needed.
1261 // We also don't use if the pattern replacement is being used to
1262 // jettison a chain result, since morphing the node in place
1263 // would leave users of the chain dangling.
12411264 //
1242 // SelectNodeTo is not profitable if it would require a dynamically
1243 // allocated operand list in a situation where getTargetNode would be
1244 // able to reuse a co-allocated operand list (as in a unary, binary or
1245 // ternary SDNode, for example).
1246 //
1247 if (!isRoot || NeedReplace ||
1248 (!IsVariadic && AllOps.size() < 4 &&
1249 Pattern->getNumChildren() + InputHasChain + NodeHasInFlag <
1250 AllOps.size())) {
1265 if (!isRoot || (InputHasChain && !NodeHasChain)) {
12511266 Code = "CurDAG->getTargetNode(" + Code;
12521267 } else {
12531268 Code = "CurDAG->SelectNodeTo(N.Val, " + Code;
19511966
19521967 OS << "// The main instruction selector code.\n"
19531968 << "SDNode *SelectCode(SDOperand N) {\n"
1954 << " if (N.getOpcode() >= ISD::BUILTIN_OP_END &&\n"
1955 << " N.getOpcode() < (ISD::BUILTIN_OP_END+" << InstNS
1956 << "INSTRUCTION_LIST_END)) {\n"
1969 << " if (N.isMachineOpcode()) {\n"
19571970 << " return NULL; // Already selected.\n"
19581971 << " }\n\n"
19591972 << " MVT::SimpleValueType NVT = N.Val->getValueType(0).getSimpleVT();\n"