llvm.org GIT mirror llvm / d63390c
Break PseudoSourceValue out of the Value hierarchy. It is now the root of its own tree containing FixedStackPseudoSourceValue (which you can use isa/dyn_cast on) and MipsCallEntry (which you can't). Anything that needs to use either a PseudoSourceValue* and Value* is strongly encouraged to use a MachinePointerInfo instead. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@206255 91177308-0d34-0410-b5e6-96231b3b80d8 Nick Lewycky 6 years ago
28 changed file(s) with 284 addition(s) and 342 deletion(s). Raw diff Collapse all Expand all
1515 #ifndef LLVM_CODEGEN_MACHINEMEMOPERAND_H
1616 #define LLVM_CODEGEN_MACHINEMEMOPERAND_H
1717
18 #include "llvm/ADT/PointerUnion.h"
19 #include "llvm/CodeGen/PseudoSourceValue.h"
20 #include "llvm/IR/Value.h" // PointerLikeTypeTraits
1821 #include "llvm/Support/DataTypes.h"
1922
2023 namespace llvm {
2124
22 class Value;
2325 class FoldingSetNodeID;
2426 class MDNode;
2527 class raw_ostream;
3234 /// V - This is the IR pointer value for the access, or it is null if unknown.
3335 /// If this is null, then the access is to a pointer in the default address
3436 /// space.
35 const Value *V;
37 PointerUnion V;
3638
3739 /// Offset - This is an offset from the base Value*.
3840 int64_t Offset;
4042 explicit MachinePointerInfo(const Value *v = nullptr, int64_t offset = 0)
4143 : V(v), Offset(offset) {}
4244
45 explicit MachinePointerInfo(const PseudoSourceValue *v,
46 int64_t offset = 0)
47 : V(v), Offset(offset) {}
48
4349 MachinePointerInfo getWithOffset(int64_t O) const {
44 if (V == nullptr) return MachinePointerInfo(nullptr, 0);
45 return MachinePointerInfo(V, Offset+O);
50 if (V.isNull()) return MachinePointerInfo();
51 if (V.is())
52 return MachinePointerInfo(V.get(), Offset+O);
53 return MachinePointerInfo(V.get(), Offset+O);
4654 }
4755
4856 /// getAddrSpace - Return the LLVM IR address space number that this pointer
120128 /// other PseudoSourceValue member functions which return objects which stand
121129 /// for frame/stack pointer relative references and other special references
122130 /// which are not representable in the high-level IR.
123 const Value *getValue() const { return PtrInfo.V; }
131 const Value *getValue() const { return PtrInfo.V.dyn_cast(); }
132
133 const PseudoSourceValue *getPseudoValue() const {
134 return PtrInfo.V.dyn_cast();
135 }
136
137 const void *getOpaqueValue() const { return PtrInfo.V.getOpaqueValue(); }
124138
125139 /// getFlags - Return the raw flags of the source value, \see MemOperandFlags.
126140 unsigned int getFlags() const { return Flags & ((1 << MOMaxBits) - 1); }
176190 /// should only be used when an object is being relocated and all references
177191 /// to it are being updated.
178192 void setValue(const Value *NewSV) { PtrInfo.V = NewSV; }
193 void setValue(const PseudoSourceValue *NewSV) { PtrInfo.V = NewSV; }
179194 void setOffset(int64_t NewOffset) { PtrInfo.Offset = NewOffset; }
180195
181196 /// Profile - Gather unique data for the object.
1717
1818 namespace llvm {
1919 class MachineFrameInfo;
20 class MachineMemOperand;
2021 class raw_ostream;
2122
2223 /// PseudoSourceValue - Special value supplied for machine level alias
2324 /// analysis. It indicates that a memory access references the functions
2425 /// stack frame (e.g., a spill slot), below the stack frame (e.g., argument
2526 /// space), or constant pool.
26 class PseudoSourceValue : public Value {
27 class PseudoSourceValue {
2728 private:
29 friend raw_ostream &llvm::operator<<(raw_ostream &OS,
30 const MachineMemOperand &MMO);
31
2832 /// printCustom - Implement printing for PseudoSourceValue. This is called
2933 /// from Value::print or Value's operator<<.
3034 ///
31 void printCustom(raw_ostream &O) const override;
35 virtual void printCustom(raw_ostream &O) const;
3236
3337 public:
34 explicit PseudoSourceValue(enum ValueTy Subclass = PseudoSourceValueVal);
38 /// isFixed - Whether this is a FixedStackPseudoSourceValue.
39 bool isFixed;
40
41 explicit PseudoSourceValue(bool isFixed = false);
42
43 virtual ~PseudoSourceValue();
3544
3645 /// isConstant - Test whether the memory pointed to by this
3746 /// PseudoSourceValue has a constant value.
4554 /// mayAlias - Return true if the memory pointed to by this
4655 /// PseudoSourceValue can ever alias an LLVM IR Value.
4756 virtual bool mayAlias(const MachineFrameInfo *) const;
48
49 /// classof - Methods for support type inquiry through isa, cast, and
50 /// dyn_cast:
51 ///
52 static inline bool classof(const Value *V) {
53 return V->getValueID() == PseudoSourceValueVal ||
54 V->getValueID() == FixedStackPseudoSourceValueVal;
55 }
5657
5758 /// A pseudo source value referencing a fixed stack frame entry,
5859 /// e.g., a spill slot.
8384 const int FI;
8485 public:
8586 explicit FixedStackPseudoSourceValue(int fi) :
86 PseudoSourceValue(FixedStackPseudoSourceValueVal), FI(fi) {}
87 PseudoSourceValue(true), FI(fi) {}
8788
8889 /// classof - Methods for support type inquiry through isa, cast, and
8990 /// dyn_cast:
9091 ///
91 static inline bool classof(const Value *V) {
92 return V->getValueID() == FixedStackPseudoSourceValueVal;
92 static inline bool classof(const PseudoSourceValue *V) {
93 return V->isFixed == true;
9394 }
9495
9596 bool isConstant(const MachineFrameInfo *MFI) const override;
704704 /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
705705 /// and chain and takes 2 operands.
706706 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
707 SDValue Ptr, SDValue Val, const Value* PtrVal,
707 SDValue Ptr, SDValue Val, const Value *PtrVal,
708708 unsigned Alignment, AtomicOrdering Ordering,
709709 SynchronizationScope SynchScope);
710710 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
714714
715715 /// getAtomic - Gets a node for an atomic op, produces result and chain and
716716 /// takes 1 operand.
717 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT,
718 SDValue Chain, SDValue Ptr, const Value* PtrVal,
719 unsigned Alignment,
720 AtomicOrdering Ordering,
721 SynchronizationScope SynchScope);
722717 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT,
723718 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
724719 AtomicOrdering Ordering,
11661161 void ExtractVectorElements(SDValue Op, SmallVectorImpl &Args,
11671162 unsigned Start = 0, unsigned Count = 0);
11681163
1164 unsigned getEVTAlignment(EVT MemoryVT) const;
1165
11691166 private:
11701167 bool RemoveNodeFromCSEMaps(SDNode *N);
11711168 void AddModifiedNodeToCSEMaps(SDNode *N);
11791176 void DeleteNodeNotInCSEMaps(SDNode *N);
11801177 void DeallocateNode(SDNode *N);
11811178
1182 unsigned getEVTAlignment(EVT MemoryVT) const;
1183
11841179 void allnodes_clear();
11851180
11861181 /// VTList - List of non-single value types.
10311031 return SynchronizationScope((SubclassData >> 12) & 1);
10321032 }
10331033
1034 /// Returns the SrcValue and offset that describes the location of the access
1035 const Value *getSrcValue() const { return MMO->getValue(); }
1034 // Returns the offset from the location of the access.
10361035 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
10371036
10381037 /// Returns the TBAAInfo that describes the dereference.
327327 MDNodeVal, // This is an instance of MDNode
328328 MDStringVal, // This is an instance of MDString
329329 InlineAsmVal, // This is an instance of InlineAsm
330 PseudoSourceValueVal, // This is an instance of PseudoSourceValue
331 FixedStackPseudoSourceValueVal, // This is an instance of
332 // FixedStackPseudoSourceValue
333330 InstructionVal, // This is an instance of Instruction
334331 // Enum values starting at InstructionVal are used for Instructions;
335332 // don't add new values here!
235235 MachineMemOperand *
236236 MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
237237 int64_t Offset, uint64_t Size) {
238 if (MMO->getValue())
239 return new (Allocator)
240 MachineMemOperand(MachinePointerInfo(MMO->getValue(),
241 MMO->getOffset()+Offset),
242 MMO->getFlags(), Size,
243 MMO->getBaseAlignment(), nullptr);
238244 return new (Allocator)
239 MachineMemOperand(MachinePointerInfo(MMO->getValue(),
245 MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(),
240246 MMO->getOffset()+Offset),
241247 MMO->getFlags(), Size,
242248 MMO->getBaseAlignment(), nullptr);
398398 /// getAddrSpace - Return the LLVM IR address space number that this pointer
399399 /// points into.
400400 unsigned MachinePointerInfo::getAddrSpace() const {
401 if (!V) return 0;
402 return cast(V->getType())->getAddressSpace();
401 if (V.isNull() || V.is()) return 0;
402 return cast(V.get()->getType())->getAddressSpace();
403403 }
404404
405405 /// getConstantPool - Return a MachinePointerInfo record that refers to the
433433 : PtrInfo(ptrinfo), Size(s),
434434 Flags((f & ((1 << MOMaxBits) - 1)) | ((Log2_32(a) + 1) << MOMaxBits)),
435435 TBAAInfo(TBAAInfo), Ranges(Ranges) {
436 assert((!PtrInfo.V || isa(PtrInfo.V->getType())) &&
436 assert((PtrInfo.V.isNull() || PtrInfo.V.is() ||
437 isa(PtrInfo.V.get()->getType())) &&
437438 "invalid pointer value");
438439 assert(getBaseAlignment() == a && "Alignment is not a power of 2!");
439440 assert((isLoad() || isStore()) && "Not a load/store!");
444445 void MachineMemOperand::Profile(FoldingSetNodeID &ID) const {
445446 ID.AddInteger(getOffset());
446447 ID.AddInteger(Size);
447 ID.AddPointer(getValue());
448 ID.AddPointer(getOpaqueValue());
448449 ID.AddInteger(Flags);
449450 }
450451
485486
486487 // Print the address information.
487488 OS << "[";
488 if (!MMO.getValue())
489 if (const Value *V = MMO.getValue())
490 V->printAsOperand(OS, /*PrintType=*/false);
491 else if (const PseudoSourceValue *PSV = MMO.getPseudoValue())
492 PSV->printCustom(OS);
493 else
489494 OS << "";
490 else
491 MMO.getValue()->printAsOperand(OS, /*PrintType=*/false);
492495
493496 unsigned AS = MMO.getAddrSpace();
494497 if (AS != 0)
13651368 if ((*I)->isStore()) return false;
13661369 if ((*I)->isInvariant()) return true;
13671370
1371
1372 // A load from a constant PseudoSourceValue is invariant.
1373 if (const PseudoSourceValue *PSV = (*I)->getPseudoValue())
1374 if (PSV->isConstant(MFI))
1375 continue;
1376
13681377 if (const Value *V = (*I)->getValue()) {
1369 // A load from a constant PseudoSourceValue is invariant.
1370 if (const PseudoSourceValue *PSV = dyn_cast(V))
1371 if (PSV->isConstant(MFI))
1372 continue;
13731378 // If we have an AliasAnalysis, ask it whether the memory is constant.
13741379 if (AA && AA->pointsToConstantMemory(
13751380 AliasAnalysis::Location(V, (*I)->getSize(),
389389 static bool InstructionStoresToFI(const MachineInstr *MI, int FI) {
390390 for (MachineInstr::mmo_iterator o = MI->memoperands_begin(),
391391 oe = MI->memoperands_end(); o != oe; ++o) {
392 if (!(*o)->isStore() || !(*o)->getValue())
392 if (!(*o)->isStore() || !(*o)->getPseudoValue())
393393 continue;
394394 if (const FixedStackPseudoSourceValue *Value =
395 dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) {
395 dyn_cast<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) {
396396 if (Value->getFrameIndex() == FI)
397397 return true;
398398 }
881881 assert (MI.mayLoad() && "Expected MI that loads!");
882882 for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
883883 E = MI.memoperands_end(); I != E; ++I) {
884 if (const Value *V = (*I)->getValue()) {
885 if (const PseudoSourceValue *PSV = dyn_cast(V))
886 if (PSV == PSV->getGOT() || PSV == PSV->getConstantPool())
887 return true;
884 if (const PseudoSourceValue *PSV = (*I)->getPseudoValue()) {
885 if (PSV == PSV->getGOT() || PSV == PSV->getConstantPool())
886 return true;
888887 }
889888 }
890889 return false;
5757 "ConstantPool"
5858 };
5959
60 // FIXME: THIS IS A HACK!!!!
61 // Eventually these should be uniqued on LLVMContext rather than in a managed
62 // static. For now, we can safely use the global context for the time being to
63 // squeak by.
64 PseudoSourceValue::PseudoSourceValue(enum ValueTy Subclass) :
65 Value(Type::getInt8PtrTy(getGlobalContext()),
66 Subclass) {}
60 PseudoSourceValue::PseudoSourceValue(bool isFixed) : isFixed(isFixed) {}
61
62 PseudoSourceValue::~PseudoSourceValue() {}
6763
6864 void PseudoSourceValue::printCustom(raw_ostream &O) const {
6965 O << PSVNames[this - PSVGlobals->PSVs];
9797 /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences.
9898 static void getUnderlyingObjects(const Value *V,
9999 SmallVectorImpl &Objects) {
100 SmallPtrSet*, 16> Visited;
100 SmallPtrSet *, 16> Visited;
101101 SmallVector Working(1, V);
102102 do {
103103 V = Working.pop_back_val();
123123 } while (!Working.empty());
124124 }
125125
126 typedef SmallVector, 4>
126 typedef PointerUnion ValueType;
127 typedef SmallVector, 4>
127128 UnderlyingObjectsVector;
128129
129130 /// getUnderlyingObjectsForInstr - If this machine instr has memory reference
133134 const MachineFrameInfo *MFI,
134135 UnderlyingObjectsVector &Objects) {
135136 if (!MI->hasOneMemOperand() ||
136 !(*MI->memoperands_begin())->getValue() ||
137 (!(*MI->memoperands_begin())->getValue() &&
138 !(*MI->memoperands_begin())->getPseudoValue()) ||
137139 (*MI->memoperands_begin())->isVolatile())
138140 return;
139141
140 const Value *V = (*MI->memoperands_begin())->getValue();
141 if (!V)
142 return;
143
144 if (const PseudoSourceValue *PSV = dyn_cast(V)) {
142 if (const PseudoSourceValue *PSV =
143 (*MI->memoperands_begin())->getPseudoValue()) {
145144 // For now, ignore PseudoSourceValues which may alias LLVM IR values
146145 // because the code that uses this function has no way to cope with
147146 // such aliases.
148147 if (!PSV->isAliased(MFI)) {
149148 bool MayAlias = PSV->mayAlias(MFI);
150 Objects.push_back(UnderlyingObjectsVector::value_type(V, MayAlias));
149 Objects.push_back(UnderlyingObjectsVector::value_type(PSV, MayAlias));
151150 }
152151 return;
153152 }
153
154 const Value *V = (*MI->memoperands_begin())->getValue();
155 if (!V)
156 return;
154157
155158 SmallVector Objs;
156159 getUnderlyingObjects(V, Objs);
158161 for (SmallVectorImpl::iterator I = Objs.begin(), IE = Objs.end();
159162 I != IE; ++I) {
160163 V = *I;
161
162 assert(!isa(V) && "Underlying value is a stack slot!");
163164
164165 if (!isIdentifiedObject(V)) {
165166 Objects.clear();
476477 if ((*MI->memoperands_begin())->isVolatile() ||
477478 MI->hasUnmodeledSideEffects())
478479 return true;
480
481 if ((*MI->memoperands_begin())->getPseudoValue()) {
482 // Similarly to getUnderlyingObjectForInstr:
483 // For now, ignore PseudoSourceValues which may alias LLVM IR values
484 // because the code that uses this function has no way to cope with
485 // such aliases.
486 return true;
487 }
488
479489 const Value *V = (*MI->memoperands_begin())->getValue();
480490 if (!V)
481491 return true;
484494 getUnderlyingObjects(V, Objs);
485495 for (SmallVectorImpl::iterator I = Objs.begin(),
486496 IE = Objs.end(); I != IE; ++I) {
487 V = *I;
488
489 if (const PseudoSourceValue *PSV = dyn_cast(V)) {
490 // Similarly to getUnderlyingObjectForInstr:
491 // For now, ignore PseudoSourceValues which may alias LLVM IR values
492 // because the code that uses this function has no way to cope with
493 // such aliases.
494 if (PSV->isAliased(MFI))
495 return true;
496 }
497
498497 // Does this pointer refer to a distinct and identifiable object?
499 if (!isIdentifiedObject(V))
498 if (!isIdentifiedObject(*I))
500499 return true;
501500 }
502501
533532
534533 MachineMemOperand *MMOa = *MIa->memoperands_begin();
535534 MachineMemOperand *MMOb = *MIb->memoperands_begin();
535
536 if (!MMOa->getValue() || !MMOb->getValue())
537 return true;
536538
537539 // The following interface to AA is fashioned after DAGCombiner::isAlias
538540 // and operates with MachineMemOperand offset with some important
750752 // so that they can be given more precise dependencies. We track
751753 // separately the known memory locations that may alias and those
752754 // that are known not to alias
753 MapVector > AliasMemDefs, NonAliasMemDefs;
754 MapVector<const Value *, std::vector > AliasMemUses, NonAliasMemUses;
755 MapVector<ValueType, std::vector > AliasMemDefs, NonAliasMemDefs;
756 MapVector > AliasMemUses, NonAliasMemUses;
755757 std::set RejectMemNodes;
756758
757759 // Remove any stale debug info; sometimes BuildSchedGraph is called again
847849 if (isGlobalMemoryObject(AA, MI)) {
848850 // Be conservative with these and add dependencies on all memory
849851 // references, even those that are known to not alias.
850 for (MapVector<const Value *, std::vector >::iterator I =
852 for (MapVector<ValueType, std::vector >::iterator I =
851853 NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) {
852854 for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
853855 I->second[i]->addPred(SDep(SU, SDep::Barrier));
854856 }
855857 }
856 for (MapVector<const Value *, std::vector >::iterator I =
858 for (MapVector<ValueType, std::vector >::iterator I =
857859 NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) {
858860 for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
859861 SDep Dep(SU, SDep::Barrier);
887889 for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
888890 addChainDependency(AAForDep, MFI, SU, PendingLoads[k], RejectMemNodes,
889891 TrueMemOrderLatency);
890 for (MapVector<const Value *, std::vector >::iterator I =
892 for (MapVector<ValueType, std::vector >::iterator I =
891893 AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) {
892894 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
893895 addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes);
894896 }
895 for (MapVector<const Value *, std::vector >::iterator I =
897 for (MapVector<ValueType, std::vector >::iterator I =
896898 AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) {
897899 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
898900 addChainDependency(AAForDep, MFI, SU, I->second[i], RejectMemNodes,
915917 bool MayAlias = false;
916918 for (UnderlyingObjectsVector::iterator K = Objs.begin(), KE = Objs.end();
917919 K != KE; ++K) {
918 const Value *V = K->getPointer();
920 ValueType V = K->getPointer();
919921 bool ThisMayAlias = K->getInt();
920922 if (ThisMayAlias)
921923 MayAlias = true;
923925 // A store to a specific PseudoSourceValue. Add precise dependencies.
924926 // Record the def in MemDefs, first adding a dep if there is
925927 // an existing def.
926 MapVector<const Value *, std::vector >::iterator I =
928 MapVector<ValueType, std::vector >::iterator I =
927929 ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V));
928 MapVector<const Value *, std::vector >::iterator IE =
930 MapVector<ValueType, std::vector >::iterator IE =
929931 ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
930932 if (I != IE) {
931933 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
948950 }
949951 }
950952 // Handle the uses in MemUses, if there are any.
951 MapVector<const Value *, std::vector >::iterator J =
953 MapVector<ValueType, std::vector >::iterator J =
952954 ((ThisMayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V));
953 MapVector<const Value *, std::vector >::iterator JE =
955 MapVector<ValueType, std::vector >::iterator JE =
954956 ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end());
955957 if (J != JE) {
956958 for (unsigned i = 0, e = J->second.size(); i != e; ++i)
995997 if (Objs.empty()) {
996998 // A load with no underlying object. Depend on all
997999 // potentially aliasing stores.
998 for (MapVector<const Value *, std::vector >::iterator I =
1000 for (MapVector<ValueType, std::vector >::iterator I =
9991001 AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I)
10001002 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
10011003 addChainDependency(AAForDep, MFI, SU, I->second[i],
10091011
10101012 for (UnderlyingObjectsVector::iterator
10111013 J = Objs.begin(), JE = Objs.end(); J != JE; ++J) {
1012 const Value *V = J->getPointer();
1014 ValueType V = J->getPointer();
10131015 bool ThisMayAlias = J->getInt();
10141016
10151017 if (ThisMayAlias)
10161018 MayAlias = true;
10171019
10181020 // A load from a specific PseudoSourceValue. Add precise dependencies.
1019 MapVector<const Value *, std::vector >::iterator I =
1021 MapVector<ValueType, std::vector >::iterator I =
10201022 ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V));
1021 MapVector<const Value *, std::vector >::iterator IE =
1023 MapVector<ValueType, std::vector >::iterator IE =
10221024 ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
10231025 if (I != IE)
10241026 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
316316
317317 /// isAlias - Return true if there is any possibility that the two addresses
318318 /// overlap.
319 bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1,
320 const Value *SrcValue1, int SrcValueOffset1,
321 unsigned SrcValueAlign1,
322 const MDNode *TBAAInfo1,
323 SDValue Ptr2, int64_t Size2, bool IsVolatile2,
324 const Value *SrcValue2, int SrcValueOffset2,
325 unsigned SrcValueAlign2,
326 const MDNode *TBAAInfo2) const;
327
328 /// isAlias - Return true if there is any possibility that the two addresses
329 /// overlap.
330 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1);
331
332 /// FindAliasInfo - Extracts the relevant alias information from the memory
333 /// node. Returns true if the operand was a load.
334 bool FindAliasInfo(SDNode *N,
335 SDValue &Ptr, int64_t &Size, bool &IsVolatile,
336 const Value *&SrcValue, int &SrcValueOffset,
337 unsigned &SrcValueAlignment,
338 const MDNode *&TBAAInfo) const;
319 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
339320
340321 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
341322 /// looking for a better chain (aliasing node.)
1129211273
1129311274 /// isAlias - Return true if there is any possibility that the two addresses
1129411275 /// overlap.
11295 bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1,
11296 const Value *SrcValue1, int SrcValueOffset1,
11297 unsigned SrcValueAlign1,
11298 const MDNode *TBAAInfo1,
11299 SDValue Ptr2, int64_t Size2, bool IsVolatile2,
11300 const Value *SrcValue2, int SrcValueOffset2,
11301 unsigned SrcValueAlign2,
11302 const MDNode *TBAAInfo2) const {
11276 bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const {
1130311277 // If they are the same then they must be aliases.
11304 if (Ptr1 == Ptr2) return true;
11278 if (Op0->getBasePtr() == Op1->getBasePtr()) return true;
1130511279
1130611280 // If they are both volatile then they cannot be reordered.
11307 if (IsVolatile1 && IsVolatile2) return true;
11281 if (Op0->isVolatile() && Op1->isVolatile()) return true;
1130811282
1130911283 // Gather base node and offset information.
1131011284 SDValue Base1, Base2;
1131111285 int64_t Offset1, Offset2;
1131211286 const GlobalValue *GV1, *GV2;
1131311287 const void *CV1, *CV2;
11314 bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1);
11315 bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2);
11288 bool isFrameIndex1 = FindBaseOffset(Op0->getBasePtr(),
11289 Base1, Offset1, GV1, CV1);
11290 bool isFrameIndex2 = FindBaseOffset(Op1->getBasePtr(),
11291 Base2, Offset2, GV2, CV2);
1131611292
1131711293 // If they have a same base address then check to see if they overlap.
1131811294 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
11319 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
11295 return !((Offset1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= Offset2 ||
11296 (Offset2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= Offset1);
1132011297
1132111298 // It is possible for different frame indices to alias each other, mostly
1132211299 // when tail call optimization reuses return address slots for arguments.
1132611303 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
1132711304 Offset1 += MFI->getObjectOffset(cast(Base1)->getIndex());
1132811305 Offset2 += MFI->getObjectOffset(cast(Base2)->getIndex());
11329 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
11306 return !((Offset1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= Offset2 ||
11307 (Offset2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= Offset1);
1133011308 }
1133111309
1133211310 // Otherwise, if we know what the bases are, and they aren't identical, then
1133811316 // compared to the size and offset of the access, we may be able to prove they
1133911317 // do not alias. This check is conservative for now to catch cases created by
1134011318 // splitting vector types.
11341 if ((SrcValueAlign1 == SrcValueAlign2) &&
11342 (SrcValueOffset1 != SrcValueOffset2) &&
11343 (Size1 == Size2) && (SrcValueAlign1 > Size1)) {
11344 int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1;
11345 int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1;
11319 if ((Op0->getOriginalAlignment() == Op1->getOriginalAlignment()) &&
11320 (Op0->getSrcValueOffset() != Op1->getSrcValueOffset()) &&
11321 (Op0->getMemoryVT().getSizeInBits() >> 3 ==
11322 Op1->getMemoryVT().getSizeInBits() >> 3) &&
11323 (Op0->getOriginalAlignment() > Op0->getMemoryVT().getSizeInBits()) >> 3) {
11324 int64_t OffAlign1 = Op0->getSrcValueOffset() % Op0->getOriginalAlignment();
11325 int64_t OffAlign2 = Op1->getSrcValueOffset() % Op1->getOriginalAlignment();
1134611326
1134711327 // There is no overlap between these relatively aligned accesses of similar
1134811328 // size, return no alias.
11349 if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1)
11329 if ((OffAlign1 + (Op0->getMemoryVT().getSizeInBits() >> 3)) <= OffAlign2 ||
11330 (OffAlign2 + (Op1->getMemoryVT().getSizeInBits() >> 3)) <= OffAlign1)
1135011331 return false;
1135111332 }
1135211333
1135711338 CombinerAAOnlyFunc != DAG.getMachineFunction().getName())
1135811339 UseAA = false;
1135911340 #endif
11360 if (UseAA && SrcValue1 && SrcValue2) {
11341 if (UseAA &&
11342 Op0->getMemOperand()->getValue() && Op1->getMemOperand()->getValue()) {
1136111343 // Use alias analysis information.
11362 int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2);
11363 int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset;
11364 int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset;
11344 int64_t MinOffset = std::min(Op0->getSrcValueOffset(),
11345 Op1->getSrcValueOffset());
11346 int64_t Overlap1 = (Op0->getMemoryVT().getSizeInBits() >> 3) +
11347 Op0->getSrcValueOffset() - MinOffset;
11348 int64_t Overlap2 = (Op1->getMemoryVT().getSizeInBits() >> 3) +
11349 Op1->getSrcValueOffset() - MinOffset;
1136511350 AliasAnalysis::AliasResult AAResult =
11366 AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1,
11367 UseTBAA ? TBAAInfo1 : nullptr),
11368 AliasAnalysis::Location(SrcValue2, Overlap2,
11369 UseTBAA ? TBAAInfo2 : nullptr));
11351 AA.alias(AliasAnalysis::Location(Op0->getMemOperand()->getValue(),
11352 Overlap1,
11353 UseTBAA ? Op0->getTBAAInfo() : nullptr),
11354 AliasAnalysis::Location(Op1->getMemOperand()->getValue(),
11355 Overlap2,
11356 UseTBAA ? Op1->getTBAAInfo() : nullptr));
1137011357 if (AAResult == AliasAnalysis::NoAlias)
1137111358 return false;
1137211359 }
1137311360
1137411361 // Otherwise we have to assume they alias.
1137511362 return true;
11376 }
11377
11378 bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) {
11379 SDValue Ptr0, Ptr1;
11380 int64_t Size0, Size1;
11381 bool IsVolatile0, IsVolatile1;
11382 const Value *SrcValue0, *SrcValue1;
11383 int SrcValueOffset0, SrcValueOffset1;
11384 unsigned SrcValueAlign0, SrcValueAlign1;
11385 const MDNode *SrcTBAAInfo0, *SrcTBAAInfo1;
11386 FindAliasInfo(Op0, Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0,
11387 SrcValueAlign0, SrcTBAAInfo0);
11388 FindAliasInfo(Op1, Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1,
11389 SrcValueAlign1, SrcTBAAInfo1);
11390 return isAlias(Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0,
11391 SrcValueAlign0, SrcTBAAInfo0,
11392 Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1,
11393 SrcValueAlign1, SrcTBAAInfo1);
11394 }
11395
11396 /// FindAliasInfo - Extracts the relevant alias information from the memory
11397 /// node. Returns true if the operand was a nonvolatile load.
11398 bool DAGCombiner::FindAliasInfo(SDNode *N,
11399 SDValue &Ptr, int64_t &Size, bool &IsVolatile,
11400 const Value *&SrcValue,
11401 int &SrcValueOffset,
11402 unsigned &SrcValueAlign,
11403 const MDNode *&TBAAInfo) const {
11404 LSBaseSDNode *LS = cast(N);
11405
11406 Ptr = LS->getBasePtr();
11407 Size = LS->getMemoryVT().getSizeInBits() >> 3;
11408 IsVolatile = LS->isVolatile();
11409 SrcValue = LS->getSrcValue();
11410 SrcValueOffset = LS->getSrcValueOffset();
11411 SrcValueAlign = LS->getOriginalAlignment();
11412 TBAAInfo = LS->getTBAAInfo();
11413 return isa(LS) && !IsVolatile;
1141411363 }
1141511364
1141611365 /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
1142111370 SmallPtrSet Visited; // Visited node set.
1142211371
1142311372 // Get alias information for node.
11424 SDValue Ptr;
11425 int64_t Size;
11426 bool IsVolatile;
11427 const Value *SrcValue;
11428 int SrcValueOffset;
11429 unsigned SrcValueAlign;
11430 const MDNode *SrcTBAAInfo;
11431 bool IsLoad = FindAliasInfo(N, Ptr, Size, IsVolatile, SrcValue,
11432 SrcValueOffset, SrcValueAlign, SrcTBAAInfo);
11373 bool IsLoad = isa(N) && !cast(N)->isVolatile();
1143311374
1143411375 // Starting off.
1143511376 Chains.push_back(OriginalChain);
1146811409 case ISD::LOAD:
1146911410 case ISD::STORE: {
1147011411 // Get alias information for Chain.
11471 SDValue OpPtr;
11472 int64_t OpSize;
11473 bool OpIsVolatile;
11474 const Value *OpSrcValue;
11475 int OpSrcValueOffset;
11476 unsigned OpSrcValueAlign;
11477 const MDNode *OpSrcTBAAInfo;
11478 bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize,
11479 OpIsVolatile, OpSrcValue, OpSrcValueOffset,
11480 OpSrcValueAlign,
11481 OpSrcTBAAInfo);
11412 bool IsOpLoad = isa(Chain.getNode()) &&
11413 !cast(Chain.getNode())->isVolatile();
1148211414
1148311415 // If chain is alias then stop here.
1148411416 if (!(IsLoad && IsOpLoad) &&
11485 isAlias(Ptr, Size, IsVolatile, SrcValue, SrcValueOffset,
11486 SrcValueAlign, SrcTBAAInfo,
11487 OpPtr, OpSize, OpIsVolatile, OpSrcValue, OpSrcValueOffset,
11488 OpSrcValueAlign, OpSrcTBAAInfo)) {
11417 isAlias(cast(N), cast(Chain.getNode()))) {
1148911418 Aliases.push_back(Chain);
1149011419 } else {
1149111420 // Look further up the chain.
43884388 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
43894389 EVT VT, SDValue Chain,
43904390 SDValue Ptr,
4391 const Value* PtrVal,
4392 unsigned Alignment,
4393 AtomicOrdering Ordering,
4394 SynchronizationScope SynchScope) {
4395 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4396 Alignment = getEVTAlignment(MemVT);
4397
4398 MachineFunction &MF = getMachineFunction();
4399 // An atomic store does not load. An atomic load does not store.
4400 // (An atomicrmw obviously both loads and stores.)
4401 // For now, atomics are considered to be volatile always, and they are
4402 // chained as such.
4403 // FIXME: Volatile isn't really correct; we should keep track of atomic
4404 // orderings in the memoperand.
4405 unsigned Flags = MachineMemOperand::MOVolatile;
4406 if (Opcode != ISD::ATOMIC_STORE)
4407 Flags |= MachineMemOperand::MOLoad;
4408 if (Opcode != ISD::ATOMIC_LOAD)
4409 Flags |= MachineMemOperand::MOStore;
4410
4411 MachineMemOperand *MMO =
4412 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4413 MemVT.getStoreSize(), Alignment);
4414
4415 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,
4416 Ordering, SynchScope);
4417 }
4418
4419 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4420 EVT VT, SDValue Chain,
4421 SDValue Ptr,
44224391 MachineMemOperand *MMO,
44234392 AtomicOrdering Ordering,
44244393 SynchronizationScope SynchScope) {
45734542
45744543 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
45754544 // clients.
4576 if (PtrInfo.V == nullptr)
4545 if (PtrInfo.V.isNull())
45774546 PtrInfo = InferPointerInfo(Ptr, Offset);
45784547
45794548 MachineFunction &MF = getMachineFunction();
47004669 if (isNonTemporal)
47014670 Flags |= MachineMemOperand::MONonTemporal;
47024671
4703 if (PtrInfo.V == nullptr)
4672 if (PtrInfo.V.isNull())
47044673 PtrInfo = InferPointerInfo(Ptr);
47054674
47064675 MachineFunction &MF = getMachineFunction();
47554724 if (isNonTemporal)
47564725 Flags |= MachineMemOperand::MONonTemporal;
47574726
4758 if (PtrInfo.V == nullptr)
4727 if (PtrInfo.V.isNull())
47594728 PtrInfo = InferPointerInfo(Ptr);
47604729
47614730 MachineFunction &MF = getMachineFunction();
36953695 if (I.getAlignment() < VT.getSizeInBits() / 8)
36963696 report_fatal_error("Cannot generate unaligned atomic load");
36973697
3698 MachineMemOperand *MMO =
3699 DAG.getMachineFunction().
3700 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3701 MachineMemOperand::MOVolatile |
3702 MachineMemOperand::MOLoad,
3703 VT.getStoreSize(),
3704 I.getAlignment() ? I.getAlignment() :
3705 DAG.getEVTAlignment(VT));
3706
36983707 InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
36993708 SDValue L =
3700 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3701 getValue(I.getPointerOperand()),
3702 I.getPointerOperand(), I.getAlignment(),
3703 TLI->getInsertFencesForAtomic() ? Monotonic : Order,
3704 Scope);
3709 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3710 getValue(I.getPointerOperand()), MMO,
3711 TLI->getInsertFencesForAtomic() ? Monotonic : Order,
3712 Scope);
37053713
37063714 SDValue OutChain = L.getValue(1);
37073715
508508
509509 // Update the MachineMemOperand to use the new alloca.
510510 for (MachineMemOperand *MMO : I.memoperands()) {
511 const Value *V = MMO->getValue();
512
513 if (!V)
514 continue;
515
516511 // FIXME: In order to enable the use of TBAA when using AA in CodeGen,
517512 // we'll also need to update the TBAA nodes in MMOs with values
518513 // derived from the merged allocas. When doing this, we'll need to use
522517
523518 // We've replaced IR-level uses of the remapped allocas, so we only
524519 // need to replace direct uses here.
525 if (!isa(V))
526 continue;
527
528 const AllocaInst *AI= cast(V);
520 const AllocaInst *AI = dyn_cast_or_null(MMO->getValue());
521 if (!AI)
522 continue;
523
529524 if (!Allocas.count(AI))
530525 continue;
531526
160160 for (MachineInstr::mmo_iterator MMOI = MI->memoperands_begin(),
161161 EE = MI->memoperands_end(); MMOI != EE; ++MMOI) {
162162 MachineMemOperand *MMO = *MMOI;
163 if (const Value *V = MMO->getValue()) {
164 if (const FixedStackPseudoSourceValue *FSV =
165 dyn_cast(V)) {
166 int FI = FSV->getFrameIndex();
167 if (FI >= 0)
168 SSRefs[FI].push_back(MMO);
169 }
163 if (const FixedStackPseudoSourceValue *FSV =
164 dyn_cast_or_null(
165 MMO->getPseudoValue())) {
166 int FI = FSV->getFrameIndex();
167 if (FI >= 0)
168 SSRefs[FI].push_back(MMO);
170169 }
171170 }
172171 }
309308 if (NewFI == -1 || (NewFI == (int)SS))
310309 continue;
311310
312 const Value *NewSV = PseudoSourceValue::getFixedStack(NewFI);
311 const PseudoSourceValue *NewSV = PseudoSourceValue::getFixedStack(NewFI);
313312 SmallVectorImpl &RefMMOs = SSRefs[SS];
314313 for (unsigned i = 0, e = RefMMOs.size(); i != e; ++i)
315314 RefMMOs[i]->setValue(NewSV);
249249 oe = MI->memoperands_end();
250250 o != oe;
251251 ++o) {
252 if ((*o)->isLoad() && (*o)->getValue())
252 if ((*o)->isLoad()) {
253253 if (const FixedStackPseudoSourceValue *Value =
254 dyn_cast((*o)->getValue())) {
254 dyn_cast_or_null(
255 (*o)->getPseudoValue())) {
255256 FrameIndex = Value->getFrameIndex();
256257 MMO = *o;
257258 return true;
258259 }
260 }
259261 }
260262 return false;
261263 }
267269 oe = MI->memoperands_end();
268270 o != oe;
269271 ++o) {
270 if ((*o)->isStore() && (*o)->getValue())
272 if ((*o)->isStore()) {
271273 if (const FixedStackPseudoSourceValue *Value =
272 dyn_cast((*o)->getValue())) {
274 dyn_cast_or_null(
275 (*o)->getPseudoValue())) {
273276 FrameIndex = Value->getFrameIndex();
274277 MMO = *o;
275278 return true;
276279 }
280 }
277281 }
278282 return false;
279283 }
11251125 return;
11261126 }
11271127
1128 if (V->getValueID() == Value::PseudoSourceValueVal ||
1129 V->getValueID() == Value::FixedStackPseudoSourceValueVal) {
1130 V->print(Out);
1131 return;
1132 }
1133
11341128 char Prefix = '%';
11351129 int Slot;
11361130 // If we have a SlotTracker, use it.
16581658 Flags.getByValAlign(),
16591659 /*isVolatile = */ false,
16601660 /*alwaysInline = */ false,
1661 DstInfo, MachinePointerInfo(0));
1661 DstInfo, MachinePointerInfo());
16621662 MemOpChains.push_back(Cpy);
16631663 } else {
16641664 // Normal stack argument, put it where it's needed.
141141 MemDefsUses(const MachineFrameInfo *MFI);
142142
143143 private:
144 typedef PointerUnion ValueType;
145
144146 virtual bool hasHazard_(const MachineInstr &MI);
145147
146148 /// Update Defs and Uses. Return true if there exist dependences that
147149 /// disqualify the delay slot candidate between V and values in Uses and
148150 /// Defs.
149 bool updateDefsUses(const Value *V, bool MayStore);
151 bool updateDefsUses(ValueType V, bool MayStore);
150152
151153 /// Get the list of underlying objects of MI's memory operand.
152154 bool getUnderlyingObjects(const MachineInstr &MI,
153 SmallVectorImpl<const Value *> &Objects) const;
155 SmallVectorImpl<ValueType> &Objects) const;
154156
155157 const MachineFrameInfo *MFI;
156 SmallPtrSet<const Value*, 4> Uses, Defs;
158 SmallPtrSet<ValueType, 4> Uses, Defs;
157159
158160 /// Flags indicating whether loads or stores with no underlying objects have
159161 /// been seen.
398400 if (MI.mayStore())
399401 return true;
400402
401 if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getValue())
403 if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getPseudoValue())
402404 return true;
403405
404 const Value *V = (*MI.memoperands_begin())->getValue();
405
406 if (isa(V))
407 return false;
408
409 if (const PseudoSourceValue *PSV = dyn_cast(V))
410 return !PSV->isConstant(0) && V != PseudoSourceValue::getStack();
406 if (const PseudoSourceValue *PSV =
407 (*MI.memoperands_begin())->getPseudoValue()) {
408 if (isa(PSV))
409 return false;
410 return !PSV->isConstant(0) && PSV != PseudoSourceValue::getStack();
411 }
411412
412413 return true;
413414 }
418419
419420 bool MemDefsUses::hasHazard_(const MachineInstr &MI) {
420421 bool HasHazard = false;
421 SmallVector<const Value *, 4> Objs;
422 SmallVector<ValueType, 4> Objs;
422423
423424 // Check underlying object list.
424425 if (getUnderlyingObjects(MI, Objs)) {
425 for (SmallVectorImpl<const Value *>::const_iterator I = Objs.begin();
426 for (SmallVectorImpl<ValueType>::const_iterator I = Objs.begin();
426427 I != Objs.end(); ++I)
427428 HasHazard |= updateDefsUses(*I, MI.mayStore());
428429
439440 return HasHazard;
440441 }
441442
442 bool MemDefsUses::updateDefsUses(const Value *V, bool MayStore) {
443 bool MemDefsUses::updateDefsUses(ValueType V, bool MayStore) {
443444 if (MayStore)
444445 return !Defs.insert(V) || Uses.count(V) || SeenNoObjStore || SeenNoObjLoad;
445446
449450
450451 bool MemDefsUses::
451452 getUnderlyingObjects(const MachineInstr &MI,
452 SmallVectorImpl &Objects) const {
453 if (!MI.hasOneMemOperand() || !(*MI.memoperands_begin())->getValue())
454 return false;
453 SmallVectorImpl &Objects) const {
454 if (!MI.hasOneMemOperand() ||
455 (!(*MI.memoperands_begin())->getValue() &&
456 !(*MI.memoperands_begin())->getPseudoValue()))
457 return false;
458
459 if (const PseudoSourceValue *PSV =
460 (*MI.memoperands_begin())->getPseudoValue()) {
461 if (!PSV->isAliased(MFI))
462 return false;
463 Objects.push_back(PSV);
464 return true;
465 }
455466
456467 const Value *V = (*MI.memoperands_begin())->getValue();
457468
460471
461472 for (SmallVectorImpl::iterator I = Objs.begin(), E = Objs.end();
462473 I != E; ++I) {
463 if (const PseudoSourceValue *PSV = dyn_cast(*I)) {
464 if (PSV->isAliased(MFI))
465 return false;
466 } else if (!isIdentifiedObject(V))
474 if (!isIdentifiedObject(V))
467475 return false;
468476
469477 Objects.push_back(*I);
35493549 DAG.getIntPtrConstant(ByVal.Address));
35503550 Chain = DAG.getMemcpy(Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, PtrTy),
35513551 Alignment, /*isVolatile=*/false, /*AlwaysInline=*/false,
3552 MachinePointerInfo(0), MachinePointerInfo(0));
3552 MachinePointerInfo(), MachinePointerInfo());
35533553 MemOpChains.push_back(Chain);
35543554 }
35553555
35913591 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy());
35923592 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
35933593 MachinePointerInfo(), false, false, 0);
3594 cast(Store.getNode())->getMemOperand()->setValue(0);
3594 cast(Store.getNode())->getMemOperand()->setValue((Value*)0);
35953595 OutChains.push_back(Store);
35963596 }
35973597 }
3434 cl::Hidden);
3535
3636 namespace {
37 typedef PointerUnion ValueType;
38
3739 typedef std::pair CntRegP;
3840 typedef RecyclingAllocator
39 ScopedHashTableVal<const Value *, CntRegP> >
41 ScopedHashTableVal<ValueType, CntRegP> >
4042 AllocatorTy;
41 typedef ScopedHashTable<const Value *, CntRegP, DenseMapInfo>,
43 typedef ScopedHashTable<ValueType, CntRegP, DenseMapInfo>,
4244 AllocatorTy> ScopedHTType;
4345
4446 class MBBInfo {
7779 /// and the underlying object in Reg and Val respectively, if the function's
7880 /// address can be resolved lazily.
7981 bool isCallViaRegister(MachineInstr &MI, unsigned &Reg,
80 const Value *&Val) const;
82 ValueType &Val) const;
8183
8284 /// \brief Return the number of instructions that dominate the current
8385 /// instruction and load the function address from object Entry.
84 unsigned getCount(const Value *Entry);
86 unsigned getCount(ValueType Entry);
8587
8688 /// \brief Return the destination virtual register of the last instruction
8789 /// that loads from object Entry.
88 unsigned getReg(const Value *Entry);
90 unsigned getReg(ValueType Entry);
8991
9092 /// \brief Update ScopedHT.
91 void incCntAndSetReg(const Value *Entry, unsigned Reg);
93 void incCntAndSetReg(ValueType Entry, unsigned Reg);
9294
9395 ScopedHTType ScopedHT;
9496 static char ID;
209211 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
210212 ++I) {
211213 unsigned Reg;
212 const Value *Entry;
214 ValueType Entry;
213215
214216 // Skip instructions that are not call instructions via registers.
215217 if (!isCallViaRegister(*I, Reg, Entry))
241243 }
242244
243245 bool OptimizePICCall::isCallViaRegister(MachineInstr &MI, unsigned &Reg,
244 const Value *&Val) const {
246 ValueType &Val) const {
245247 if (!MI.isCall())
246248 return false;
247249
253255
254256 // Get the instruction that loads the function address from the GOT.
255257 Reg = MO->getReg();
256 Val = 0;
258 Val = (Value*)0;
257259 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
258260 MachineInstr *DefMI = MRI.getVRegDef(Reg);
259261
272274 // Return the underlying object for the GOT entry in Val.
273275 assert(DefMI->hasOneMemOperand());
274276 Val = (*DefMI->memoperands_begin())->getValue();
277 if (!Val)
278 Val = (*DefMI->memoperands_begin())->getPseudoValue();
275279 return true;
276280 }
277281
278 unsigned OptimizePICCall::getCount(const Value *Entry) {
282 unsigned OptimizePICCall::getCount(ValueType Entry) {
279283 return ScopedHT.lookup(Entry).first;
280284 }
281285
282 unsigned OptimizePICCall::getReg(const Value *Entry) {
286 unsigned OptimizePICCall::getReg(ValueType Entry) {
283287 unsigned Reg = ScopedHT.lookup(Entry).second;
284288 assert(Reg);
285289 return Reg;
286290 }
287291
288 void OptimizePICCall::incCntAndSetReg(const Value *Entry, unsigned Reg) {
292 void OptimizePICCall::incCntAndSetReg(ValueType Entry, unsigned Reg) {
289293 CntRegP P = ScopedHT.lookup(Entry);
290294 ScopedHT.insert(Entry, std::make_pair(P.first + 1, Reg));
291295 }
266266
267267 static unsigned int getCodeAddrSpace(MemSDNode *N,
268268 const NVPTXSubtarget &Subtarget) {
269 const Value *Src = N->getSrcValue();
269 const Value *Src = N->getMemOperand()->getValue();
270270
271271 if (!Src)
272272 return NVPTX::PTXLdStInstCode::GENERIC;
30603060 // the classof() for MemSDNode does not include MemIntrinsicSDNode
30613061 // (See SelectionDAGNodes.h). So we need to check for both.
30623062 if (MemSDNode *mN = dyn_cast(N)) {
3063 Src = mN->getSrcValue();
3063 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
3064 return true;
3065 Src = mN->getMemOperand()->getValue();
30643066 } else if (MemSDNode *mN = dyn_cast(N)) {
3065 Src = mN->getSrcValue();
3067 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
3068 return true;
3069 Src = mN->getMemOperand()->getValue();
30663070 }
30673071 if (!Src)
30683072 return false;
33143314 SDLoc dl) {
33153315 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
33163316 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
3317 false, false, MachinePointerInfo(0),
3318 MachinePointerInfo(0));
3317 false, false, MachinePointerInfo(),
3318 MachinePointerInfo());
33193319 }
33203320
33213321 /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
6060 bool SelectADDR64(SDValue N, SDValue &R1, SDValue &R2);
6161
6262 static bool checkType(const Value *ptr, unsigned int addrspace);
63 static bool checkPrivateAddress(const MachineMemOperand *Op);
6364
6465 static bool isGlobalStore(const StoreSDNode *N);
6566 static bool isPrivateStore(const StoreSDNode *N);
430431
431432
432433 bool AMDGPUDAGToDAGISel::checkType(const Value *ptr, unsigned int addrspace) {
434 assert(addrspace != 0 && "Use checkPrivateAddress instead.");
433435 if (!ptr) {
434436 return false;
435437 }
437439 return dyn_cast(ptrType)->getAddressSpace() == addrspace;
438440 }
439441
442 bool AMDGPUDAGToDAGISel::checkPrivateAddress(const MachineMemOperand *Op) {
443 if (Op->getPseudoValue()) return true;
444 const Value *ptr = Op->getValue();
445 if (!ptr) return false;
446 PointerType *ptrType = dyn_cast(ptr->getType());
447 return ptrType->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS;
448 }
449
440450 bool AMDGPUDAGToDAGISel::isGlobalStore(const StoreSDNode *N) {
441 return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS);
451 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS);
442452 }
443453
444454 bool AMDGPUDAGToDAGISel::isPrivateStore(const StoreSDNode *N) {
445 return (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS)
446 && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS)
447 && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS));
455 return (!checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS)
456 && !checkType(N->getMemOperand()->getValue(),
457 AMDGPUAS::GLOBAL_ADDRESS)
458 && !checkType(N->getMemOperand()->getValue(),
459 AMDGPUAS::REGION_ADDRESS));
448460 }
449461
450462 bool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) {
451 return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS);
463 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS);
452464 }
453465
454466 bool AMDGPUDAGToDAGISel::isRegionStore(const StoreSDNode *N) {
455 return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS);
467 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS);
456468 }
457469
458470 bool AMDGPUDAGToDAGISel::isConstantLoad(const LoadSDNode *N, int CbId) const {
459471 if (CbId == -1) {
460 return checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS);
461 }
462 return checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_BUFFER_0 + CbId);
472 return checkType(N->getMemOperand()->getValue(),
473 AMDGPUAS::CONSTANT_ADDRESS);
474 }
475 return checkType(N->getMemOperand()->getValue(),
476 AMDGPUAS::CONSTANT_BUFFER_0 + CbId);
463477 }
464478
465479 bool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) const {
470484 return true;
471485 }
472486 }
473 return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS);
487 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS);
474488 }
475489
476490 bool AMDGPUDAGToDAGISel::isParamLoad(const LoadSDNode *N) const {
477 return checkType(N->getSrcValue(), AMDGPUAS::PARAM_I_ADDRESS);
491 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS);
478492 }
479493
480494 bool AMDGPUDAGToDAGISel::isLocalLoad(const LoadSDNode *N) const {
481 return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS);
495 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS);
482496 }
483497
484498 bool AMDGPUDAGToDAGISel::isRegionLoad(const LoadSDNode *N) const {
485 return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS);
499 return checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS);
486500 }
487501
488502 bool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) const {
489503 MachineMemOperand *MMO = N->getMemOperand();
490 if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) {
504 if (checkPrivateAddress(N->getMemOperand())) {
491505 if (MMO) {
492 const Value *V = MMO->getValue();
493 const PseudoSourceValue *PSV = dyn_cast(V);
506 const PseudoSourceValue *PSV = MMO->getPseudoValue();
494507 if (PSV && PSV == PseudoSourceValue::getConstantPool()) {
495508 return true;
496509 }
500513 }
501514
502515 bool AMDGPUDAGToDAGISel::isPrivateLoad(const LoadSDNode *N) const {
503 if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) {
516 if (checkPrivateAddress(N->getMemOperand())) {
504517 // Check to make sure we are not a constant pool load or a constant load
505518 // that is marked as a private load
506519 if (isCPLoad(N) || isConstantLoad(N, -1)) {
507520 return false;
508521 }
509522 }
510 if (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS)
511 && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS)
512 && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS)
513 && !checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS)
514 && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_D_ADDRESS)
515 && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_I_ADDRESS)) {
523 if (!checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS)
524 && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS)
525 && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::REGION_ADDRESS)
526 && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::CONSTANT_ADDRESS)
527 && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_D_ADDRESS)
528 && !checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS)){
516529 return true;
517530 }
518531 return false;
777777
778778 // Lower loads constant address space global variable loads
779779 if (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
780 isa(GetUnderlyingObject(Load->getPointerInfo().V))) {
780 isa(
781 GetUnderlyingObject(Load->getMemOperand()->getValue()))) {
781782
782783 SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL,
783784 getPointerTy(AMDGPUAS::PRIVATE_ADDRESS));
12321232 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) ||
12331233 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) {
12341234 SDValue Result;
1235 if (isa(LoadNode->getSrcValue()) ||
1236 isa(LoadNode->getSrcValue()) ||
1235 if (isa(LoadNode->getMemOperand()->getValue()) ||
1236 isa(LoadNode->getMemOperand()->getValue()) ||
12371237 isa(Ptr)) {
12381238 SDValue Slots[4];
12391239 for (unsigned i = 0; i < 4; i++) {
985985 return true;
986986
987987 // Otherwise we need to check whether there's an alias.
988 const Value *V1 = Load->getSrcValue();
989 const Value *V2 = Store->getSrcValue();
988 const Value *V1 = Load->getMemOperand()->getValue();
989 const Value *V2 = Store->getMemOperand()->getValue();
990990 if (!V1 || !V2)
991991 return false;
992992
1372513725 cast(Node)->getMemoryVT(),
1372613726 Node->getOperand(0),
1372713727 Node->getOperand(1), negOp,
13728 cast(Node)->getSrcValue(),
13729 cast(Node)->getAlignment(),
13728 cast(Node)->getMemOperand(),
1373013729 cast(Node)->getOrdering(),
1373113730 cast(Node)->getSynchScope());
1373213731 }