llvm.org GIT mirror llvm / f96e4bd
Make CodeGen TBAA-aware. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@116890 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 9 years ago
10 changed file(s) with 92 addition(s) and 37 deletion(s). Raw diff Collapse all Expand all
370370 /// explicitly deallocated.
371371 MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo,
372372 unsigned f, uint64_t s,
373 unsigned base_alignment);
373 unsigned base_alignment,
374 const MDNode *TBAAInfo = 0);
374375
375376 /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
376377 /// an existing one, adjusting by an offset and using the given size.
8181 MachinePointerInfo PtrInfo;
8282 uint64_t Size;
8383 unsigned Flags;
84 const MDNode *TBAAInfo;
8485
8586 public:
8687 /// Flags values. These may be or'd together.
100101 /// MachineMemOperand - Construct an MachineMemOperand object with the
101102 /// specified PtrInfo, flags, size, and base alignment.
102103 MachineMemOperand(MachinePointerInfo PtrInfo, unsigned flags, uint64_t s,
103 unsigned base_alignment);
104 unsigned base_alignment, const MDNode *TBAAInfo = 0);
104105
105106 const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
106107
132133 /// base address, without the offset.
133134 uint64_t getBaseAlignment() const { return (1u << (Flags >> MOMaxBits)) >> 1; }
134135
136 /// getTBAAInfo - Return the TBAA tag for the memory reference.
137 const MDNode *getTBAAInfo() const { return TBAAInfo; }
138
135139 bool isLoad() const { return Flags & MOLoad; }
136140 bool isStore() const { return Flags & MOStore; }
137141 bool isVolatile() const { return Flags & MOVolatile; }
630630 ///
631631 SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr,
632632 MachinePointerInfo PtrInfo, bool isVolatile,
633 bool isNonTemporal, unsigned Alignment);
633 bool isNonTemporal, unsigned Alignment,
634 const MDNode *TBAAInfo = 0);
634635 SDValue getExtLoad(ISD::LoadExtType ExtType, EVT VT, DebugLoc dl,
635636 SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo,
636637 EVT MemVT, bool isVolatile,
637 bool isNonTemporal, unsigned Alignment);
638 bool isNonTemporal, unsigned Alignment,
639 const MDNode *TBAAInfo = 0);
638640 SDValue getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base,
639641 SDValue Offset, ISD::MemIndexedMode AM);
640642 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
641643 EVT VT, DebugLoc dl,
642644 SDValue Chain, SDValue Ptr, SDValue Offset,
643645 MachinePointerInfo PtrInfo, EVT MemVT,
644 bool isVolatile, bool isNonTemporal, unsigned Alignment);
646 bool isVolatile, bool isNonTemporal, unsigned Alignment,
647 const MDNode *TBAAInfo = 0);
645648 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
646649 EVT VT, DebugLoc dl,
647650 SDValue Chain, SDValue Ptr, SDValue Offset,
651654 ///
652655 SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
653656 MachinePointerInfo PtrInfo, bool isVolatile,
654 bool isNonTemporal, unsigned Alignment);
657 bool isNonTemporal, unsigned Alignment,
658 const MDNode *TBAAInfo = 0);
655659 SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
656660 MachineMemOperand *MMO);
657661 SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
658662 MachinePointerInfo PtrInfo, EVT TVT,
659663 bool isNonTemporal, bool isVolatile,
660 unsigned Alignment);
664 unsigned Alignment,
665 const MDNode *TBAAInfo = 0);
661666 SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
662667 EVT TVT, MachineMemOperand *MMO);
663668 SDValue getIndexedStore(SDValue OrigStoe, DebugLoc dl, SDValue Base,
901901 const Value *getSrcValue() const { return MMO->getValue(); }
902902 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
903903
904 /// Returns the TBAAInfo that describes the dereference.
905 const MDNode *getTBAAInfo() const { return MMO->getTBAAInfo(); }
906
904907 /// getMemoryVT - Return the type of the in-memory value.
905908 EVT getMemoryVT() const { return MemoryVT; }
906909
190190
191191 MachineMemOperand *
192192 MachineFunction::getMachineMemOperand(MachinePointerInfo PtrInfo, unsigned f,
193 uint64_t s, unsigned base_alignment) {
194 return new (Allocator) MachineMemOperand(PtrInfo, f, s, base_alignment);
193 uint64_t s, unsigned base_alignment,
194 const MDNode *TBAAInfo) {
195 return new (Allocator) MachineMemOperand(PtrInfo, f, s, base_alignment,
196 TBAAInfo);
195197 }
196198
197199 MachineMemOperand *
200202 return new (Allocator)
201203 MachineMemOperand(MachinePointerInfo(MMO->getValue(),
202204 MMO->getOffset()+Offset),
203 MMO->getFlags(), Size, MMO->getBaseAlignment());
205 MMO->getFlags(), Size,
206 MMO->getBaseAlignment(), 0);
204207 }
205208
206209 MachineInstr::mmo_iterator
230233 MachineMemOperand *JustLoad =
231234 getMachineMemOperand((*I)->getPointerInfo(),
232235 (*I)->getFlags() & ~MachineMemOperand::MOStore,
233 (*I)->getSize(), (*I)->getBaseAlignment());
236 (*I)->getSize(), (*I)->getBaseAlignment(),
237 (*I)->getTBAAInfo());
234238 Result[Index] = JustLoad;
235239 }
236240 ++Index;
261265 MachineMemOperand *JustStore =
262266 getMachineMemOperand((*I)->getPointerInfo(),
263267 (*I)->getFlags() & ~MachineMemOperand::MOLoad,
264 (*I)->getSize(), (*I)->getBaseAlignment());
268 (*I)->getSize(), (*I)->getBaseAlignment(),
269 (*I)->getTBAAInfo());
265270 Result[Index] = JustStore;
266271 }
267272 ++Index;
366366 }
367367
368368 MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, unsigned f,
369 uint64_t s, unsigned int a)
369 uint64_t s, unsigned int a,
370 const MDNode *TBAAInfo)
370371 : PtrInfo(ptrinfo), Size(s),
371 Flags((f & ((1 << MOMaxBits) - 1)) | ((Log2_32(a) + 1) << MOMaxBits)) {
372 Flags((f & ((1 << MOMaxBits) - 1)) | ((Log2_32(a) + 1) << MOMaxBits)),
373 TBAAInfo(TBAAInfo) {
372374 assert((PtrInfo.V == 0 || isa(PtrInfo.V->getType())) &&
373375 "invalid pointer value");
374376 assert(getBaseAlignment() == a && "Alignment is not a power of 2!");
440442 if (MMO.getBaseAlignment() != MMO.getAlignment() ||
441443 MMO.getBaseAlignment() != MMO.getSize())
442444 OS << "(align=" << MMO.getAlignment() << ")";
445
446 // Print TBAA info.
447 if (const MDNode *TBAAInfo = MMO.getTBAAInfo()) {
448 OS << "(tbaa=";
449 if (TBAAInfo->getNumOperands() > 0)
450 WriteAsOperand(OS, TBAAInfo->getOperand(0), /*PrintType=*/false);
451 else
452 OS << "";
453 OS << ")";
454 }
443455
444456 return OS;
445457 }
11971209 if (PSV->isConstant(MFI))
11981210 continue;
11991211 // If we have an AliasAnalysis, ask it whether the memory is constant.
1200 if (AA && AA->pointsToConstantMemory(V))
1212 if (AA && AA->pointsToConstantMemory(
1213 AliasAnalysis::Location(V, (*I)->getSize(),
1214 (*I)->getTBAAInfo())))
12011215 continue;
12021216 }
12031217
776776 MachineFunction &MF = *MI->getParent()->getParent();
777777 return PSV->isConstant(MF.getFrameInfo());
778778 } else {
779 return AA->pointsToConstantMemory(MMO->getValue());
779 return AA->pointsToConstantMemory(AliasAnalysis::Location(MMO->getValue(),
780 MMO->getSize(),
781 MMO->getTBAAInfo()));
780782 }
781783 }
782784
247247 bool isAlias(SDValue Ptr1, int64_t Size1,
248248 const Value *SrcValue1, int SrcValueOffset1,
249249 unsigned SrcValueAlign1,
250 const MDNode *TBAAInfo1,
250251 SDValue Ptr2, int64_t Size2,
251252 const Value *SrcValue2, int SrcValueOffset2,
252 unsigned SrcValueAlign2) const;
253 unsigned SrcValueAlign2,
254 const MDNode *TBAAInfo2) const;
253255
254256 /// FindAliasInfo - Extracts the relevant alias information from the memory
255257 /// node. Returns true if the operand was a load.
256258 bool FindAliasInfo(SDNode *N,
257259 SDValue &Ptr, int64_t &Size,
258260 const Value *&SrcValue, int &SrcValueOffset,
259 unsigned &SrcValueAlignment) const;
261 unsigned &SrcValueAlignment,
262 const MDNode *&TBAAInfo) const;
260263
261264 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
262265 /// looking for a better chain (aliasing node.)
70447047 bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1,
70457048 const Value *SrcValue1, int SrcValueOffset1,
70467049 unsigned SrcValueAlign1,
7050 const MDNode *TBAAInfo1,
70477051 SDValue Ptr2, int64_t Size2,
70487052 const Value *SrcValue2, int SrcValueOffset2,
7049 unsigned SrcValueAlign2) const {
7053 unsigned SrcValueAlign2,
7054 const MDNode *TBAAInfo2) const {
70507055 // If they are the same then they must be aliases.
70517056 if (Ptr1 == Ptr2) return true;
70527057
71007105 int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset;
71017106 int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset;
71027107 AliasAnalysis::AliasResult AAResult =
7103 AA.alias(SrcValue1, Overlap1, SrcValue2, Overlap2);
7108 AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, TBAAInfo1),
7109 AliasAnalysis::Location(SrcValue2, Overlap2, TBAAInfo2));
71047110 if (AAResult == AliasAnalysis::NoAlias)
71057111 return false;
71067112 }
71157121 SDValue &Ptr, int64_t &Size,
71167122 const Value *&SrcValue,
71177123 int &SrcValueOffset,
7118 unsigned &SrcValueAlign) const {
7124 unsigned &SrcValueAlign,
7125 const MDNode *&TBAAInfo) const {
71197126 if (LoadSDNode *LD = dyn_cast(N)) {
71207127 Ptr = LD->getBasePtr();
71217128 Size = LD->getMemoryVT().getSizeInBits() >> 3;
71227129 SrcValue = LD->getSrcValue();
71237130 SrcValueOffset = LD->getSrcValueOffset();
71247131 SrcValueAlign = LD->getOriginalAlignment();
7132 TBAAInfo = LD->getTBAAInfo();
71257133 return true;
71267134 } else if (StoreSDNode *ST = dyn_cast(N)) {
71277135 Ptr = ST->getBasePtr();
71297137 SrcValue = ST->getSrcValue();
71307138 SrcValueOffset = ST->getSrcValueOffset();
71317139 SrcValueAlign = ST->getOriginalAlignment();
7140 TBAAInfo = ST->getTBAAInfo();
71327141 } else {
71337142 llvm_unreachable("FindAliasInfo expected a memory operand");
71347143 }
71497158 const Value *SrcValue;
71507159 int SrcValueOffset;
71517160 unsigned SrcValueAlign;
7161 const MDNode *SrcTBAAInfo;
71527162 bool IsLoad = FindAliasInfo(N, Ptr, Size, SrcValue, SrcValueOffset,
7153 SrcValueAlign);
7163 SrcValueAlign, SrcTBAAInfo);
71547164
71557165 // Starting off.
71567166 Chains.push_back(OriginalChain);
71947204 const Value *OpSrcValue;
71957205 int OpSrcValueOffset;
71967206 unsigned OpSrcValueAlign;
7207 const MDNode *OpSrcTBAAInfo;
71977208 bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize,
71987209 OpSrcValue, OpSrcValueOffset,
7199 OpSrcValueAlign);
7210 OpSrcValueAlign,
7211 OpSrcTBAAInfo);
72007212
72017213 // If chain is alias then stop here.
72027214 if (!(IsLoad && IsOpLoad) &&
72037215 isAlias(Ptr, Size, SrcValue, SrcValueOffset, SrcValueAlign,
7216 SrcTBAAInfo,
72047217 OpPtr, OpSize, OpSrcValue, OpSrcValueOffset,
7205 OpSrcValueAlign)) {
7218 OpSrcValueAlign, OpSrcTBAAInfo)) {
72067219 Aliases.push_back(Chain);
72077220 } else {
72087221 // Look further up the chain.
38943894 SDValue Ptr, SDValue Offset,
38953895 MachinePointerInfo PtrInfo, EVT MemVT,
38963896 bool isVolatile, bool isNonTemporal,
3897 unsigned Alignment) {
3897 unsigned Alignment, const MDNode *TBAAInfo) {
38983898 if (Alignment == 0) // Ensure that codegen never sees alignment 0
38993899 Alignment = getEVTAlignment(VT);
39003900
39113911
39123912 MachineFunction &MF = getMachineFunction();
39133913 MachineMemOperand *MMO =
3914 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
3914 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
3915 TBAAInfo);
39153916 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
39163917 }
39173918
39653966 SDValue Chain, SDValue Ptr,
39663967 MachinePointerInfo PtrInfo,
39673968 bool isVolatile, bool isNonTemporal,
3968 unsigned Alignment) {
3969 unsigned Alignment, const MDNode *TBAAInfo) {
39693970 SDValue Undef = getUNDEF(Ptr.getValueType());
39703971 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
3971 PtrInfo, VT, isVolatile, isNonTemporal, Alignment);
3972 PtrInfo, VT, isVolatile, isNonTemporal, Alignment, TBAAInfo);
39723973 }
39733974
39743975 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, EVT VT, DebugLoc dl,
39753976 SDValue Chain, SDValue Ptr,
39763977 MachinePointerInfo PtrInfo, EVT MemVT,
39773978 bool isVolatile, bool isNonTemporal,
3978 unsigned Alignment) {
3979 unsigned Alignment, const MDNode *TBAAInfo) {
39793980 SDValue Undef = getUNDEF(Ptr.getValueType());
39803981 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
3981 PtrInfo, MemVT, isVolatile, isNonTemporal, Alignment);
3982 PtrInfo, MemVT, isVolatile, isNonTemporal, Alignment,
3983 TBAAInfo);
39823984 }
39833985
39843986
39973999 SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
39984000 SDValue Ptr, MachinePointerInfo PtrInfo,
39994001 bool isVolatile, bool isNonTemporal,
4000 unsigned Alignment) {
4002 unsigned Alignment, const MDNode *TBAAInfo) {
40014003 if (Alignment == 0) // Ensure that codegen never sees alignment 0
40024004 Alignment = getEVTAlignment(Val.getValueType());
40034005
40134015 MachineFunction &MF = getMachineFunction();
40144016 MachineMemOperand *MMO =
40154017 MF.getMachineMemOperand(PtrInfo, Flags,
4016 Val.getValueType().getStoreSize(), Alignment);
4018 Val.getValueType().getStoreSize(), Alignment,
4019 TBAAInfo);
40174020
40184021 return getStore(Chain, dl, Val, Ptr, MMO);
40194022 }
40444047 SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
40454048 SDValue Ptr, MachinePointerInfo PtrInfo,
40464049 EVT SVT,bool isVolatile, bool isNonTemporal,
4047 unsigned Alignment) {
4050 unsigned Alignment,
4051 const MDNode *TBAAInfo) {
40484052 if (Alignment == 0) // Ensure that codegen never sees alignment 0
40494053 Alignment = getEVTAlignment(SVT);
40504054
40594063
40604064 MachineFunction &MF = getMachineFunction();
40614065 MachineMemOperand *MMO =
4062 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment);
4066 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
4067 TBAAInfo);
40634068
40644069 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
40654070 }
29332933 bool isVolatile = I.isVolatile();
29342934 bool isNonTemporal = I.getMetadata("nontemporal") != 0;
29352935 unsigned Alignment = I.getAlignment();
2936 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
29362937
29372938 SmallVector ValueVTs;
29382939 SmallVector Offsets;
29462947 if (I.isVolatile())
29472948 // Serialize volatile loads with other side effects.
29482949 Root = getRoot();
2949 else if (AA->pointsToConstantMemory(SV)) {
2950 else if (AA->pointsToConstantMemory(
2951 AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) {
29502952 // Do not serialize (non-volatile) loads of constant memory with anything.
29512953 Root = DAG.getEntryNode();
29522954 ConstantMemory = true;
29642966 DAG.getConstant(Offsets[i], PtrVT));
29652967 SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root,
29662968 A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
2967 isNonTemporal, Alignment);
2969 isNonTemporal, Alignment, TBAAInfo);
29682970
29692971 Values[i] = L;
29702972 Chains[i] = L.getValue(1);
30073009 bool isVolatile = I.isVolatile();
30083010 bool isNonTemporal = I.getMetadata("nontemporal") != 0;
30093011 unsigned Alignment = I.getAlignment();
3012 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
30103013
30113014 for (unsigned i = 0; i != NumValues; ++i) {
30123015 SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr,
30143017 Chains[i] = DAG.getStore(Root, getCurDebugLoc(),
30153018 SDValue(Src.getNode(), Src.getResNo() + i),
30163019 Add, MachinePointerInfo(PtrV, Offsets[i]),
3017 isVolatile, isNonTemporal, Alignment);
3020 isVolatile, isNonTemporal, Alignment, TBAAInfo);
30183021 }
30193022
30203023 DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),