llvm.org GIT mirror llvm / e33c848
Reapply address space patch after fixing an issue in MemCopyOptimizer. Added support for address spaces and added a isVolatile field to memcpy, memmove, and memset, e.g., llvm.memcpy.i32(i8*, i8*, i32, i32) -> llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1) git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@100191 91177308-0d34-0410-b5e6-96231b3b80d8 Mon P Wang 10 years ago
28 changed file(s) with 317 addition(s) and 146 deletion(s). Raw diff Collapse all Expand all
533533 SDValue getStackArgumentTokenFactor(SDValue Chain);
534534
535535 SDValue getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src,
536 SDValue Size, unsigned Align, bool AlwaysInline,
536 SDValue Size, unsigned Align, bool isVol, bool AlwaysInline,
537537 const Value *DstSV, uint64_t DstSVOff,
538538 const Value *SrcSV, uint64_t SrcSVOff);
539539
540540 SDValue getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src,
541 SDValue Size, unsigned Align,
541 SDValue Size, unsigned Align, bool isVol,
542542 const Value *DstSV, uint64_t DstOSVff,
543543 const Value *SrcSV, uint64_t SrcSVOff);
544544
545545 SDValue getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src,
546 SDValue Size, unsigned Align,
546 SDValue Size, unsigned Align, bool isVol,
547547 const Value *DstSV, uint64_t DstSVOff);
548548
549549 /// getSetCC - Helper function to make it easier to build SetCC's if you just
132132 return getAlignmentCst()->getZExtValue();
133133 }
134134
135 ConstantInt *getVolatileCst() const {
136 return cast(const_cast(getOperand(5)));
137 }
138 bool isVolatile() const {
139 return getVolatileCst()->getZExtValue() != 0;
140 }
141
135142 /// getDest - This is just like getRawDest, but it strips off any cast
136143 /// instructions that feed it, giving the original input. The returned
137144 /// value is guaranteed to be a pointer.
154161 void setAlignment(Constant* A) {
155162 setOperand(4, A);
156163 }
157
164
165 void setVolatile(Constant* V) {
166 setOperand(5, V);
167 }
168
158169 const Type *getAlignmentType() const {
159170 return getOperand(4)->getType();
160171 }
223223 //
224224
225225 def int_memcpy : Intrinsic<[],
226 [llvm_ptr_ty, llvm_ptr_ty, llvm_anyint_ty,
227 llvm_i32_ty],
226 [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty,
227 llvm_i32_ty, llvm_i1_ty],
228228 [IntrWriteArgMem, NoCapture<0>, NoCapture<1>]>;
229229 def int_memmove : Intrinsic<[],
230 [llvm_ptr_ty, llvm_ptr_ty, llvm_anyint_ty,
231 llvm_i32_ty],
230 [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty,
231 llvm_i32_ty, llvm_i1_ty],
232232 [IntrWriteArgMem, NoCapture<0>, NoCapture<1>]>;
233233 def int_memset : Intrinsic<[],
234 [llvm_ptr_ty, llvm_i8_ty, llvm_anyint_ty,
235 llvm_i32_ty],
234 [llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty,
235 llvm_i32_ty, llvm_i1_ty],
236236 [IntrWriteArgMem, NoCapture<0>]>;
237237
238238 // These functions do not actually read memory, but they are sensitive to the
916916 Value *Args[] = { Arg1, Arg2, Arg3, Arg4 };
917917 return Insert(CallInst::Create(Callee, Args, Args+4), Name);
918918 }
919 CallInst *CreateCall5(Value *Callee, Value *Arg1, Value *Arg2, Value *Arg3,
920 Value *Arg4, Value *Arg5, const Twine &Name = "") {
921 Value *Args[] = { Arg1, Arg2, Arg3, Arg4, Arg5 };
922 return Insert(CallInst::Create(Callee, Args, Args+5), Name);
923 }
919924
920925 template
921926 CallInst *CreateCall(Value *Callee, InputIterator ArgBegin,
11861186 EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
11871187 SDValue Chain,
11881188 SDValue Op1, SDValue Op2,
1189 SDValue Op3, unsigned Align,
1189 SDValue Op3, unsigned Align, bool isVolatile,
11901190 bool AlwaysInline,
11911191 const Value *DstSV, uint64_t DstOff,
11921192 const Value *SrcSV, uint64_t SrcOff) {
12031203 EmitTargetCodeForMemmove(SelectionDAG &DAG, DebugLoc dl,
12041204 SDValue Chain,
12051205 SDValue Op1, SDValue Op2,
1206 SDValue Op3, unsigned Align,
1206 SDValue Op3, unsigned Align, bool isVolatile,
12071207 const Value *DstSV, uint64_t DstOff,
12081208 const Value *SrcSV, uint64_t SrcOff) {
12091209 return SDValue();
12191219 EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
12201220 SDValue Chain,
12211221 SDValue Op1, SDValue Op2,
1222 SDValue Op3, unsigned Align,
1222 SDValue Op3, unsigned Align, bool isVolatile,
12231223 const Value *DstSV, uint64_t DstOff) {
12241224 return SDValue();
12251225 }
4545
4646 /// EmitMemCpy - Emit a call to the memcpy function to the builder. This
4747 /// always expects that the size has type 'intptr_t' and Dst/Src are pointers.
48 Value *EmitMemCpy(Value *Dst, Value *Src, Value *Len,
49 unsigned Align, IRBuilder<> &B, const TargetData *TD);
48 Value *EmitMemCpy(Value *Dst, Value *Src, Value *Len, unsigned Align,
49 bool isVolatile, IRBuilder<> &B, const TargetData *TD);
5050
5151 /// EmitMemCpyChk - Emit a call to the __memcpy_chk function to the builder.
5252 /// This expects that the Len and ObjSize have type 'intptr_t' and Dst/Src
5656
5757 /// EmitMemMove - Emit a call to the memmove function to the builder. This
5858 /// always expects that the size has type 'intptr_t' and Dst/Src are pointers.
59 Value *EmitMemMove(Value *Dst, Value *Src, Value *Len,
60 unsigned Align, IRBuilder<> &B, const TargetData *TD);
59 Value *EmitMemMove(Value *Dst, Value *Src, Value *Len, unsigned Align,
60 bool isVolatile, IRBuilder<> &B, const TargetData *TD);
6161
6262 /// EmitMemChr - Emit a call to the memchr function. This assumes that Ptr is
6363 /// a pointer, Val is an i32 value, and Len is an 'intptr_t' value.
6969 const TargetData *TD);
7070
7171 /// EmitMemSet - Emit a call to the memset function
72 Value *EmitMemSet(Value *Dst, Value *Val, Value *Len, IRBuilder<> &B,
73 const TargetData *TD);
72 Value *EmitMemSet(Value *Dst, Value *Val, Value *Len, bool isVolatile,
73 IRBuilder<> &B, const TargetData *TD);
7474
7575 /// EmitUnaryFloatFnCall - Emit a call to the unary function named 'Name'
7676 /// (e.g. 'floor'). This function is known to take a single of type matching
32623262 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
32633263 SDValue Chain, SDValue Dst,
32643264 SDValue Src, uint64_t Size,
3265 unsigned Align, bool AlwaysInline,
3265 unsigned Align, bool isVol,
3266 bool AlwaysInline,
32663267 const Value *DstSV, uint64_t DstSVOff,
32673268 const Value *SrcSV, uint64_t SrcSVOff) {
32683269 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
33183319 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff);
33193320 Store = DAG.getStore(Chain, dl, Value,
33203321 getMemBasePlusOffset(Dst, DstOff, DAG),
3321 DstSV, DstSVOff + DstOff, false, false, Align);
3322 DstSV, DstSVOff + DstOff, isVol, false, Align);
33223323 } else {
33233324 // The type might not be legal for the target. This should only happen
33243325 // if the type is smaller than a legal type, as on PPC, so the right
33293330 assert(NVT.bitsGE(VT));
33303331 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
33313332 getMemBasePlusOffset(Src, SrcOff, DAG),
3332 SrcSV, SrcSVOff + SrcOff, VT, false, false,
3333 SrcSV, SrcSVOff + SrcOff, VT, isVol, false,
33333334 MinAlign(SrcAlign, SrcOff));
33343335 Store = DAG.getTruncStore(Chain, dl, Value,
33353336 getMemBasePlusOffset(Dst, DstOff, DAG),
3336 DstSV, DstSVOff + DstOff, VT, false, false,
3337 DstSV, DstSVOff + DstOff, VT, isVol, false,
33373338 Align);
33383339 }
33393340 OutChains.push_back(Store);
33483349 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
33493350 SDValue Chain, SDValue Dst,
33503351 SDValue Src, uint64_t Size,
3351 unsigned Align,bool AlwaysInline,
3352 unsigned Align, bool isVol,
3353 bool AlwaysInline,
33523354 const Value *DstSV, uint64_t DstSVOff,
33533355 const Value *SrcSV, uint64_t SrcSVOff) {
33543356 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
33963398
33973399 Value = DAG.getLoad(VT, dl, Chain,
33983400 getMemBasePlusOffset(Src, SrcOff, DAG),
3399 SrcSV, SrcSVOff + SrcOff, false, false, SrcAlign);
3401 SrcSV, SrcSVOff + SrcOff, isVol, false, SrcAlign);
34003402 LoadValues.push_back(Value);
34013403 LoadChains.push_back(Value.getValue(1));
34023404 SrcOff += VTSize;
34113413
34123414 Store = DAG.getStore(Chain, dl, LoadValues[i],
34133415 getMemBasePlusOffset(Dst, DstOff, DAG),
3414 DstSV, DstSVOff + DstOff, false, false, Align);
3416 DstSV, DstSVOff + DstOff, isVol, false, Align);
34153417 OutChains.push_back(Store);
34163418 DstOff += VTSize;
34173419 }
34233425 static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl,
34243426 SDValue Chain, SDValue Dst,
34253427 SDValue Src, uint64_t Size,
3426 unsigned Align,
3428 unsigned Align, bool isVol,
34273429 const Value *DstSV, uint64_t DstSVOff) {
34283430 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
34293431
34623464 SDValue Value = getMemsetValue(Src, VT, DAG, dl);
34633465 SDValue Store = DAG.getStore(Chain, dl, Value,
34643466 getMemBasePlusOffset(Dst, DstOff, DAG),
3465 DstSV, DstSVOff + DstOff, false, false, 0);
3467 DstSV, DstSVOff + DstOff, isVol, false, 0);
34663468 OutChains.push_back(Store);
34673469 DstOff += VTSize;
34683470 }
34733475
34743476 SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst,
34753477 SDValue Src, SDValue Size,
3476 unsigned Align, bool AlwaysInline,
3478 unsigned Align, bool isVol, bool AlwaysInline,
34773479 const Value *DstSV, uint64_t DstSVOff,
34783480 const Value *SrcSV, uint64_t SrcSVOff) {
34793481
34873489
34883490 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
34893491 ConstantSize->getZExtValue(),Align,
3490 false, DstSV, DstSVOff, SrcSV, SrcSVOff);
3492 isVol, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
34913493 if (Result.getNode())
34923494 return Result;
34933495 }
34963498 // code. If the target chooses to do this, this is the next best.
34973499 SDValue Result =
34983500 TLI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
3499 AlwaysInline,
3501 isVol, AlwaysInline,
35003502 DstSV, DstSVOff, SrcSV, SrcSVOff);
35013503 if (Result.getNode())
35023504 return Result;
35063508 if (AlwaysInline) {
35073509 assert(ConstantSize && "AlwaysInline requires a constant size!");
35083510 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
3509 ConstantSize->getZExtValue(), Align, true,
3510 DstSV, DstSVOff, SrcSV, SrcSVOff);
3511 ConstantSize->getZExtValue(), Align, isVol,
3512 true, DstSV, DstSVOff, SrcSV, SrcSVOff);
35113513 }
35123514
35133515 // Emit a library call.
3516 assert(!isVol && "library memcpy does not support volatile");
35143517 TargetLowering::ArgListTy Args;
35153518 TargetLowering::ArgListEntry Entry;
35163519 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext());
35313534
35323535 SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst,
35333536 SDValue Src, SDValue Size,
3534 unsigned Align,
3537 unsigned Align, bool isVol,
35353538 const Value *DstSV, uint64_t DstSVOff,
35363539 const Value *SrcSV, uint64_t SrcSVOff) {
35373540
35453548
35463549 SDValue Result =
35473550 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
3548 ConstantSize->getZExtValue(),
3549 Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
3551 ConstantSize->getZExtValue(), Align, isVol,
3552 false, DstSV, DstSVOff, SrcSV, SrcSVOff);
35503553 if (Result.getNode())
35513554 return Result;
35523555 }
35543557 // Then check to see if we should lower the memmove with target-specific
35553558 // code. If the target chooses to do this, this is the next best.
35563559 SDValue Result =
3557 TLI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align,
3560 TLI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol,
35583561 DstSV, DstSVOff, SrcSV, SrcSVOff);
35593562 if (Result.getNode())
35603563 return Result;
35613564
35623565 // Emit a library call.
3566 assert(!isVol && "library memmove does not support volatile");
35633567 TargetLowering::ArgListTy Args;
35643568 TargetLowering::ArgListEntry Entry;
35653569 Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext());
35803584
35813585 SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
35823586 SDValue Src, SDValue Size,
3583 unsigned Align,
3587 unsigned Align, bool isVol,
35843588 const Value *DstSV, uint64_t DstSVOff) {
35853589
35863590 // Check to see if we should lower the memset to stores first.
35933597
35943598 SDValue Result =
35953599 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
3596 Align, DstSV, DstSVOff);
3600 Align, isVol, DstSV, DstSVOff);
35973601 if (Result.getNode())
35983602 return Result;
35993603 }
36013605 // Then check to see if we should lower the memset with target-specific
36023606 // code. If the target chooses to do this, this is the next best.
36033607 SDValue Result =
3604 TLI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align,
3608 TLI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol,
36053609 DstSV, DstSVOff);
36063610 if (Result.getNode())
36073611 return Result;
36083612
36093613 // Emit a library call.
3614 assert(!isVol && "library memset does not support volatile");
36103615 const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext());
36113616 TargetLowering::ArgListTy Args;
36123617 TargetLowering::ArgListEntry Entry;
37303730 case Intrinsic::longjmp:
37313731 return "_longjmp"+!TLI.usesUnderscoreLongJmp();
37323732 case Intrinsic::memcpy: {
3733 // Assert for address < 256 since we support only user defined address
3734 // spaces.
3735 assert(cast(I.getOperand(1)->getType())->getAddressSpace()
3736 < 256 &&
3737 cast(I.getOperand(2)->getType())->getAddressSpace()
3738 < 256 &&
3739 "Unknown address space");
37333740 SDValue Op1 = getValue(I.getOperand(1));
37343741 SDValue Op2 = getValue(I.getOperand(2));
37353742 SDValue Op3 = getValue(I.getOperand(3));
37363743 unsigned Align = cast(I.getOperand(4))->getZExtValue();
3737 DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
3744 bool isVol = cast(I.getOperand(5))->getZExtValue();
3745 DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, isVol, false,
37383746 I.getOperand(1), 0, I.getOperand(2), 0));
37393747 return 0;
37403748 }
37413749 case Intrinsic::memset: {
3750 // Assert for address < 256 since we support only user defined address
3751 // spaces.
3752 assert(cast(I.getOperand(1)->getType())->getAddressSpace()
3753 < 256 &&
3754 "Unknown address space");
37423755 SDValue Op1 = getValue(I.getOperand(1));
37433756 SDValue Op2 = getValue(I.getOperand(2));
37443757 SDValue Op3 = getValue(I.getOperand(3));
37453758 unsigned Align = cast(I.getOperand(4))->getZExtValue();
3746 DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align,
3759 bool isVol = cast(I.getOperand(5))->getZExtValue();
3760 DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, isVol,
37473761 I.getOperand(1), 0));
37483762 return 0;
37493763 }
37503764 case Intrinsic::memmove: {
3765 // Assert for address < 256 since we support only user defined address
3766 // spaces.
3767 assert(cast(I.getOperand(1)->getType())->getAddressSpace()
3768 < 256 &&
3769 cast(I.getOperand(2)->getType())->getAddressSpace()
3770 < 256 &&
3771 "Unknown address space");
37513772 SDValue Op1 = getValue(I.getOperand(1));
37523773 SDValue Op2 = getValue(I.getOperand(2));
37533774 SDValue Op3 = getValue(I.getOperand(3));
37543775 unsigned Align = cast(I.getOperand(4))->getZExtValue();
3776 bool isVol = cast(I.getOperand(5))->getZExtValue();
37553777
37563778 // If the source and destination are known to not be aliases, we can
37573779 // lower memmove as memcpy.
37603782 Size = C->getZExtValue();
37613783 if (AA->alias(I.getOperand(1), Size, I.getOperand(2), Size) ==
37623784 AliasAnalysis::NoAlias) {
3763 DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
3764 I.getOperand(1), 0, I.getOperand(2), 0));
3785 DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, isVol,
3786 false, I.getOperand(1), 0, I.getOperand(2), 0));
37653787 return 0;
37663788 }
37673789
3768 DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align,
3790 DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, isVol,
37693791 I.getOperand(1), 0, I.getOperand(2), 0));
37703792 return 0;
37713793 }
860860 DebugLoc dl) {
861861 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
862862 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
863 /*AlwaysInline=*/false, NULL, 0, NULL, 0);
863 /*isVolatile=*/false, /*AlwaysInline=*/false,
864 NULL, 0, NULL, 0);
864865 }
865866
866867 /// LowerMemOpCallTo - Store the argument to the stack.
20522053 SDValue Chain,
20532054 SDValue Dst, SDValue Src,
20542055 SDValue Size, unsigned Align,
2055 bool AlwaysInline,
2056 bool isVolatile, bool AlwaysInline,
20562057 const Value *DstSV, uint64_t DstSVOff,
20572058 const Value *SrcSV, uint64_t SrcSVOff){
20582059 // Do repeated 4-byte loads and stores. To be improved.
20882089 Loads[i] = DAG.getLoad(VT, dl, Chain,
20892090 DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
20902091 DAG.getConstant(SrcOff, MVT::i32)),
2091 SrcSV, SrcSVOff + SrcOff, false, false, 0);
2092 SrcSV, SrcSVOff + SrcOff, isVolatile, false, 0);
20922093 TFOps[i] = Loads[i].getValue(1);
20932094 SrcOff += VTSize;
20942095 }
20992100 TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
21002101 DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
21012102 DAG.getConstant(DstOff, MVT::i32)),
2102 DstSV, DstSVOff + DstOff, false, false, 0);
2103 DstSV, DstSVOff + DstOff, isVolatile, false, 0);
21032104 DstOff += VTSize;
21042105 }
21052106 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
304304 SDValue Chain,
305305 SDValue Dst, SDValue Src,
306306 SDValue Size, unsigned Align,
307 bool AlwaysInline,
307 bool isVolatile, bool AlwaysInline,
308308 const Value *DstSV, uint64_t DstSVOff,
309309 const Value *SrcSV, uint64_t SrcSVOff);
310310 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
23912391 DebugLoc dl) {
23922392 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
23932393 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
2394 false, NULL, 0, NULL, 0);
2394 false, false, NULL, 0, NULL, 0);
23952395 }
23962396
23972397 /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
14321432 DebugLoc dl) {
14331433 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
14341434 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
1435 /*AlwaysInline=*/true, NULL, 0, NULL, 0);
1435 /*isVolatile*/false, /*AlwaysInline=*/true,
1436 NULL, 0, NULL, 0);
14361437 }
14371438
14381439 /// IsTailCallConvention - Return true if the calling convention is one that
65496550 SDValue Chain,
65506551 SDValue Dst, SDValue Src,
65516552 SDValue Size, unsigned Align,
6553 bool isVolatile,
65526554 const Value *DstSV,
65536555 uint64_t DstSVOff) {
65546556 ConstantSDNode *ConstantSize = dyn_cast(Size);
66776679 DAG.getConstant(Offset, AddrVT)),
66786680 Src,
66796681 DAG.getConstant(BytesLeft, SizeVT),
6680 Align, DstSV, DstSVOff + Offset);
6682 Align, isVolatile, DstSV, DstSVOff + Offset);
66816683 }
66826684
66836685 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
66886690 X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
66896691 SDValue Chain, SDValue Dst, SDValue Src,
66906692 SDValue Size, unsigned Align,
6691 bool AlwaysInline,
6693 bool isVolatile, bool AlwaysInline,
66926694 const Value *DstSV, uint64_t DstSVOff,
66936695 const Value *SrcSV, uint64_t SrcSVOff) {
66946696 // This requires the copy size to be a constant, preferrably
67476749 DAG.getNode(ISD::ADD, dl, SrcVT, Src,
67486750 DAG.getConstant(Offset, SrcVT)),
67496751 DAG.getConstant(BytesLeft, SizeVT),
6750 Align, AlwaysInline,
6752 Align, isVolatile, AlwaysInline,
67516753 DstSV, DstSVOff + Offset,
67526754 SrcSV, SrcSVOff + Offset));
67536755 }
68306832 DebugLoc dl = Op.getDebugLoc();
68316833
68326834 return DAG.getMemcpy(Chain, dl, DstPtr, SrcPtr,
6833 DAG.getIntPtrConstant(24), 8, false,
6834 DstSV, 0, SrcSV, 0);
6835 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false,
6836 false, DstSV, 0, SrcSV, 0);
68356837 }
68366838
68376839 SDValue
736736 SDValue Chain,
737737 SDValue Dst, SDValue Src,
738738 SDValue Size, unsigned Align,
739 bool isVolatile,
739740 const Value *DstSV, uint64_t DstSVOff);
740741 SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
741742 SDValue Chain,
742743 SDValue Dst, SDValue Src,
743744 SDValue Size, unsigned Align,
744 bool AlwaysInline,
745 bool isVolatile, bool AlwaysInline,
745746 const Value *DstSV, uint64_t DstSVOff,
746747 const Value *SrcSV, uint64_t SrcSVOff);
747748
751752 /// block, the number of args, and whether or not the second arg is
752753 /// in memory or not.
753754 MachineBasicBlock *EmitPCMP(MachineInstr *BInstr, MachineBasicBlock *BB,
754 unsigned argNum, bool inMem) const;
755 unsigned argNum, bool inMem) const;
755756
756757 /// Utility function to emit atomic bitwise operations (and, or, xor).
757758 /// It takes the bitwise instruction to expand, the associated machine basic
14421442 return DAG.getMemmove(Chain, dl, ST->getBasePtr(),
14431443 LD->getBasePtr(),
14441444 DAG.getConstant(StoreBits/8, MVT::i32),
1445 Alignment, ST->getSrcValue(),
1445 Alignment, false, ST->getSrcValue(),
14461446 ST->getSrcValueOffset(), LD->getSrcValue(),
14471447 LD->getSrcValueOffset());
14481448 }
135135 return 0; // If not 1/2/4/8 bytes, exit.
136136
137137 // Use an integer load+store unless we can find something better.
138 Type *NewPtrTy =
139 PointerType::getUnqual(IntegerType::get(MI->getContext(), Size<<3));
138 unsigned SrcAddrSp =
139 cast(MI->getOperand(2)->getType())->getAddressSpace();
140 unsigned DstAddrSp =
141 cast(MI->getOperand(1)->getType())->getAddressSpace();
142
143 const IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
144 Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp);
145 Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp);
140146
141147 // Memcpy forces the use of i8* for the source and destination. That means
142148 // that if you're using memcpy to move one double around, you'll get a cast
166172 break;
167173 }
168174
169 if (SrcETy->isSingleValueType())
170 NewPtrTy = PointerType::getUnqual(SrcETy);
175 if (SrcETy->isSingleValueType()) {
176 NewSrcPtrTy = PointerType::get(SrcETy, SrcAddrSp);
177 NewDstPtrTy = PointerType::get(SrcETy, DstAddrSp);
178 }
171179 }
172180 }
173181
177185 SrcAlign = std::max(SrcAlign, CopyAlign);
178186 DstAlign = std::max(DstAlign, CopyAlign);
179187
180 Value *Src = Builder->CreateBitCast(MI->getOperand(2), NewPtrTy);
181 Value *Dest = Builder->CreateBitCast(MI->getOperand(1), NewPtrTy);
182 Instruction *L = new LoadInst(Src, "tmp", false, SrcAlign);
188 Value *Src = Builder->CreateBitCast(MI->getOperand(2), NewSrcPtrTy);
189 Value *Dest = Builder->CreateBitCast(MI->getOperand(1), NewDstPtrTy);
190 Instruction *L = new LoadInst(Src, "tmp", MI->isVolatile(), SrcAlign);
183191 InsertNewInstBefore(L, *MI);
184 InsertNewInstBefore(new StoreInst(L, Dest, false, DstAlign), *MI);
192 InsertNewInstBefore(new StoreInst(L, Dest, MI->isVolatile(), DstAlign),
193 *MI);
185194
186195 // Set the size of the copy to 0, it will be deleted on the next iteration.
187196 MI->setOperand(3, Constant::getNullValue(MemOpLength->getType()));
274283 if (GVSrc->isConstant()) {
275284 Module *M = CI.getParent()->getParent()->getParent();
276285 Intrinsic::ID MemCpyID = Intrinsic::memcpy;
277 const Type *Tys[1];
278 Tys[0] = CI.getOperand(3)->getType();
286 const Type *Tys[3] = { CI.getOperand(1)->getType(),
287 CI.getOperand(2)->getType(),
288 CI.getOperand(3)->getType() };
279289 CI.setOperand(0,
280 Intrinsic::getDeclaration(M, MemCpyID, Tys, 1));
290 Intrinsic::getDeclaration(M, MemCpyID, Tys, 3));
281291 Changed = true;
282292 }
283293 }
412412 // interesting as a small compile-time optimization.
413413 Ranges.addStore(0, SI);
414414
415 Function *MemSetF = 0;
416415
417416 // Now that we have full information about ranges, loop over the ranges and
418417 // emit memset's for anything big enough to be worthwhile.
432431 // memset block. This ensure that the memset is dominated by any addressing
433432 // instruction needed by the start of the block.
434433 BasicBlock::iterator InsertPt = BI;
435
436 if (MemSetF == 0) {
437 const Type *Ty = Type::getInt64Ty(Context);
438 MemSetF = Intrinsic::getDeclaration(M, Intrinsic::memset, &Ty, 1);
439 }
440
434
441435 // Get the starting pointer of the block.
442436 StartPtr = Range.StartPtr;
443
437
438 // Determine alignment
439 unsigned Alignment = Range.Alignment;
440 if (Alignment == 0) {
441 const Type *EltType =
442 cast(StartPtr->getType())->getElementType();
443 Alignment = TD->getABITypeAlignment(EltType);
444 }
445
444446 // Cast the start ptr to be i8* as memset requires.
445 const Type *i8Ptr = Type::getInt8PtrTy(Context);
446 if (StartPtr->getType() != i8Ptr)
447 const PointerType* StartPTy = cast(StartPtr->getType());
448 const PointerType *i8Ptr = Type::getInt8PtrTy(Context,
449 StartPTy->getAddressSpace());
450 if (StartPTy!= i8Ptr)
447451 StartPtr = new BitCastInst(StartPtr, i8Ptr, StartPtr->getName(),
448452 InsertPt);
449
453
450454 Value *Ops[] = {
451455 StartPtr, ByteVal, // Start, value
452456 // size
453457 ConstantInt::get(Type::getInt64Ty(Context), Range.End-Range.Start),
454458 // align
455 ConstantInt::get(Type::getInt32Ty(Context), Range.Alignment)
459 ConstantInt::get(Type::getInt32Ty(Context), Alignment),
460 // volatile
461 ConstantInt::get(Type::getInt1Ty(Context), 0),
456462 };
457 Value *C = CallInst::Create(MemSetF, Ops, Ops+4, "", InsertPt);
463 const Type *Tys[] = { Ops[0]->getType(), Ops[2]->getType() };
464
465 Function *MemSetF = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys, 2);
466
467 Value *C = CallInst::Create(MemSetF, Ops, Ops+5, "", InsertPt);
458468 DEBUG(dbgs() << "Replace stores:\n";
459469 for (unsigned i = 0, e = Range.TheStores.size(); i != e; ++i)
460470 dbgs() << *Range.TheStores[i];
679689 return false;
680690
681691 // If all checks passed, then we can transform these memcpy's
682 const Type *Ty = M->getLength()->getType();
692 const Type *ArgTys[3] = { M->getRawDest()->getType(),
693 MDep->getRawSource()->getType(),
694 M->getLength()->getType() };
683695 Function *MemCpyFun = Intrinsic::getDeclaration(
684696 M->getParent()->getParent()->getParent(),
685 M->getIntrinsicID(), &Ty, 1);
697 M->getIntrinsicID(), ArgTys, 3);
686698
687 Value *Args[4] = {
688 M->getRawDest(), MDep->getRawSource(), M->getLength(), M->getAlignmentCst()
699 Value *Args[5] = {
700 M->getRawDest(), MDep->getRawSource(), M->getLength(),
701 M->getAlignmentCst(), M->getVolatileCst()
689702 };
690703
691 CallInst *C = CallInst::Create(MemCpyFun, Args, Args+4, "", M);
704 CallInst *C = CallInst::Create(MemCpyFun, Args, Args+5, "", M);
692705
693706
694707 // If C and M don't interfere, then this is a valid transformation. If they
727740
728741 // If not, then we know we can transform this.
729742 Module *Mod = M->getParent()->getParent()->getParent();
730 const Type *Ty = M->getLength()->getType();
731 M->setOperand(0, Intrinsic::getDeclaration(Mod, Intrinsic::memcpy, &Ty, 1));
743 const Type *ArgTys[3] = { M->getRawDest()->getType(),
744 M->getRawSource()->getType(),
745 M->getLength()->getType() };
746 M->setOperand(0,Intrinsic::getDeclaration(Mod, Intrinsic::memcpy, ArgTys, 3));
732747
733748 // MemDep may have over conservative information about this instruction, just
734749 // conservatively flush it from the cache.
857857 EltPtr = new BitCastInst(EltPtr, BytePtrTy, EltPtr->getName(), MI);
858858
859859 // Cast the other pointer (if we have one) to BytePtrTy.
860 if (OtherElt && OtherElt->getType() != BytePtrTy)
861 OtherElt = new BitCastInst(OtherElt, BytePtrTy, OtherElt->getName(), MI);
860 if (OtherElt && OtherElt->getType() != BytePtrTy) {
861 // Preserve address space of OtherElt
862 const PointerType* OtherPTy = cast(OtherElt->getType());
863 const PointerType* PTy = cast(BytePtrTy);
864 if (OtherPTy->getElementType() != PTy->getElementType()) {
865 Type *NewOtherPTy = PointerType::get(PTy->getElementType(),
866 OtherPTy->getAddressSpace());
867 OtherElt = new BitCastInst(OtherElt, NewOtherPTy,
868 OtherElt->getNameStr(), MI);
869 }
870 }
862871
863872 unsigned EltSize = TD->getTypeAllocSize(EltTy);
864873
869878 SROADest ? OtherElt : EltPtr, // Src ptr
870879 ConstantInt::get(MI->getOperand(3)->getType(), EltSize), // Size
871880 // Align
872 ConstantInt::get(Type::getInt32Ty(MI->getContext()), OtherEltAlign)
881 ConstantInt::get(Type::getInt32Ty(MI->getContext()), OtherEltAlign),
882 MI->getVolatileCst()
873883 };
874 CallInst::Create(TheFn, Ops, Ops + 4, "", MI);
884 // In case we fold the address space overloaded memcpy of A to B
885 // with memcpy of B to C, change the function to be a memcpy of A to C.
886 const Type *Tys[] = { Ops[0]->getType(), Ops[1]->getType(),
887 Ops[2]->getType() };
888 Module *M = MI->getParent()->getParent()->getParent();
889 TheFn = Intrinsic::getDeclaration(M, MI->getIntrinsicID(), Tys, 3);
890 CallInst::Create(TheFn, Ops, Ops + 5, "", MI);
875891 } else {
876892 assert(isa(MI));
877893 Value *Ops[] = {
878894 EltPtr, MI->getOperand(2), // Dest, Value,
879895 ConstantInt::get(MI->getOperand(3)->getType(), EltSize), // Size
880 Zero // Align
896 Zero, // Align
897 ConstantInt::get(Type::getInt1Ty(MI->getContext()), 0) // isVolatile
881898 };
882 CallInst::Create(TheFn, Ops, Ops + 4, "", MI);
899 const Type *Tys[] = { Ops[0]->getType(), Ops[2]->getType() };
900 Module *M = MI->getParent()->getParent()->getParent();
901 TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys, 2);
902 CallInst::Create(TheFn, Ops, Ops + 5, "", MI);
883903 }
884904 }
885905 DeadInsts.push_back(MI);
141141 // We have enough information to now generate the memcpy call to do the
142142 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
143143 EmitMemCpy(CpyDst, Src,
144 ConstantInt::get(TD->getIntPtrType(*Context), Len+1), 1, B, TD);
144 ConstantInt::get(TD->getIntPtrType(*Context), Len+1),
145 1, false, B, TD);
145146 }
146147 };
147148
382383 CI->getOperand(3), B, TD);
383384 else
384385 EmitMemCpy(Dst, Src,
385 ConstantInt::get(TD->getIntPtrType(*Context), Len), 1, B, TD);
386 ConstantInt::get(TD->getIntPtrType(*Context), Len),
387 1, false, B, TD);
386388 return Dst;
387389 }
388390 };
410412
411413 if (SrcLen == 0) {
412414 // strncpy(x, "", y) -> memset(x, '\0', y, 1)
413 EmitMemSet(Dst, ConstantInt::get(Type::getInt8Ty(*Context), '\0'), LenOp,
414 B, TD);
415 EmitMemSet(Dst, ConstantInt::get(Type::getInt8Ty(*Context), '\0'),
416 LenOp, false, B, TD);
415417 return Dst;
416418 }
417419
431433
432434 // strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant]
433435 EmitMemCpy(Dst, Src,
434 ConstantInt::get(TD->getIntPtrType(*Context), Len), 1, B, TD);
436 ConstantInt::get(TD->getIntPtrType(*Context), Len),
437 1, false, B, TD);
435438
436439 return Dst;
437440 }
592595
593596 // memcpy(x, y, n) -> llvm.memcpy(x, y, n, 1)
594597 EmitMemCpy(CI->getOperand(1), CI->getOperand(2),
595 CI->getOperand(3), 1, B, TD);
598 CI->getOperand(3), 1, false, B, TD);
596599 return CI->getOperand(1);
597600 }
598601 };
614617
615618 // memmove(x, y, n) -> llvm.memmove(x, y, n, 1)
616619 EmitMemMove(CI->getOperand(1), CI->getOperand(2),
617 CI->getOperand(3), 1, B, TD);
620 CI->getOperand(3), 1, false, B, TD);
618621 return CI->getOperand(1);
619622 }
620623 };
636639
637640 // memset(p, v, n) -> llvm.memset(p, v, n, 1)
638641 Value *Val = B.CreateIntCast(CI->getOperand(2), Type::getInt8Ty(*Context),
639 false);
640 EmitMemSet(CI->getOperand(1), Val, CI->getOperand(3), B, TD);
642 false);
643 EmitMemSet(CI->getOperand(1), Val, CI->getOperand(3), false, B, TD);
641644 return CI->getOperand(1);
642645 }
643646 };
9981001 // sprintf(str, fmt) -> llvm.memcpy(str, fmt, strlen(fmt)+1, 1)
9991002 EmitMemCpy(CI->getOperand(1), CI->getOperand(2), // Copy the nul byte.
10001003 ConstantInt::get(TD->getIntPtrType(*Context),
1001 FormatStr.size()+1), 1, B, TD);
1004 FormatStr.size()+1), 1, false, B, TD);
10021005 return ConstantInt::get(CI->getType(), FormatStr.size());
10031006 }
10041007
10121015 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
10131016 if (!CI->getOperand(3)->getType()->isIntegerTy()) return 0;
10141017 Value *V = B.CreateTrunc(CI->getOperand(3),
1015 Type::getInt8Ty(*Context), "char");
1018 Type::getInt8Ty(*Context), "char");
10161019 Value *Ptr = CastToCStr(CI->getOperand(1), B);
10171020 B.CreateStore(V, Ptr);
10181021 Ptr = B.CreateGEP(Ptr, ConstantInt::get(Type::getInt32Ty(*Context), 1),
1019 "nul");
1022 "nul");
10201023 B.CreateStore(Constant::getNullValue(Type::getInt8Ty(*Context)), Ptr);
10211024
10221025 return ConstantInt::get(CI->getType(), 1);
10331036 Value *IncLen = B.CreateAdd(Len,
10341037 ConstantInt::get(Len->getType(), 1),
10351038 "leninc");
1036 EmitMemCpy(CI->getOperand(1), CI->getOperand(3), IncLen, 1, B, TD);
1039 EmitMemCpy(CI->getOperand(1), CI->getOperand(3), IncLen, 1, false, B, TD);
10371040
10381041 // The sprintf result is the unincremented number of bytes in the string.
10391042 return B.CreateIntCast(Len, CI->getType(), false);
108108
109109 /// EmitMemCpy - Emit a call to the memcpy function to the builder. This always
110110 /// expects that Len has type 'intptr_t' and Dst/Src are pointers.
111 Value *llvm::EmitMemCpy(Value *Dst, Value *Src, Value *Len,
112 unsigned Align, IRBuilder<> &B, const TargetData *TD) {
113 Module *M = B.GetInsertBlock()->getParent()->getParent();
114 const Type *Ty = Len->getType();
115 Value *MemCpy = Intrinsic::getDeclaration(M, Intrinsic::memcpy, &Ty, 1);
111 Value *llvm::EmitMemCpy(Value *Dst, Value *Src, Value *Len, unsigned Align,
112 bool isVolatile, IRBuilder<> &B, const TargetData *TD) {
113 Module *M = B.GetInsertBlock()->getParent()->getParent();
114 const Type *ArgTys[3] = { Dst->getType(), Src->getType(), Len->getType() };
115 Value *MemCpy = Intrinsic::getDeclaration(M, Intrinsic::memcpy, ArgTys, 3);
116116 Dst = CastToCStr(Dst, B);
117117 Src = CastToCStr(Src, B);
118 return B.CreateCall4(MemCpy, Dst, Src, Len,
119 ConstantInt::get(B.getInt32Ty(), Align));
118 return B.CreateCall5(MemCpy, Dst, Src, Len,
119 ConstantInt::get(B.getInt32Ty(), Align),
120 ConstantInt::get(B.getInt1Ty(), isVolatile));
120121 }
121122
122123 /// EmitMemCpyChk - Emit a call to the __memcpy_chk function to the builder.
145146
146147 /// EmitMemMove - Emit a call to the memmove function to the builder. This
147148 /// always expects that the size has type 'intptr_t' and Dst/Src are pointers.
148 Value *llvm::EmitMemMove(Value *Dst, Value *Src, Value *Len,
149 unsigned Align, IRBuilder<> &B, const TargetData *TD) {
150 Module *M = B.GetInsertBlock()->getParent()->getParent();
151 LLVMContext &Context = B.GetInsertBlock()->getContext();
152 const Type *Ty = TD->getIntPtrType(Context);
153 Value *MemMove = Intrinsic::getDeclaration(M, Intrinsic::memmove, &Ty, 1);
149 Value *llvm::EmitMemMove(Value *Dst, Value *Src, Value *Len, unsigned Align,
150 bool isVolatile, IRBuilder<> &B, const TargetData *TD) {
151 Module *M = B.GetInsertBlock()->getParent()->getParent();
152 LLVMContext &Context = B.GetInsertBlock()->getContext();
153 const Type *ArgTys[3] = { Dst->getType(), Src->getType(),
154 TD->getIntPtrType(Context) };
155 Value *MemMove = Intrinsic::getDeclaration(M, Intrinsic::memmove, ArgTys, 3);
154156 Dst = CastToCStr(Dst, B);
155157 Src = CastToCStr(Src, B);
156158 Value *A = ConstantInt::get(B.getInt32Ty(), Align);
157 return B.CreateCall4(MemMove, Dst, Src, Len, A);
159 Value *Vol = ConstantInt::get(B.getInt1Ty(), isVolatile);
160 return B.CreateCall5(MemMove, Dst, Src, Len, A, Vol);
158161 }
159162
160163 /// EmitMemChr - Emit a call to the memchr function. This assumes that Ptr is
205208 }
206209
207210 /// EmitMemSet - Emit a call to the memset function
208 Value *llvm::EmitMemSet(Value *Dst, Value *Val,
209 Value *Len, IRBuilder<> &B, const TargetData *TD) {
211 Value *llvm::EmitMemSet(Value *Dst, Value *Val, Value *Len, bool isVolatile,
212 IRBuilder<> &B, const TargetData *TD) {
210213 Module *M = B.GetInsertBlock()->getParent()->getParent();
211214 Intrinsic::ID IID = Intrinsic::memset;
212 const Type *Tys[1];
213 Tys[0] = Len->getType();
214 Value *MemSet = Intrinsic::getDeclaration(M, IID, Tys, 1);
215 const Type *Tys[2] = { Dst->getType(), Len->getType() };
216 Value *MemSet = Intrinsic::getDeclaration(M, IID, Tys, 2);
215217 Value *Align = ConstantInt::get(B.getInt32Ty(), 1);
216 return B.CreateCall4(MemSet, CastToCStr(Dst, B), Val, Len, Align);
218 Value *Vol = ConstantInt::get(B.getInt1Ty(), isVolatile);
219 return B.CreateCall5(MemSet, CastToCStr(Dst, B), Val, Len, Align, Vol);
217220 }
218221
219222 /// EmitUnaryFloatFnCall - Emit a call to the unary function named 'Name' (e.g.
380383 if (Name == "__memcpy_chk") {
381384 if (isFoldable(4, 3, false)) {
382385 EmitMemCpy(CI->getOperand(1), CI->getOperand(2), CI->getOperand(3),
383 1, B, TD);
386 1, false, B, TD);
384387 replaceCall(CI->getOperand(1));
385388 return true;
386389 }
395398 if (Name == "__memmove_chk") {
396399 if (isFoldable(4, 3, false)) {
397400 EmitMemMove(CI->getOperand(1), CI->getOperand(2), CI->getOperand(3),
398 1, B, TD);
401 1, false, B, TD);
399402 replaceCall(CI->getOperand(1));
400403 return true;
401404 }
406409 if (isFoldable(4, 3, false)) {
407410 Value *Val = B.CreateIntCast(CI->getOperand(2), B.getInt8Ty(),
408411 false);
409 EmitMemSet(CI->getOperand(1), Val, CI->getOperand(3), B, TD);
412 EmitMemSet(CI->getOperand(1), Val, CI->getOperand(3), false, B, TD);
410413 replaceCall(CI->getOperand(1));
411414 return true;
412415 }
296296 I->getName(),
297297 &*Caller->begin()->begin());
298298 // Emit a memcpy.
299 const Type *Tys[] = { Type::getInt64Ty(Context) };
299 const Type *Tys[3] = {VoidPtrTy, VoidPtrTy, Type::getInt64Ty(Context)};
300300 Function *MemCpyFn = Intrinsic::getDeclaration(Caller->getParent(),
301301 Intrinsic::memcpy,
302 Tys, 1);
302 Tys, 3);
303303 Value *DestCast = new BitCastInst(NewAlloca, VoidPtrTy, "tmp", TheCall);
304304 Value *SrcCast = new BitCastInst(*AI, VoidPtrTy, "tmp", TheCall);
305305
308308 Size = ConstantExpr::getSizeOf(AggTy);
309309 else
310310 Size = ConstantInt::get(Type::getInt64Ty(Context),
311 TD->getTypeStoreSize(AggTy));
311 TD->getTypeStoreSize(AggTy));
312312
313313 // Always generate a memcpy of alignment 1 here because we don't know
314314 // the alignment of the src pointer. Other optimizations can infer
315315 // better alignment.
316316 Value *CallArgs[] = {
317317 DestCast, SrcCast, Size,
318 ConstantInt::get(Type::getInt32Ty(Context), 1)
318 ConstantInt::get(Type::getInt32Ty(Context), 1),
319 ConstantInt::get(Type::getInt1Ty(Context), 0)
319320 };
320321 CallInst *TheMemCpy =
321 CallInst::Create(MemCpyFn, CallArgs, CallArgs+4, "", TheCall);
322 CallInst::Create(MemCpyFn, CallArgs, CallArgs+5, "", TheCall);
322323
323324 // If we have a call graph, update it.
324325 if (CG) {
144144 }
145145 break;
146146
147 case 'm': {
148 // This upgrades the llvm.memcpy, llvm.memmove, and llvm.memset to the
149 // new format that allows overloading the pointer for different address
150 // space (e.g., llvm.memcpy.i16 => llvm.memcpy.p0i8.p0i8.i16)
151 const char* NewFnName = NULL;
152 if (Name.compare(5,8,"memcpy.i",8) == 0) {
153 if (Name[13] == '8')
154 NewFnName = "llvm.memcpy.p0i8.p0i8.i8";
155 else if (Name.compare(13,2,"16") == 0)
156 NewFnName = "llvm.memcpy.p0i8.p0i8.i16";
157 else if (Name.compare(13,2,"32") == 0)
158 NewFnName = "llvm.memcpy.p0i8.p0i8.i32";
159 else if (Name.compare(13,2,"64") == 0)
160 NewFnName = "llvm.memcpy.p0i8.p0i8.i64";
161 } else if (Name.compare(5,9,"memmove.i",9) == 0) {
162 if (Name[14] == '8')
163 NewFnName = "llvm.memmove.p0i8.p0i8.i8";
164 else if (Name.compare(14,2,"16") == 0)
165 NewFnName = "llvm.memmove.p0i8.p0i8.i16";
166 else if (Name.compare(14,2,"32") == 0)
167 NewFnName = "llvm.memmove.p0i8.p0i8.i32";
168 else if (Name.compare(14,2,"64") == 0)
169 NewFnName = "llvm.memmove.p0i8.p0i8.i64";
170 }
171 else if (Name.compare(5,8,"memset.i",8) == 0) {
172 if (Name[13] == '8')
173 NewFnName = "llvm.memset.p0i8.i8";
174 else if (Name.compare(13,2,"16") == 0)
175 NewFnName = "llvm.memset.p0i8.i16";
176 else if (Name.compare(13,2,"32") == 0)
177 NewFnName = "llvm.memset.p0i8.i32";
178 else if (Name.compare(13,2,"64") == 0)
179 NewFnName = "llvm.memset.p0i8.i64";
180 }
181 if (NewFnName) {
182 const FunctionType *FTy = F->getFunctionType();
183 NewFn = cast(M->getOrInsertFunction(NewFnName,
184 FTy->getReturnType(),
185 FTy->getParamType(0),
186 FTy->getParamType(1),
187 FTy->getParamType(2),
188 FTy->getParamType(3),
189 Type::getInt1Ty(F->getContext()),
190 (Type *)0));
191 return true;
192 }
193 break;
194 }
147195 case 'p':
148196 // This upgrades the llvm.part.select overloaded intrinsic names to only
149197 // use one type specifier in the name. We only care about the old format
471519 CI->eraseFromParent();
472520 }
473521 break;
522 case Intrinsic::memcpy:
523 case Intrinsic::memmove:
524 case Intrinsic::memset: {
525 // Add isVolatile
526 const llvm::Type *I1Ty = llvm::Type::getInt1Ty(CI->getContext());
527 Value *Operands[5] = { CI->getOperand(1), CI->getOperand(2),
528 CI->getOperand(3), CI->getOperand(4),
529 llvm::ConstantInt::get(I1Ty, 0) };
530 CallInst *NewCI = CallInst::Create(NewFn, Operands, Operands+5,
531 CI->getName(), CI);
532 NewCI->setTailCall(CI->isTailCall());
533 NewCI->setCallingConv(CI->getCallingConv());
534 // Handle any uses of the old CallInst.
535 if (!CI->use_empty())
536 // Replace all uses of the old call with the new cast which has the
537 // correct type.
538 CI->replaceAllUsesWith(NewCI);
539
540 // Clean up the old call now that it has been completely upgraded.
541 CI->eraseFromParent();
542 break;
543 }
474544 }
475545 }
476546
102102 ret i32 %sub
103103 ; CHECK: @test4
104104 ; CHECK: load i32* @G
105 ; CHECK: memset.i32
105 ; CHECK: memset.p0i8.i32
106106 ; CHECK-NOT: load
107107 ; CHECK: sub i32 %tmp, %tmp
108108 }
117117 ret i32 %sub
118118 ; CHECK: @test5
119119 ; CHECK: load i32* @G
120 ; CHECK: memcpy.i32
120 ; CHECK: memcpy.p0i8.p0i8.i32
121121 ; CHECK-NOT: load
122122 ; CHECK: sub i32 %tmp, %tmp
123123 }
77 tail call void @llvm.memcpy.i64( i8* %tmp.1, i8* %tmp.3, i64 100000, i32 1 )
88 tail call void @llvm.memset.i32( i8* %tmp.3, i8 14, i32 10000, i32 0 )
99 tail call void @llvm.memmove.i32( i8* %tmp.1, i8* %tmp.3, i32 123124, i32 1 )
10 tail call void @llvm.memmove.i64( i8* %tmp.1, i8* %tmp.3, i64 123124, i32 1 )
1011 ret void
1112 }
1213
1819
1920 declare void @llvm.memmove.i32(i8*, i8*, i32, i32)
2021
22 declare void @llvm.memmove.i64(i8*, i8*, i32, i32)
66
77 define i32 @t() nounwind ssp {
88 ; CHECK: @t
9 ; CHECK: @llvm.memset.i64
9 ; CHECK: @llvm.memset.p0i8.i64
1010 entry:
1111 %0 = alloca %struct.data, align 8 ; <%struct.data*> [#uses=1]
1212 %1 = bitcast %struct.data* %0 to i8* ; [#uses=1]
112112 %1 = bitcast %struct.data* %0 to i8*
113113 %2 = call i64 @llvm.objectsize.i64(i8* %1, i1 false) nounwind
114114 ; CHECK-NOT: @llvm.objectsize
115 ; CHECK: @llvm.memset.i64(i8* %1, i8 0, i64 1824, i32 8)
115 ; CHECK: @llvm.memset.p0i8.i64(i8* %1, i8 0, i64 1824, i32 8, i1 false)
116116 %3 = call i8* @__memset_chk(i8* %1, i32 0, i64 1824, i64 %2) nounwind
117117 ret i32 0
118118 }
127127 %1 = tail call i32 @llvm.objectsize.i32(i8* %0, i1 false)
128128 %2 = load i8** @s, align 8
129129 ; CHECK-NOT: @llvm.objectsize
130 ; CHECK: @llvm.memcpy.i32(i8* %0, i8* %1, i32 10, i32 1)
130 ; CHECK: @llvm.memcpy.p0i8.p0i8.i32(i8* %0, i8* %1, i32 10, i32 1, i1 false)
131131 %3 = tail call i8* @__memcpy_chk(i8* %0, i8* %2, i32 10, i32 %1) nounwind
132132 ret void
133133 }
33 ; The resulting memset is only 4-byte aligned, despite containing
44 ; a 16-byte alignmed store in the middle.
55
6 ; CHECK: call void @llvm.memset.i64(i8* %a01, i8 0, i64 16, i32 4)
6 ; CHECK: call void @llvm.memset.p0i8.i64(i8* %a01, i8 0, i64 16, i32 4, i1 false)
77
88 define void @foo(i32* %p) {
99 %a0 = getelementptr i32* %p, i64 0
2020 %arg1 = getelementptr [1024 x i8]* %target, i32 0, i32 0
2121 %arg2 = getelementptr [6 x i8]* @hello, i32 0, i32 0
2222 %rslt1 = call i8* @strcpy( i8* %arg1, i8* %arg2 )
23 ; CHECK: @llvm.memcpy.i32
23 ; CHECK: @llvm.memcpy.p0i8.p0i8.i32
2424 ret i32 0
2525 }
2626
0 ; RUN: not llvm-as < %s |& grep {llvm intrinsics cannot be defined}
11 ; PR1047
22
3 define void @llvm.memcpy.i32(i8*, i8*, i32, i32) {
3 define void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i32, i1) {
44 entry:
55 ret void
66 }