llvm.org GIT mirror llvm / 1d8008b
Re-apply r291205, "LowerTypeTests: Split the pass in two: a resolution phase and a lowering phase.", with a fix for an off-by-one error. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@291699 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 2 years ago
9 changed file(s) with 166 addition(s) and 128 deletion(s). Raw diff Collapse all Expand all
5858 }
5959
6060 bool containsGlobalOffset(uint64_t Offset) const;
61
62 bool containsValue(const DataLayout &DL,
63 const DenseMap &GlobalLayout,
64 Value *V, uint64_t COffset = 0) const;
6561
6662 void print(raw_ostream &OS) const;
6763 };
9090 return Bits.count(BitOffset);
9191 }
9292
93 bool BitSetInfo::containsValue(
94 const DataLayout &DL,
95 const DenseMap &GlobalLayout, Value *V,
96 uint64_t COffset) const {
97 if (auto GV = dyn_cast(V)) {
98 auto I = GlobalLayout.find(GV);
99 if (I == GlobalLayout.end())
100 return false;
101 return containsGlobalOffset(I->second + COffset);
102 }
103
104 if (auto GEP = dyn_cast(V)) {
105 APInt APOffset(DL.getPointerSizeInBits(0), 0);
106 bool Result = GEP->accumulateConstantOffset(DL, APOffset);
107 if (!Result)
108 return false;
109 COffset += APOffset.getZExtValue();
110 return containsValue(DL, GlobalLayout, GEP->getPointerOperand(), COffset);
111 }
112
113 if (auto Op = dyn_cast(V)) {
114 if (Op->getOpcode() == Instruction::BitCast)
115 return containsValue(DL, GlobalLayout, Op->getOperand(0), COffset);
116
117 if (Op->getOpcode() == Instruction::Select)
118 return containsValue(DL, GlobalLayout, Op->getOperand(1), COffset) &&
119 containsValue(DL, GlobalLayout, Op->getOperand(2), COffset);
120 }
121
122 return false;
123 }
124
12593 void BitSetInfo::print(raw_ostream &OS) const {
12694 OS << "offset " << ByteOffset << " size " << BitSize << " align "
12795 << (1 << AlignLog2);
228196 std::set Bits;
229197 uint64_t BitSize;
230198 GlobalVariable *ByteArray;
231 Constant *Mask;
199 GlobalVariable *MaskGlobal;
232200 };
233201
234202 /// A POD-like structure that we use to store a global reference together with
275243
276244 IntegerType *Int1Ty = Type::getInt1Ty(M.getContext());
277245 IntegerType *Int8Ty = Type::getInt8Ty(M.getContext());
246 PointerType *Int8PtrTy = Type::getInt8PtrTy(M.getContext());
278247 IntegerType *Int32Ty = Type::getInt32Ty(M.getContext());
279248 PointerType *Int32PtrTy = PointerType::getUnqual(Int32Ty);
280249 IntegerType *Int64Ty = Type::getInt64Ty(M.getContext());
285254
286255 // Mapping from type identifiers to the call sites that test them.
287256 DenseMap> TypeTestCallSites;
257
258 /// This structure describes how to lower type tests for a particular type
259 /// identifier. It is either built directly from the global analysis (during
260 /// regular LTO or the regular LTO phase of ThinLTO), or indirectly using type
261 /// identifier summaries and external symbol references (in ThinLTO backends).
262 struct TypeIdLowering {
263 TypeTestResolution::Kind TheKind;
264
265 /// All except Unsat: the start address within the combined global.
266 Constant *OffsetedGlobal;
267
268 /// ByteArray, Inline, AllOnes: log2 of the required global alignment
269 /// relative to the start address.
270 Constant *AlignLog2;
271
272 /// ByteArray, Inline, AllOnes: size of the memory region covering members
273 /// of this type identifier as a multiple of 2^AlignLog2.
274 Constant *Size;
275
276 /// ByteArray, Inline, AllOnes: range of the size expressed as a bit width.
277 unsigned SizeBitWidth;
278
279 /// ByteArray: the byte array to test the address against.
280 Constant *TheByteArray;
281
282 /// ByteArray: the bit mask to apply to bytes loaded from the byte array.
283 Constant *BitMask;
284
285 /// Inline: the bit mask to test the address against.
286 Constant *InlineBits;
287 };
288288
289289 std::vector ByteArrayInfos;
290290
295295 const DenseMap &GlobalLayout);
296296 ByteArrayInfo *createByteArray(BitSetInfo &BSI);
297297 void allocateByteArrays();
298 Value *createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI, ByteArrayInfo *&BAI,
298 Value *createBitSetTest(IRBuilder<> &B, const TypeIdLowering &TIL,
299299 Value *BitOffset);
300300 void lowerTypeTestCalls(
301301 ArrayRef TypeIds, Constant *CombinedGlobalAddr,
302302 const DenseMap &GlobalLayout);
303 Value *
304 lowerBitSetCall(CallInst *CI, BitSetInfo &BSI, ByteArrayInfo *&BAI,
305 Constant *CombinedGlobal,
306 const DenseMap &GlobalLayout);
303 Value *lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
304 const TypeIdLowering &TIL);
307305 void buildBitSetsFromGlobalVariables(ArrayRef TypeIds,
308306 ArrayRef Globals);
309307 unsigned getJumpTableEntrySize();
428426 BAI->Bits = BSI.Bits;
429427 BAI->BitSize = BSI.BitSize;
430428 BAI->ByteArray = ByteArrayGlobal;
431 BAI->Mask = ConstantExpr::getPtrToInt(MaskGlobal, Int8Ty);
429 BAI->MaskGlobal = MaskGlobal;
432430 return BAI;
433431 }
434432
447445 uint8_t Mask;
448446 BAB.allocate(BAI->Bits, BAI->BitSize, ByteArrayOffsets[I], Mask);
449447
450 BAI->Mask->replaceAllUsesWith(ConstantInt::get(Int8Ty, Mask));
451 cast(BAI->Mask->getOperand(0))->eraseFromParent();
448 BAI->MaskGlobal->replaceAllUsesWith(
449 ConstantExpr::getIntToPtr(ConstantInt::get(Int8Ty, Mask), Int8PtrTy));
450 BAI->MaskGlobal->eraseFromParent();
452451 }
453452
454453 Constant *ByteArrayConst = ConstantDataArray::get(M.getContext(), BAB.Bytes);
483482 ByteArraySizeBytes = BAB.Bytes.size();
484483 }
485484
486 /// Build a test that bit BitOffset is set in BSI, where
487 /// BitSetGlobal is a global containing the bits in BSI.
488 Value *LowerTypeTestsModule::createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI,
489 ByteArrayInfo *&BAI,
485 /// Build a test that bit BitOffset is set in the type identifier that was
486 /// lowered to TIL, which must be either an Inline or a ByteArray.
487 Value *LowerTypeTestsModule::createBitSetTest(IRBuilder<> &B,
488 const TypeIdLowering &TIL,
490489 Value *BitOffset) {
491 if (BSI.BitSize <= 64) {
490 if (TIL.TheKind == TypeTestResolution::Inline) {
492491 // If the bit set is sufficiently small, we can avoid a load by bit testing
493492 // a constant.
494 IntegerType *BitsTy;
495 if (BSI.BitSize <= 32)
496 BitsTy = Int32Ty;
497 else
498 BitsTy = Int64Ty;
499
500 uint64_t Bits = 0;
501 for (auto Bit : BSI.Bits)
502 Bits |= uint64_t(1) << Bit;
503 Constant *BitsConst = ConstantInt::get(BitsTy, Bits);
504 return createMaskedBitTest(B, BitsConst, BitOffset);
493 return createMaskedBitTest(B, TIL.InlineBits, BitOffset);
505494 } else {
506 if (!BAI) {
507 ++NumByteArraysCreated;
508 BAI = createByteArray(BSI);
509 }
510
511 Constant *ByteArray = BAI->ByteArray;
512 Type *Ty = BAI->ByteArray->getValueType();
495 Constant *ByteArray = TIL.TheByteArray;
513496 if (!LinkerSubsectionsViaSymbols && AvoidReuse) {
514497 // Each use of the byte array uses a different alias. This makes the
515498 // backend less likely to reuse previously computed byte array addresses,
516499 // improving the security of the CFI mechanism based on this pass.
517 ByteArray = GlobalAlias::create(BAI->ByteArray->getValueType(), 0,
518 GlobalValue::PrivateLinkage, "bits_use",
519 ByteArray, &M);
520 }
521
522 Value *ByteAddr = B.CreateGEP(Ty, ByteArray, BitOffset);
500 ByteArray = GlobalAlias::create(Int8Ty, 0, GlobalValue::PrivateLinkage,
501 "bits_use", ByteArray, &M);
502 }
503
504 Value *ByteAddr = B.CreateGEP(Int8Ty, ByteArray, BitOffset);
523505 Value *Byte = B.CreateLoad(ByteAddr);
524506
525 Value *ByteAndMask = B.CreateAnd(Byte, BAI->Mask);
507 Value *ByteAndMask =
508 B.CreateAnd(Byte, ConstantExpr::getPtrToInt(TIL.BitMask, Int8Ty));
526509 return B.CreateICmpNE(ByteAndMask, ConstantInt::get(Int8Ty, 0));
527510 }
511 }
512
513 static bool isKnownTypeIdMember(Metadata *TypeId, const DataLayout &DL,
514 Value *V, uint64_t COffset) {
515 if (auto GV = dyn_cast(V)) {
516 SmallVector Types;
517 GV->getMetadata(LLVMContext::MD_type, Types);
518 for (MDNode *Type : Types) {
519 if (Type->getOperand(1) != TypeId)
520 continue;
521 uint64_t Offset =
522 cast(
523 cast(Type->getOperand(0))->getValue())
524 ->getZExtValue();
525 if (COffset == Offset)
526 return true;
527 }
528 return false;
529 }
530
531 if (auto GEP = dyn_cast(V)) {
532 APInt APOffset(DL.getPointerSizeInBits(0), 0);
533 bool Result = GEP->accumulateConstantOffset(DL, APOffset);
534 if (!Result)
535 return false;
536 COffset += APOffset.getZExtValue();
537 return isKnownTypeIdMember(TypeId, DL, GEP->getPointerOperand(), COffset);
538 }
539
540 if (auto Op = dyn_cast(V)) {
541 if (Op->getOpcode() == Instruction::BitCast)
542 return isKnownTypeIdMember(TypeId, DL, Op->getOperand(0), COffset);
543
544 if (Op->getOpcode() == Instruction::Select)
545 return isKnownTypeIdMember(TypeId, DL, Op->getOperand(1), COffset) &&
546 isKnownTypeIdMember(TypeId, DL, Op->getOperand(2), COffset);
547 }
548
549 return false;
528550 }
529551
530552 /// Lower a llvm.type.test call to its implementation. Returns the value to
531553 /// replace the call with.
532 Value *LowerTypeTestsModule::lowerBitSetCall(
533 CallInst *CI, BitSetInfo &BSI, ByteArrayInfo *&BAI,
534 Constant *CombinedGlobalIntAddr,
535 const DenseMap &GlobalLayout) {
554 Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
555 const TypeIdLowering &TIL) {
556 if (TIL.TheKind == TypeTestResolution::Unsat)
557 return ConstantInt::getFalse(M.getContext());
558
536559 Value *Ptr = CI->getArgOperand(0);
537560 const DataLayout &DL = M.getDataLayout();
538
539 if (BSI.containsValue(DL, GlobalLayout, Ptr))
561 if (isKnownTypeIdMember(TypeId, DL, Ptr, 0))
540562 return ConstantInt::getTrue(M.getContext());
541563
542 Constant *OffsetedGlobalAsInt = ConstantExpr::getAdd(
543 CombinedGlobalIntAddr, ConstantInt::get(IntPtrTy, BSI.ByteOffset));
544
545564 BasicBlock *InitialBB = CI->getParent();
546565
547566 IRBuilder<> B(CI);
548567
549568 Value *PtrAsInt = B.CreatePtrToInt(Ptr, IntPtrTy);
550569
551 if (BSI.isSingleOffset())
570 Constant *OffsetedGlobalAsInt =
571 ConstantExpr::getPtrToInt(TIL.OffsetedGlobal, IntPtrTy);
572 if (TIL.TheKind == TypeTestResolution::Single)
552573 return B.CreateICmpEQ(PtrAsInt, OffsetedGlobalAsInt);
553574
554575 Value *PtrOffset = B.CreateSub(PtrAsInt, OffsetedGlobalAsInt);
555576
556 Value *BitOffset;
557 if (BSI.AlignLog2 == 0) {
558 BitOffset = PtrOffset;
559 } else {
560 // We need to check that the offset both falls within our range and is
561 // suitably aligned. We can check both properties at the same time by
562 // performing a right rotate by log2(alignment) followed by an integer
563 // comparison against the bitset size. The rotate will move the lower
564 // order bits that need to be zero into the higher order bits of the
565 // result, causing the comparison to fail if they are nonzero. The rotate
566 // also conveniently gives us a bit offset to use during the load from
567 // the bitset.
568 Value *OffsetSHR =
569 B.CreateLShr(PtrOffset, ConstantInt::get(IntPtrTy, BSI.AlignLog2));
570 Value *OffsetSHL = B.CreateShl(
571 PtrOffset,
572 ConstantInt::get(IntPtrTy, DL.getPointerSizeInBits(0) - BSI.AlignLog2));
573 BitOffset = B.CreateOr(OffsetSHR, OffsetSHL);
574 }
575
576 Constant *BitSizeConst = ConstantInt::get(IntPtrTy, BSI.BitSize);
577 // We need to check that the offset both falls within our range and is
578 // suitably aligned. We can check both properties at the same time by
579 // performing a right rotate by log2(alignment) followed by an integer
580 // comparison against the bitset size. The rotate will move the lower
581 // order bits that need to be zero into the higher order bits of the
582 // result, causing the comparison to fail if they are nonzero. The rotate
583 // also conveniently gives us a bit offset to use during the load from
584 // the bitset.
585 Value *OffsetSHR =
586 B.CreateLShr(PtrOffset, ConstantExpr::getZExt(TIL.AlignLog2, IntPtrTy));
587 Value *OffsetSHL = B.CreateShl(
588 PtrOffset, ConstantExpr::getZExt(
589 ConstantExpr::getSub(
590 ConstantInt::get(Int8Ty, DL.getPointerSizeInBits(0)),
591 TIL.AlignLog2),
592 IntPtrTy));
593 Value *BitOffset = B.CreateOr(OffsetSHR, OffsetSHL);
594
595 Constant *BitSizeConst = ConstantExpr::getZExt(TIL.Size, IntPtrTy);
577596 Value *OffsetInRange = B.CreateICmpULT(BitOffset, BitSizeConst);
578597
579598 // If the bit set is all ones, testing against it is unnecessary.
580 if (BSI.isAllOnes())
599 if (TIL.TheKind == TypeTestResolution::AllOnes)
581600 return OffsetInRange;
582601
583602 TerminatorInst *Term = SplitBlockAndInsertIfThen(OffsetInRange, CI, false);
585604
586605 // Now that we know that the offset is in range and aligned, load the
587606 // appropriate bit from the bitset.
588 Value *Bit = createBitSetTest(ThenB, BSI, BAI, BitOffset);
607 Value *Bit = createBitSetTest(ThenB, TIL, BitOffset);
589608
590609 // The value we want is 0 if we came directly from the initial block
591610 // (having failed the range or alignment checks), or the loaded bit if
670689 void LowerTypeTestsModule::lowerTypeTestCalls(
671690 ArrayRef TypeIds, Constant *CombinedGlobalAddr,
672691 const DenseMap &GlobalLayout) {
673 Constant *CombinedGlobalIntAddr =
674 ConstantExpr::getPtrToInt(CombinedGlobalAddr, IntPtrTy);
675 DenseMap GlobalObjLayout;
676 for (auto &P : GlobalLayout)
677 GlobalObjLayout[P.first->getGlobal()] = P.second;
692 CombinedGlobalAddr = ConstantExpr::getBitCast(CombinedGlobalAddr, Int8PtrTy);
678693
679694 // For each type identifier in this disjoint set...
680695 for (Metadata *TypeId : TypeIds) {
688703 BSI.print(dbgs());
689704 });
690705
691 ByteArrayInfo *BAI = nullptr;
706 TypeIdLowering TIL;
707 TIL.OffsetedGlobal = ConstantExpr::getGetElementPtr(
708 Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, BSI.ByteOffset)),
709 TIL.AlignLog2 = ConstantInt::get(Int8Ty, BSI.AlignLog2);
710 if (BSI.isAllOnes()) {
711 TIL.TheKind = (BSI.BitSize == 1) ? TypeTestResolution::Single
712 : TypeTestResolution::AllOnes;
713 TIL.SizeBitWidth = (BSI.BitSize < 256) ? 8 : 32;
714 TIL.Size =
715 ConstantInt::get((BSI.BitSize < 256) ? Int8Ty : Int32Ty, BSI.BitSize);
716 } else if (BSI.BitSize <= 64) {
717 TIL.TheKind = TypeTestResolution::Inline;
718 TIL.SizeBitWidth = (BSI.BitSize <= 32) ? 5 : 6;
719 TIL.Size = ConstantInt::get(Int8Ty, BSI.BitSize);
720 uint64_t InlineBits = 0;
721 for (auto Bit : BSI.Bits)
722 InlineBits |= uint64_t(1) << Bit;
723 if (InlineBits == 0)
724 TIL.TheKind = TypeTestResolution::Unsat;
725 else
726 TIL.InlineBits = ConstantInt::get(
727 (BSI.BitSize <= 32) ? Int32Ty : Int64Ty, InlineBits);
728 } else {
729 TIL.TheKind = TypeTestResolution::ByteArray;
730 TIL.SizeBitWidth = (BSI.BitSize < 256) ? 8 : 32;
731 TIL.Size =
732 ConstantInt::get((BSI.BitSize < 256) ? Int8Ty : Int32Ty, BSI.BitSize);
733 ++NumByteArraysCreated;
734 ByteArrayInfo *BAI = createByteArray(BSI);
735 TIL.TheByteArray = BAI->ByteArray;
736 TIL.BitMask = BAI->MaskGlobal;
737 }
692738
693739 // Lower each call to llvm.type.test for this type identifier.
694740 for (CallInst *CI : TypeTestCallSites[TypeId]) {
695741 ++NumTypeTestCallsLowered;
696 Value *Lowered =
697 lowerBitSetCall(CI, BSI, BAI, CombinedGlobalIntAddr, GlobalObjLayout);
742 Value *Lowered = lowerTypeTestCall(TypeId, CI, TIL);
698743 CI->replaceAllUsesWith(Lowered);
699744 CI->eraseFromParent();
700745 }
2929
3030 define i1 @foo(i8* %p) {
3131 ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT0]] to i64)
32 ; WASM32: icmp eq i64 {{.*}}, 1
32 ; WASM32: icmp eq i64 {{.*}}, ptrtoint (i8* getelementptr (i8, i8* null, i64 1) to i64)
3333 %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
3434 ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT1]] to i64)
35 ; WASM32: icmp eq i64 {{.*}}, 2
35 ; WASM32: icmp eq i64 {{.*}}, mul (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 2)
3636 %y = call i1 @llvm.type.test(i8* %p, metadata !"typeid2")
3737 %z = add i1 %x, %y
3838 ret i1 %z
4545 ; X64: call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(void ()* @g.cfi)
4646
4747 ; WASM32: ![[I0]] = !{i64 1}
48 ; WASM32: ![[I1]] = !{i64 2}
48 ; WASM32: ![[I1]] = !{i64 2}
1010
1111 define i1 @bar(i8* %ptr) {
1212 ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT:.*]] to i64)
13 ; WASM32: sub i64 {{.*}}, 0
14 ; WASM32: icmp ult i64 {{.*}}, 1
13 ; WASM32: ret i1 false
1514 %p = call i1 @llvm.type.test(i8* %ptr, metadata !"void")
1615 ret i1 %p
1716 }
4141
4242 define i1 @foo(i8* %p) {
4343 ; NATIVE: sub i64 {{.*}}, ptrtoint (void ()* @[[JT]] to i64)
44 ; WASM32: sub i64 {{.*}}, 1
44 ; WASM32: sub i64 {{.*}}, ptrtoint (i8* getelementptr (i8, i8* null, i64 1) to i64)
4545 ; WASM32: icmp ult i64 {{.*}}, 2
4646 %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
4747 ret i1 %x
0 ; Test that we correctly import an unsat resolution for type identifier "typeid1".
1 ; FIXME: We should not require -O2 to simplify this to return false.
2 ; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-unsat.yaml -lowertypetests-write-summary=%t -O2 < %s | FileCheck %s
1 ; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-unsat.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
32 ; RUN: FileCheck --check-prefix=SUMMARY %s < %t
43
54 ; SUMMARY: GlobalValueMap:
9191 ; CHECK: [[S0:%[^ ]*]] = bitcast i32* [[B0]] to i8*
9292 %pi8 = bitcast i32* %p to i8*
9393 ; CHECK: [[S1:%[^ ]*]] = ptrtoint i8* [[S0]] to i32
94 ; CHECK: [[S2:%[^ ]*]] = sub i32 [[S1]], add (i32 ptrtoint ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]] to i32), i32 4)
94 ; CHECK: [[S2:%[^ ]*]] = sub i32 [[S1]], ptrtoint (i8* getelementptr (i8, i8* bitcast ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]] to i8*), i32 4) to i32)
9595 ; CHECK: [[S3:%[^ ]*]] = lshr i32 [[S2]], 8
9696 ; CHECK: [[S4:%[^ ]*]] = shl i32 [[S2]], 24
9797 ; CHECK: [[S5:%[^ ]*]] = or i32 [[S3]], [[S4]]
2323 ; CHECK: @bar(i8* [[B0:%[^ ]*]])
2424 define i1 @bar(i8* %p) {
2525 ; CHECK: [[S0:%[^ ]*]] = ptrtoint i8* [[B0]] to i32
26 ; CHECK: [[S1:%[^ ]*]] = icmp eq i32 [[S0]], add (i32 ptrtoint ({ i32, [0 x i8], i32 }* [[G]] to i32), i32 4)
26 ; CHECK: [[S1:%[^ ]*]] = icmp eq i32 [[S0]], ptrtoint (i8* getelementptr (i8, i8* bitcast ({ i32, [0 x i8], i32 }* [[G]] to i8*), i32 4) to i32)
2727 %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid3")
2828 ; CHECK: ret i1 [[S1]]
2929 ret i1 %x
None ; FIXME: We should not require -O2 to simplify this to return false.
1 ; RUN: opt -S -lowertypetests -O2 < %s | FileCheck %s
0 ; RUN: opt -S -lowertypetests < %s | FileCheck %s
21
32 target datalayout = "e-p:32:32"
43