182#include "llvm/IR/IntrinsicsX86.h"
211#define DEBUG_TYPE "msan"
214 "Controls which checks to insert");
217 "Controls which instruction to instrument");
235 "msan-track-origins",
240 cl::desc(
"keep going after reporting a UMR"),
249 "msan-poison-stack-with-call",
254 "msan-poison-stack-pattern",
255 cl::desc(
"poison uninitialized stack variables with the given pattern"),
260 cl::desc(
"Print name of local stack variable"),
269 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
274 cl::desc(
"exact handling of relational integer ICmp"),
278 "msan-handle-lifetime-intrinsics",
280 "when possible, poison scoped variables at the beginning of the scope "
281 "(slower, but more precise)"),
292 "msan-handle-asm-conservative",
303 "msan-check-access-address",
304 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
309 cl::desc(
"check arguments and return values at function call boundaries"),
313 "msan-dump-strict-instructions",
314 cl::desc(
"print out instructions with default strict semantics"),
318 "msan-instrumentation-with-call-threshold",
320 "If the function being instrumented requires more than "
321 "this number of checks and origin stores, use callbacks instead of "
322 "inline checks (-1 means never use callbacks)."),
327 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
337 cl::desc(
"Insert checks for constant shadow values"),
344 cl::desc(
"Place MSan constructors in comdat sections"),
350 cl::desc(
"Define custom MSan AndMask"),
354 cl::desc(
"Define custom MSan XorMask"),
358 cl::desc(
"Define custom MSan ShadowBase"),
362 cl::desc(
"Define custom MSan OriginBase"),
367 cl::desc(
"Define threshold for number of checks per "
368 "debug location to force origin update."),
380struct MemoryMapParams {
387struct PlatformMemoryMapParams {
388 const MemoryMapParams *bits32;
389 const MemoryMapParams *bits64;
535class MemorySanitizer {
544 MemorySanitizer(MemorySanitizer &&) =
delete;
545 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
546 MemorySanitizer(
const MemorySanitizer &) =
delete;
547 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
552 friend struct MemorySanitizerVisitor;
553 friend struct VarArgHelperBase;
554 friend struct VarArgAMD64Helper;
555 friend struct VarArgMIPS64Helper;
556 friend struct VarArgAArch64Helper;
557 friend struct VarArgPowerPC64Helper;
558 friend struct VarArgSystemZHelper;
560 void initializeModule(
Module &M);
565 template <
typename... ArgsTy>
592 Value *ParamOriginTLS;
598 Value *RetvalOriginTLS;
604 Value *VAArgOriginTLS;
607 Value *VAArgOverflowSizeTLS;
610 bool CallbacksInitialized =
false;
655 Value *MsanMetadataAlloca;
661 const MemoryMapParams *MapParams;
665 MemoryMapParams CustomMapParams;
670 MDNode *OriginStoreWeights;
673void insertModuleCtor(
Module &M) {
701 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
716 MemorySanitizer Msan(*
F.getParent(),
Options);
735 OS, MapClassName2PassName);
742 OS <<
"eager-checks;";
743 OS <<
"track-origins=" <<
Options.TrackOrigins;
759template <
typename... ArgsTy>
767 std::forward<ArgsTy>(Args)...);
770 return M.getOrInsertFunction(
Name, MsanMetadata,
771 std::forward<ArgsTy>(Args)...);
780 RetvalOriginTLS =
nullptr;
782 ParamOriginTLS =
nullptr;
784 VAArgOriginTLS =
nullptr;
785 VAArgOverflowSizeTLS =
nullptr;
787 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
789 IRB.getVoidTy(), IRB.getInt32Ty());
800 MsanGetContextStateFn =
M.getOrInsertFunction(
806 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
807 std::string name_load =
808 "__msan_metadata_ptr_for_load_" + std::to_string(size);
809 std::string name_store =
810 "__msan_metadata_ptr_for_store_" + std::to_string(size);
811 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
813 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
817 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
820 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
821 M,
"__msan_metadata_ptr_for_store_n",
825 MsanPoisonAllocaFn =
M.getOrInsertFunction(
826 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
827 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
828 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
832 return M.getOrInsertGlobal(
Name, Ty, [&] {
834 nullptr,
Name,
nullptr,
847 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
848 :
"__msan_warning_with_origin_noreturn";
849 WarningFn =
M.getOrInsertFunction(WarningFnName,
851 IRB.getVoidTy(), IRB.getInt32Ty());
854 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
855 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
881 VAArgOverflowSizeTLS =
886 unsigned AccessSize = 1 << AccessSizeIndex;
887 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
888 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
890 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
892 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
893 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
895 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
899 MsanSetAllocaOriginWithDescriptionFn =
900 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
901 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
902 MsanSetAllocaOriginNoDescriptionFn =
903 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
904 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
905 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
906 IRB.getVoidTy(), PtrTy, IntptrTy);
912 if (CallbacksInitialized)
918 MsanChainOriginFn =
M.getOrInsertFunction(
919 "__msan_chain_origin",
922 MsanSetOriginFn =
M.getOrInsertFunction(
924 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
926 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
928 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
929 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
931 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
933 MsanInstrumentAsmStoreFn =
934 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
938 createKernelApi(M, TLI);
940 createUserspaceApi(M, TLI);
942 CallbacksInitialized =
true;
948 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
966void MemorySanitizer::initializeModule(
Module &M) {
967 auto &
DL =
M.getDataLayout();
969 TargetTriple =
Triple(
M.getTargetTriple());
971 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
972 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
974 if (ShadowPassed || OriginPassed) {
979 MapParams = &CustomMapParams;
981 switch (TargetTriple.getOS()) {
983 switch (TargetTriple.getArch()) {
998 switch (TargetTriple.getArch()) {
1007 switch (TargetTriple.getArch()) {
1041 C = &(
M.getContext());
1043 IntptrTy = IRB.getIntPtrTy(
DL);
1044 OriginTy = IRB.getInt32Ty();
1045 PtrTy = IRB.getPtrTy();
1050 if (!CompileKernel) {
1052 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1053 return new GlobalVariable(
1054 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1055 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1059 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1060 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1061 GlobalValue::WeakODRLinkage,
1062 IRB.getInt32(Recover),
"__msan_keep_going");
1077struct VarArgHelper {
1078 virtual ~VarArgHelper() =
default;
1093 virtual void finalizeInstrumentation() = 0;
1096struct MemorySanitizerVisitor;
1101 MemorySanitizerVisitor &Visitor);
1108 if (TypeSizeFixed <= 8)
1117class NextNodeIRBuilder :
public IRBuilder<> {
1130struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1132 MemorySanitizer &MS;
1135 std::unique_ptr<VarArgHelper> VAHelper;
1142 bool PropagateShadow;
1146 struct ShadowOriginAndInsertPoint {
1152 : Shadow(S), Origin(
O), OrigIns(
I) {}
1160 int64_t SplittableBlocksCount = 0;
1162 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1165 bool SanitizeFunction =
1167 InsertChecks = SanitizeFunction;
1168 PropagateShadow = SanitizeFunction;
1178 MS.initializeCallbacks(*
F.getParent(), TLI);
1179 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1182 if (MS.CompileKernel) {
1184 insertKmsanPrologue(IRB);
1188 <<
"MemorySanitizer is not inserting checks into '"
1189 <<
F.getName() <<
"'\n");
1192 bool instrumentWithCalls(
Value *V) {
1194 if (isa<Constant>(V))
1197 ++SplittableBlocksCount;
1203 return I.getParent() == FnPrologueEnd->
getParent() &&
1204 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1212 if (MS.TrackOrigins <= 1)
1214 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1219 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1231 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1232 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1242 auto [InsertPt,
Index] =
1254 Align CurrentAlignment = Alignment;
1255 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1256 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1257 Value *IntptrOriginPtr =
1259 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1264 CurrentAlignment = IntptrAlignment;
1281 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1282 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1290 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1299 if (instrumentWithCalls(ConvertedShadow) &&
1302 Value *ConvertedShadow2 =
1308 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1312 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1317 void materializeStores() {
1320 Value *Val =
SI->getValueOperand();
1322 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1323 Value *ShadowPtr, *OriginPtr;
1325 const Align Alignment =
SI->getAlign();
1327 std::tie(ShadowPtr, OriginPtr) =
1328 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1337 if (MS.TrackOrigins && !
SI->isAtomic())
1338 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1345 if (MS.TrackOrigins < 2)
1348 if (LazyWarningDebugLocationCount.
empty())
1349 for (
const auto &
I : InstrumentationList)
1350 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1364 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1366 auto NewDebugLoc = OI->getDebugLoc();
1373 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1374 Origin = updateOrigin(Origin, IRBOrigin);
1379 if (MS.CompileKernel || MS.TrackOrigins)
1393 if (instrumentWithCalls(ConvertedShadow) &&
1396 Value *ConvertedShadow2 =
1399 Fn, {ConvertedShadow2,
1400 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1404 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1407 !MS.Recover, MS.ColdCallWeights);
1410 insertWarningFn(IRB, Origin);
1415 void materializeInstructionChecks(
1420 bool Combine = !MS.TrackOrigins;
1422 Value *Shadow =
nullptr;
1423 for (
const auto &ShadowData : InstructionChecks) {
1427 Value *ConvertedShadow = ShadowData.Shadow;
1429 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1436 insertWarningFn(IRB, ShadowData.Origin);
1446 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1451 Shadow = ConvertedShadow;
1455 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1456 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1457 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1463 materializeOneCheck(IRB, Shadow,
nullptr);
1467 void materializeChecks() {
1473 for (
auto I = InstrumentationList.begin();
1474 I != InstrumentationList.end();) {
1475 auto OrigIns =
I->OrigIns;
1479 auto J = std::find_if(
I + 1, InstrumentationList.end(),
1480 [OrigIns](
const ShadowOriginAndInsertPoint &R) {
1481 return OrigIns != R.OrigIns;
1495 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1496 {Zero, IRB.getInt32(0)},
"param_shadow");
1497 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1498 {Zero, IRB.getInt32(1)},
"retval_shadow");
1499 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1500 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1501 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1502 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1503 MS.VAArgOverflowSizeTLS =
1504 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1505 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1506 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1507 {Zero, IRB.getInt32(5)},
"param_origin");
1508 MS.RetvalOriginTLS =
1509 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1510 {Zero, IRB.getInt32(6)},
"retval_origin");
1512 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1524 for (
PHINode *PN : ShadowPHINodes) {
1525 PHINode *PNS = cast<PHINode>(getShadow(PN));
1526 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1527 size_t NumValues = PN->getNumIncomingValues();
1528 for (
size_t v = 0;
v < NumValues;
v++) {
1529 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1531 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1535 VAHelper->finalizeInstrumentation();
1539 if (InstrumentLifetimeStart) {
1540 for (
auto Item : LifetimeStartList) {
1541 instrumentAlloca(*Item.second, Item.first);
1542 AllocaSet.
remove(Item.second);
1548 instrumentAlloca(*AI);
1551 materializeChecks();
1555 materializeStores();
1561 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1573 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1574 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1576 VT->getElementCount());
1578 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1579 return ArrayType::get(getShadowTy(AT->getElementType()),
1580 AT->getNumElements());
1582 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1584 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1585 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1587 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1603 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1605 if (Aggregator != FalseVal)
1606 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1608 Aggregator = ShadowBool;
1617 if (!
Array->getNumElements())
1621 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1625 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1626 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1636 return collapseStructShadow(
Struct, V, IRB);
1637 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1638 return collapseArrayShadow(Array, V, IRB);
1639 if (isa<VectorType>(
V->getType())) {
1640 if (isa<ScalableVectorType>(
V->getType()))
1643 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1651 Type *VTy =
V->getType();
1653 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1660 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1661 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1662 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1663 VectTy->getElementCount());
1669 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1670 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1671 return VectorType::get(
1672 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1673 VectTy->getElementCount());
1675 assert(IntPtrTy == MS.IntptrTy);
1676 return PointerType::get(*MS.C, 0);
1680 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1682 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1684 assert(IntPtrTy == MS.IntptrTy);
1685 return ConstantInt::get(MS.IntptrTy,
C);
1696 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1699 if (
uint64_t AndMask = MS.MapParams->AndMask)
1700 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1702 if (
uint64_t XorMask = MS.MapParams->XorMask)
1703 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1715 std::pair<Value *, Value *>
1722 assert(VectTy->getElementType()->isPointerTy());
1724 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1725 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1726 Value *ShadowLong = ShadowOffset;
1727 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1729 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1732 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1734 Value *OriginPtr =
nullptr;
1735 if (MS.TrackOrigins) {
1736 Value *OriginLong = ShadowOffset;
1737 uint64_t OriginBase = MS.MapParams->OriginBase;
1738 if (OriginBase != 0)
1740 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1743 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1746 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1748 return std::make_pair(ShadowPtr, OriginPtr);
1751 template <
typename... ArgsTy>
1756 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1757 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1760 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1763 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1767 Value *ShadowOriginPtrs;
1775 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1777 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1778 ShadowOriginPtrs = createMetadataCall(
1780 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1787 return std::make_pair(ShadowPtr, OriginPtr);
1793 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1800 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1804 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1805 Value *ShadowPtrs = ConstantInt::getNullValue(
1807 Value *OriginPtrs =
nullptr;
1808 if (MS.TrackOrigins)
1809 OriginPtrs = ConstantInt::getNullValue(
1811 for (
unsigned i = 0; i < NumElements; ++i) {
1814 auto [ShadowPtr, OriginPtr] =
1815 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1818 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1819 if (MS.TrackOrigins)
1821 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1823 return {ShadowPtrs, OriginPtrs};
1830 if (MS.CompileKernel)
1831 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1832 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1847 if (!MS.TrackOrigins)
1861 Value *getOriginPtrForRetval() {
1863 return MS.RetvalOriginTLS;
1868 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1869 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1874 if (!MS.TrackOrigins)
1876 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1877 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1878 OriginMap[
V] = Origin;
1882 Type *ShadowTy = getShadowTy(OrigTy);
1892 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1897 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1899 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1901 getPoisonedShadow(AT->getElementType()));
1904 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1906 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1907 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1915 Type *ShadowTy = getShadowTy(V);
1918 return getPoisonedShadow(ShadowTy);
1930 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1931 return getCleanShadow(V);
1933 Value *Shadow = ShadowMap[
V];
1935 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1937 assert(Shadow &&
"No shadow for a value");
1941 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1942 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1943 : getCleanShadow(V);
1948 if (
Argument *
A = dyn_cast<Argument>(V)) {
1950 Value *&ShadowPtr = ShadowMap[
V];
1955 unsigned ArgOffset = 0;
1957 for (
auto &FArg :
F->args()) {
1958 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1960 ?
"vscale not fully supported\n"
1961 :
"Arg is not sized\n"));
1963 ShadowPtr = getCleanShadow(V);
1964 setOrigin(
A, getCleanOrigin());
1970 unsigned Size = FArg.hasByValAttr()
1971 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1972 :
DL.getTypeAllocSize(FArg.getType());
1976 if (FArg.hasByValAttr()) {
1980 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1981 FArg.getParamAlign(), FArg.getParamByValType());
1982 Value *CpShadowPtr, *CpOriginPtr;
1983 std::tie(CpShadowPtr, CpOriginPtr) =
1984 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1986 if (!PropagateShadow || Overflow) {
1988 EntryIRB.CreateMemSet(
1992 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
1994 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
1999 if (MS.TrackOrigins) {
2001 getOriginPtrForArgument(EntryIRB, ArgOffset);
2005 EntryIRB.CreateMemCpy(
2014 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2015 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2016 ShadowPtr = getCleanShadow(V);
2017 setOrigin(
A, getCleanOrigin());
2020 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2021 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2023 if (MS.TrackOrigins) {
2025 getOriginPtrForArgument(EntryIRB, ArgOffset);
2026 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2030 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2036 assert(ShadowPtr &&
"Could not find shadow for an argument");
2040 return getCleanShadow(V);
2045 return getShadow(
I->getOperand(i));
2050 if (!MS.TrackOrigins)
2052 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2053 return getCleanOrigin();
2054 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2055 "Unexpected value type in getOrigin()");
2057 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2058 return getCleanOrigin();
2060 Value *Origin = OriginMap[
V];
2061 assert(Origin &&
"Missing origin");
2067 return getOrigin(
I->getOperand(i));
2080 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2081 << *OrigIns <<
"\n");
2086 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2087 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2088 "Can only insert checks for integer, vector, and aggregate shadow "
2091 InstrumentationList.push_back(
2092 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2101 Value *Shadow, *Origin;
2103 Shadow = getShadow(Val);
2106 Origin = getOrigin(Val);
2108 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2111 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2113 insertShadowCheck(Shadow, Origin, OrigIns);
2118 case AtomicOrdering::NotAtomic:
2119 return AtomicOrdering::NotAtomic;
2120 case AtomicOrdering::Unordered:
2121 case AtomicOrdering::Monotonic:
2122 case AtomicOrdering::Release:
2123 return AtomicOrdering::Release;
2124 case AtomicOrdering::Acquire:
2125 case AtomicOrdering::AcquireRelease:
2126 return AtomicOrdering::AcquireRelease;
2127 case AtomicOrdering::SequentiallyConsistent:
2128 return AtomicOrdering::SequentiallyConsistent;
2134 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2135 uint32_t OrderingTable[NumOrderings] = {};
2137 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2138 OrderingTable[(
int)AtomicOrderingCABI::release] =
2139 (int)AtomicOrderingCABI::release;
2140 OrderingTable[(int)AtomicOrderingCABI::consume] =
2141 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2142 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2143 (
int)AtomicOrderingCABI::acq_rel;
2144 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2145 (
int)AtomicOrderingCABI::seq_cst;
2152 case AtomicOrdering::NotAtomic:
2153 return AtomicOrdering::NotAtomic;
2154 case AtomicOrdering::Unordered:
2155 case AtomicOrdering::Monotonic:
2156 case AtomicOrdering::Acquire:
2157 return AtomicOrdering::Acquire;
2158 case AtomicOrdering::Release:
2159 case AtomicOrdering::AcquireRelease:
2160 return AtomicOrdering::AcquireRelease;
2161 case AtomicOrdering::SequentiallyConsistent:
2162 return AtomicOrdering::SequentiallyConsistent;
2168 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2169 uint32_t OrderingTable[NumOrderings] = {};
2171 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2172 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2173 OrderingTable[(int)AtomicOrderingCABI::consume] =
2174 (
int)AtomicOrderingCABI::acquire;
2175 OrderingTable[(int)AtomicOrderingCABI::release] =
2176 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2177 (int)AtomicOrderingCABI::acq_rel;
2178 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2179 (
int)AtomicOrderingCABI::seq_cst;
2187 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2190 if (isInPrologue(
I))
2195 setShadow(&
I, getCleanShadow(&
I));
2196 setOrigin(&
I, getCleanOrigin());
2207 assert(
I.getType()->isSized() &&
"Load type must have size");
2208 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2209 NextNodeIRBuilder IRB(&
I);
2210 Type *ShadowTy = getShadowTy(&
I);
2212 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2213 const Align Alignment =
I.getAlign();
2214 if (PropagateShadow) {
2215 std::tie(ShadowPtr, OriginPtr) =
2216 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2220 setShadow(&
I, getCleanShadow(&
I));
2224 insertShadowCheck(
I.getPointerOperand(), &
I);
2229 if (MS.TrackOrigins) {
2230 if (PropagateShadow) {
2235 setOrigin(&
I, getCleanOrigin());
2245 StoreList.push_back(&
I);
2247 insertShadowCheck(
I.getPointerOperand(), &
I);
2251 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2255 Value *Val =
I.getOperand(1);
2256 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2261 insertShadowCheck(
Addr, &
I);
2266 if (isa<AtomicCmpXchgInst>(
I))
2267 insertShadowCheck(Val, &
I);
2271 setShadow(&
I, getCleanShadow(&
I));
2272 setOrigin(&
I, getCleanOrigin());
2287 insertShadowCheck(
I.getOperand(1), &
I);
2291 setOrigin(&
I, getOrigin(&
I, 0));
2295 insertShadowCheck(
I.getOperand(2), &
I);
2297 auto *Shadow0 = getShadow(&
I, 0);
2298 auto *Shadow1 = getShadow(&
I, 1);
2301 setOriginForNaryOp(
I);
2306 auto *Shadow0 = getShadow(&
I, 0);
2307 auto *Shadow1 = getShadow(&
I, 1);
2310 setOriginForNaryOp(
I);
2316 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2317 setOrigin(&
I, getOrigin(&
I, 0));
2322 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2323 setOrigin(&
I, getOrigin(&
I, 0));
2328 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2329 setOrigin(&
I, getOrigin(&
I, 0));
2336 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2337 if (CI->isMustTailCall())
2341 setOrigin(&
I, getOrigin(&
I, 0));
2347 "_msprop_ptrtoint"));
2348 setOrigin(&
I, getOrigin(&
I, 0));
2354 "_msprop_inttoptr"));
2355 setOrigin(&
I, getOrigin(&
I, 0));
2358 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2359 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2360 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2361 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2362 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2363 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2378 Value *S2 = getShadow(&
I, 1);
2379 Value *V1 =
I.getOperand(0);
2388 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2389 setOriginForNaryOp(
I);
2400 Value *S2 = getShadow(&
I, 1);
2410 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2411 setOriginForNaryOp(
I);
2429 template <
bool CombineShadow>
class Combiner {
2430 Value *Shadow =
nullptr;
2431 Value *Origin =
nullptr;
2433 MemorySanitizerVisitor *MSV;
2437 : IRB(IRB), MSV(MSV) {}
2441 if (CombineShadow) {
2446 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2447 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2451 if (MSV->MS.TrackOrigins) {
2456 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2458 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2459 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2469 Value *OpShadow = MSV->getShadow(V);
2470 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2471 return Add(OpShadow, OpOrigin);
2477 if (CombineShadow) {
2479 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2480 MSV->setShadow(
I, Shadow);
2482 if (MSV->MS.TrackOrigins) {
2484 MSV->setOrigin(
I, Origin);
2494 if (!MS.TrackOrigins)
2497 OriginCombiner
OC(
this, IRB);
2498 for (
Use &
Op :
I.operands())
2503 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2505 "Vector of pointers is not a valid shadow type");
2506 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2515 Type *srcTy =
V->getType();
2518 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2519 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2520 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2526 cast<VectorType>(dstTy)->getElementCount() ==
2527 cast<VectorType>(srcTy)->getElementCount())
2538 Type *ShadowTy = getShadowTy(V);
2539 if (
V->getType() == ShadowTy)
2541 if (
V->getType()->isPtrOrPtrVectorTy())
2550 ShadowAndOriginCombiner
SC(
this, IRB);
2551 for (
Use &
Op :
I.operands())
2571 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2572 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2573 Type *EltTy = VTy->getElementType();
2575 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2578 const APInt &
V = Elt->getValue();
2580 Elements.push_back(ConstantInt::get(EltTy, V2));
2582 Elements.push_back(ConstantInt::get(EltTy, 1));
2587 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2588 const APInt &
V = Elt->getValue();
2590 ShadowMul = ConstantInt::get(Ty, V2);
2592 ShadowMul = ConstantInt::get(Ty, 1);
2598 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2599 setOrigin(&
I, getOrigin(OtherArg));
2603 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2604 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2605 if (constOp0 && !constOp1)
2606 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2607 else if (constOp1 && !constOp0)
2608 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2623 insertShadowCheck(
I.getOperand(1), &
I);
2624 setShadow(&
I, getShadow(&
I, 0));
2625 setOrigin(&
I, getOrigin(&
I, 0));
2642 void handleEqualityComparison(
ICmpInst &
I) {
2646 Value *Sa = getShadow(
A);
2647 Value *Sb = getShadow(
B);
2673 setOriginForNaryOp(
I);
2715 void handleRelationalComparisonExact(
ICmpInst &
I) {
2719 Value *Sa = getShadow(
A);
2720 Value *Sb = getShadow(
B);
2731 bool IsSigned =
I.isSigned();
2733 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2734 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2736 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2737 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2740 setOriginForNaryOp(
I);
2747 void handleSignedRelationalComparison(
ICmpInst &
I) {
2751 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2752 op =
I.getOperand(0);
2753 pre =
I.getPredicate();
2754 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2755 op =
I.getOperand(1);
2756 pre =
I.getSwappedPredicate();
2769 setShadow(&
I, Shadow);
2770 setOrigin(&
I, getOrigin(
op));
2781 if (
I.isEquality()) {
2782 handleEqualityComparison(
I);
2788 handleRelationalComparisonExact(
I);
2792 handleSignedRelationalComparison(
I);
2797 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2798 handleRelationalComparisonExact(
I);
2805 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2812 Value *S2 = getShadow(&
I, 1);
2817 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2818 setOriginForNaryOp(
I);
2829 Value *S0 = getShadow(&
I, 0);
2831 Value *S2 = getShadow(&
I, 2);
2836 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2838 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2839 setOriginForNaryOp(
I);
2853 getShadow(
I.getArgOperand(1));
2856 {I.getArgOperand(0), I.getArgOperand(1),
2857 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2858 I.eraseFromParent();
2876 getShadow(
I.getArgOperand(1));
2879 {I.getArgOperand(0), I.getArgOperand(1),
2880 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2881 I.eraseFromParent();
2889 {I.getArgOperand(0),
2890 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2891 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2892 I.eraseFromParent();
2895 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2897 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2906 Value *Shadow = getShadow(&
I, 1);
2907 Value *ShadowPtr, *OriginPtr;
2911 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2916 insertShadowCheck(
Addr, &
I);
2919 if (MS.TrackOrigins)
2932 Type *ShadowTy = getShadowTy(&
I);
2933 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2934 if (PropagateShadow) {
2938 std::tie(ShadowPtr, OriginPtr) =
2939 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2943 setShadow(&
I, getCleanShadow(&
I));
2947 insertShadowCheck(
Addr, &
I);
2949 if (MS.TrackOrigins) {
2950 if (PropagateShadow)
2951 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2953 setOrigin(&
I, getCleanOrigin());
2966 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2967 RetTy->isX86_MMXTy()))
2970 unsigned NumArgOperands =
I.arg_size();
2971 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2972 Type *Ty =
I.getArgOperand(i)->getType();
2978 ShadowAndOriginCombiner
SC(
this, IRB);
2979 for (
unsigned i = 0; i < NumArgOperands; ++i)
2980 SC.Add(
I.getArgOperand(i));
2997 unsigned NumArgOperands =
I.arg_size();
2998 if (NumArgOperands == 0)
3001 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3002 I.getArgOperand(1)->getType()->isVectorTy() &&
3003 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
3005 return handleVectorStoreIntrinsic(
I);
3008 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3009 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
3011 return handleVectorLoadIntrinsic(
I);
3014 if (
I.doesNotAccessMemory())
3015 if (maybeHandleSimpleNomemIntrinsic(
I))
3023 setShadow(&
I, getShadow(&
I, 0));
3024 setOrigin(&
I, getOrigin(&
I, 0));
3032 InstrumentLifetimeStart =
false;
3033 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3039 Type *OpType =
Op->getType();
3041 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3043 setOrigin(&
I, getOrigin(
Op));
3048 Value *Src =
I.getArgOperand(0);
3054 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3057 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3060 Value *OutputShadow =
3061 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3063 setShadow(&
I, OutputShadow);
3064 setOriginForNaryOp(
I);
3082 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3083 bool HasRoundingMode =
false) {
3085 Value *CopyOp, *ConvertOp;
3087 assert((!HasRoundingMode ||
3088 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3089 "Invalid rounding mode");
3091 switch (
I.arg_size() - HasRoundingMode) {
3093 CopyOp =
I.getArgOperand(0);
3094 ConvertOp =
I.getArgOperand(1);
3097 ConvertOp =
I.getArgOperand(0);
3111 Value *ConvertShadow = getShadow(ConvertOp);
3112 Value *AggShadow =
nullptr;
3115 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3116 for (
int i = 1; i < NumUsedElements; ++i) {
3118 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3119 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3122 AggShadow = ConvertShadow;
3125 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3132 Value *ResultShadow = getShadow(CopyOp);
3133 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3134 for (
int i = 0; i < NumUsedElements; ++i) {
3136 ResultShadow, ConstantInt::getNullValue(EltTy),
3139 setShadow(&
I, ResultShadow);
3140 setOrigin(&
I, getOrigin(CopyOp));
3142 setShadow(&
I, getCleanShadow(&
I));
3143 setOrigin(&
I, getCleanOrigin());
3151 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3154 return CreateShadowCast(IRB, S2,
T,
true);
3162 return CreateShadowCast(IRB, S2,
T,
true);
3179 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3185 Value *S2 = getShadow(&
I, 1);
3186 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3187 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3188 Value *V1 =
I.getOperand(0);
3191 {IRB.CreateBitCast(S1, V1->getType()), V2});
3193 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3194 setOriginForNaryOp(
I);
3198 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3199 const unsigned X86_MMXSizeInBits = 64;
3200 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3201 "Illegal MMX vector element size");
3203 X86_MMXSizeInBits / EltSizeInBits);
3210 case Intrinsic::x86_sse2_packsswb_128:
3211 case Intrinsic::x86_sse2_packuswb_128:
3212 return Intrinsic::x86_sse2_packsswb_128;
3214 case Intrinsic::x86_sse2_packssdw_128:
3215 case Intrinsic::x86_sse41_packusdw:
3216 return Intrinsic::x86_sse2_packssdw_128;
3218 case Intrinsic::x86_avx2_packsswb:
3219 case Intrinsic::x86_avx2_packuswb:
3220 return Intrinsic::x86_avx2_packsswb;
3222 case Intrinsic::x86_avx2_packssdw:
3223 case Intrinsic::x86_avx2_packusdw:
3224 return Intrinsic::x86_avx2_packssdw;
3226 case Intrinsic::x86_mmx_packsswb:
3227 case Intrinsic::x86_mmx_packuswb:
3228 return Intrinsic::x86_mmx_packsswb;
3230 case Intrinsic::x86_mmx_packssdw:
3231 return Intrinsic::x86_mmx_packssdw;
3244 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3246 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3249 Value *S2 = getShadow(&
I, 1);
3250 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3255 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3271 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3274 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3278 setOriginForNaryOp(
I);
3283 const unsigned SignificantBitsPerResultElement = 16;
3284 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3286 unsigned ZeroBitsPerResultElement =
3290 auto *Shadow0 = getShadow(&
I, 0);
3291 auto *Shadow1 = getShadow(&
I, 1);
3296 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3299 setOriginForNaryOp(
I);
3304 unsigned EltSizeInBits = 0) {
3305 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3306 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3308 auto *Shadow0 = getShadow(&
I, 0);
3309 auto *Shadow1 = getShadow(&
I, 1);
3316 setOriginForNaryOp(
I);
3324 Type *ResTy = getShadowTy(&
I);
3325 auto *Shadow0 = getShadow(&
I, 0);
3326 auto *Shadow1 = getShadow(&
I, 1);
3331 setOriginForNaryOp(
I);
3339 auto *Shadow0 = getShadow(&
I, 0);
3340 auto *Shadow1 = getShadow(&
I, 1);
3342 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3344 setOriginForNaryOp(
I);
3353 setOrigin(&
I, getOrigin(&
I, 0));
3361 Value *OperandShadow = getShadow(&
I, 0);
3363 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3371 setOrigin(&
I, getOrigin(&
I, 0));
3379 Value *OperandShadow = getShadow(&
I, 0);
3380 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3388 setOrigin(&
I, getOrigin(&
I, 0));
3396 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3401 insertShadowCheck(
Addr, &
I);
3412 Value *ShadowPtr, *OriginPtr;
3413 std::tie(ShadowPtr, OriginPtr) =
3414 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3417 insertShadowCheck(
Addr, &
I);
3420 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3422 insertShadowCheck(Shadow, Origin, &
I);
3429 Value *PassThru =
I.getArgOperand(2);
3432 insertShadowCheck(
Ptr, &
I);
3433 insertShadowCheck(Mask, &
I);
3436 if (!PropagateShadow) {
3437 setShadow(&
I, getCleanShadow(&
I));
3438 setOrigin(&
I, getCleanOrigin());
3442 Type *ShadowTy = getShadowTy(&
I);
3443 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3444 auto [ShadowPtr, OriginPtr] =
3445 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3448 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3450 setShadow(&
I, Shadow);
3453 setOrigin(&
I, getCleanOrigin());
3458 Value *Values =
I.getArgOperand(0);
3463 insertShadowCheck(
Ptr, &
I);
3464 insertShadowCheck(Mask, &
I);
3467 Value *Shadow = getShadow(Values);
3468 Type *ElementShadowTy =
3469 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3470 auto [ShadowPtr, OriginPtrs] =
3471 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3480 Value *Ptrs =
I.getArgOperand(0);
3481 const Align Alignment(
3482 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3484 Value *PassThru =
I.getArgOperand(3);
3486 Type *PtrsShadowTy = getShadowTy(Ptrs);
3488 insertShadowCheck(Mask, &
I);
3492 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3495 if (!PropagateShadow) {
3496 setShadow(&
I, getCleanShadow(&
I));
3497 setOrigin(&
I, getCleanOrigin());
3501 Type *ShadowTy = getShadowTy(&
I);
3502 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3503 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3504 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3508 getShadow(PassThru),
"_msmaskedgather");
3510 setShadow(&
I, Shadow);
3513 setOrigin(&
I, getCleanOrigin());
3518 Value *Values =
I.getArgOperand(0);
3519 Value *Ptrs =
I.getArgOperand(1);
3520 const Align Alignment(
3521 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3524 Type *PtrsShadowTy = getShadowTy(Ptrs);
3526 insertShadowCheck(Mask, &
I);
3530 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3533 Value *Shadow = getShadow(Values);
3534 Type *ElementShadowTy =
3535 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3536 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3537 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3546 Value *
V =
I.getArgOperand(0);
3548 const Align Alignment(
3549 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3551 Value *Shadow = getShadow(V);
3554 insertShadowCheck(
Ptr, &
I);
3555 insertShadowCheck(Mask, &
I);
3560 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3561 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3565 if (!MS.TrackOrigins)
3568 auto &
DL =
F.getParent()->getDataLayout();
3569 paintOrigin(IRB, getOrigin(V), OriginPtr,
3577 const Align Alignment(
3578 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3580 Value *PassThru =
I.getArgOperand(3);
3583 insertShadowCheck(
Ptr, &
I);
3584 insertShadowCheck(Mask, &
I);
3587 if (!PropagateShadow) {
3588 setShadow(&
I, getCleanShadow(&
I));
3589 setOrigin(&
I, getCleanOrigin());
3593 Type *ShadowTy = getShadowTy(&
I);
3594 Value *ShadowPtr, *OriginPtr;
3595 std::tie(ShadowPtr, OriginPtr) =
3596 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3598 getShadow(PassThru),
"_msmaskedld"));
3600 if (!MS.TrackOrigins)
3607 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3612 setOrigin(&
I, Origin);
3622 Type *ShadowTy = getShadowTy(&
I);
3625 Value *SMask = getShadow(&
I, 1);
3630 {getShadow(&I, 0), I.getOperand(1)});
3633 setOriginForNaryOp(
I);
3638 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3655 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3656 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3657 "pclmul 3rd operand must be a constant");
3658 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3660 getPclmulMask(Width, Imm & 0x01));
3662 getPclmulMask(Width, Imm & 0x10));
3663 ShadowAndOriginCombiner SOC(
this, IRB);
3664 SOC.Add(Shuf0, getOrigin(&
I, 0));
3665 SOC.Add(Shuf1, getOrigin(&
I, 1));
3673 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3675 Value *Second = getShadow(&
I, 1);
3678 Mask.push_back(Width);
3679 for (
unsigned i = 1; i < Width; i++)
3683 setShadow(&
I, Shadow);
3684 setOriginForNaryOp(
I);
3689 Value *Shadow0 = getShadow(&
I, 0);
3690 Value *Shadow1 = getShadow(&
I, 1);
3696 setShadow(&
I, Shadow);
3697 setOriginForNaryOp(
I);
3703 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3705 Value *Second = getShadow(&
I, 1);
3709 Mask.push_back(Width);
3710 for (
unsigned i = 1; i < Width; i++)
3714 setShadow(&
I, Shadow);
3715 setOriginForNaryOp(
I);
3722 assert(
I.getType()->isIntOrIntVectorTy());
3723 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3727 setShadow(&
I, getShadow(&
I, 0));
3728 setOrigin(&
I, getOrigin(&
I, 0));
3733 Value *Shadow = getShadow(&
I, 0);
3734 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3735 setOrigin(&
I, getOrigin(&
I, 0));
3740 Value *Shadow0 = getShadow(&
I, 0);
3741 Value *Shadow1 = getShadow(&
I, 1);
3744 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3750 setShadow(&
I, Shadow);
3751 setOriginForNaryOp(
I);
3755 switch (
I.getIntrinsicID()) {
3756 case Intrinsic::uadd_with_overflow:
3757 case Intrinsic::sadd_with_overflow:
3758 case Intrinsic::usub_with_overflow:
3759 case Intrinsic::ssub_with_overflow:
3760 case Intrinsic::umul_with_overflow:
3761 case Intrinsic::smul_with_overflow:
3762 handleArithmeticWithOverflow(
I);
3764 case Intrinsic::abs:
3765 handleAbsIntrinsic(
I);
3767 case Intrinsic::is_fpclass:
3770 case Intrinsic::lifetime_start:
3771 handleLifetimeStart(
I);
3773 case Intrinsic::launder_invariant_group:
3774 case Intrinsic::strip_invariant_group:
3775 handleInvariantGroup(
I);
3777 case Intrinsic::bswap:
3780 case Intrinsic::ctlz:
3781 case Intrinsic::cttz:
3782 handleCountZeroes(
I);
3784 case Intrinsic::masked_compressstore:
3785 handleMaskedCompressStore(
I);
3787 case Intrinsic::masked_expandload:
3788 handleMaskedExpandLoad(
I);
3790 case Intrinsic::masked_gather:
3791 handleMaskedGather(
I);
3793 case Intrinsic::masked_scatter:
3794 handleMaskedScatter(
I);
3796 case Intrinsic::masked_store:
3797 handleMaskedStore(
I);
3799 case Intrinsic::masked_load:
3800 handleMaskedLoad(
I);
3802 case Intrinsic::vector_reduce_and:
3803 handleVectorReduceAndIntrinsic(
I);
3805 case Intrinsic::vector_reduce_or:
3806 handleVectorReduceOrIntrinsic(
I);
3808 case Intrinsic::vector_reduce_add:
3809 case Intrinsic::vector_reduce_xor:
3810 case Intrinsic::vector_reduce_mul:
3811 handleVectorReduceIntrinsic(
I);
3813 case Intrinsic::x86_sse_stmxcsr:
3816 case Intrinsic::x86_sse_ldmxcsr:
3819 case Intrinsic::x86_avx512_vcvtsd2usi64:
3820 case Intrinsic::x86_avx512_vcvtsd2usi32:
3821 case Intrinsic::x86_avx512_vcvtss2usi64:
3822 case Intrinsic::x86_avx512_vcvtss2usi32:
3823 case Intrinsic::x86_avx512_cvttss2usi64:
3824 case Intrinsic::x86_avx512_cvttss2usi:
3825 case Intrinsic::x86_avx512_cvttsd2usi64:
3826 case Intrinsic::x86_avx512_cvttsd2usi:
3827 case Intrinsic::x86_avx512_cvtusi2ss:
3828 case Intrinsic::x86_avx512_cvtusi642sd:
3829 case Intrinsic::x86_avx512_cvtusi642ss:
3830 handleVectorConvertIntrinsic(
I, 1,
true);
3832 case Intrinsic::x86_sse2_cvtsd2si64:
3833 case Intrinsic::x86_sse2_cvtsd2si:
3834 case Intrinsic::x86_sse2_cvtsd2ss:
3835 case Intrinsic::x86_sse2_cvttsd2si64:
3836 case Intrinsic::x86_sse2_cvttsd2si:
3837 case Intrinsic::x86_sse_cvtss2si64:
3838 case Intrinsic::x86_sse_cvtss2si:
3839 case Intrinsic::x86_sse_cvttss2si64:
3840 case Intrinsic::x86_sse_cvttss2si:
3841 handleVectorConvertIntrinsic(
I, 1);
3843 case Intrinsic::x86_sse_cvtps2pi:
3844 case Intrinsic::x86_sse_cvttps2pi:
3845 handleVectorConvertIntrinsic(
I, 2);
3848 case Intrinsic::x86_avx512_psll_w_512:
3849 case Intrinsic::x86_avx512_psll_d_512:
3850 case Intrinsic::x86_avx512_psll_q_512:
3851 case Intrinsic::x86_avx512_pslli_w_512:
3852 case Intrinsic::x86_avx512_pslli_d_512:
3853 case Intrinsic::x86_avx512_pslli_q_512:
3854 case Intrinsic::x86_avx512_psrl_w_512:
3855 case Intrinsic::x86_avx512_psrl_d_512:
3856 case Intrinsic::x86_avx512_psrl_q_512:
3857 case Intrinsic::x86_avx512_psra_w_512:
3858 case Intrinsic::x86_avx512_psra_d_512:
3859 case Intrinsic::x86_avx512_psra_q_512:
3860 case Intrinsic::x86_avx512_psrli_w_512:
3861 case Intrinsic::x86_avx512_psrli_d_512:
3862 case Intrinsic::x86_avx512_psrli_q_512:
3863 case Intrinsic::x86_avx512_psrai_w_512:
3864 case Intrinsic::x86_avx512_psrai_d_512:
3865 case Intrinsic::x86_avx512_psrai_q_512:
3866 case Intrinsic::x86_avx512_psra_q_256:
3867 case Intrinsic::x86_avx512_psra_q_128:
3868 case Intrinsic::x86_avx512_psrai_q_256:
3869 case Intrinsic::x86_avx512_psrai_q_128:
3870 case Intrinsic::x86_avx2_psll_w:
3871 case Intrinsic::x86_avx2_psll_d:
3872 case Intrinsic::x86_avx2_psll_q:
3873 case Intrinsic::x86_avx2_pslli_w:
3874 case Intrinsic::x86_avx2_pslli_d:
3875 case Intrinsic::x86_avx2_pslli_q:
3876 case Intrinsic::x86_avx2_psrl_w:
3877 case Intrinsic::x86_avx2_psrl_d:
3878 case Intrinsic::x86_avx2_psrl_q:
3879 case Intrinsic::x86_avx2_psra_w:
3880 case Intrinsic::x86_avx2_psra_d:
3881 case Intrinsic::x86_avx2_psrli_w:
3882 case Intrinsic::x86_avx2_psrli_d:
3883 case Intrinsic::x86_avx2_psrli_q:
3884 case Intrinsic::x86_avx2_psrai_w:
3885 case Intrinsic::x86_avx2_psrai_d:
3886 case Intrinsic::x86_sse2_psll_w:
3887 case Intrinsic::x86_sse2_psll_d:
3888 case Intrinsic::x86_sse2_psll_q:
3889 case Intrinsic::x86_sse2_pslli_w:
3890 case Intrinsic::x86_sse2_pslli_d:
3891 case Intrinsic::x86_sse2_pslli_q:
3892 case Intrinsic::x86_sse2_psrl_w:
3893 case Intrinsic::x86_sse2_psrl_d:
3894 case Intrinsic::x86_sse2_psrl_q:
3895 case Intrinsic::x86_sse2_psra_w:
3896 case Intrinsic::x86_sse2_psra_d:
3897 case Intrinsic::x86_sse2_psrli_w:
3898 case Intrinsic::x86_sse2_psrli_d:
3899 case Intrinsic::x86_sse2_psrli_q:
3900 case Intrinsic::x86_sse2_psrai_w:
3901 case Intrinsic::x86_sse2_psrai_d:
3902 case Intrinsic::x86_mmx_psll_w:
3903 case Intrinsic::x86_mmx_psll_d:
3904 case Intrinsic::x86_mmx_psll_q:
3905 case Intrinsic::x86_mmx_pslli_w:
3906 case Intrinsic::x86_mmx_pslli_d:
3907 case Intrinsic::x86_mmx_pslli_q:
3908 case Intrinsic::x86_mmx_psrl_w:
3909 case Intrinsic::x86_mmx_psrl_d:
3910 case Intrinsic::x86_mmx_psrl_q:
3911 case Intrinsic::x86_mmx_psra_w:
3912 case Intrinsic::x86_mmx_psra_d:
3913 case Intrinsic::x86_mmx_psrli_w:
3914 case Intrinsic::x86_mmx_psrli_d:
3915 case Intrinsic::x86_mmx_psrli_q:
3916 case Intrinsic::x86_mmx_psrai_w:
3917 case Intrinsic::x86_mmx_psrai_d:
3918 handleVectorShiftIntrinsic(
I,
false);
3920 case Intrinsic::x86_avx2_psllv_d:
3921 case Intrinsic::x86_avx2_psllv_d_256:
3922 case Intrinsic::x86_avx512_psllv_d_512:
3923 case Intrinsic::x86_avx2_psllv_q:
3924 case Intrinsic::x86_avx2_psllv_q_256:
3925 case Intrinsic::x86_avx512_psllv_q_512:
3926 case Intrinsic::x86_avx2_psrlv_d:
3927 case Intrinsic::x86_avx2_psrlv_d_256:
3928 case Intrinsic::x86_avx512_psrlv_d_512:
3929 case Intrinsic::x86_avx2_psrlv_q:
3930 case Intrinsic::x86_avx2_psrlv_q_256:
3931 case Intrinsic::x86_avx512_psrlv_q_512:
3932 case Intrinsic::x86_avx2_psrav_d:
3933 case Intrinsic::x86_avx2_psrav_d_256:
3934 case Intrinsic::x86_avx512_psrav_d_512:
3935 case Intrinsic::x86_avx512_psrav_q_128:
3936 case Intrinsic::x86_avx512_psrav_q_256:
3937 case Intrinsic::x86_avx512_psrav_q_512:
3938 handleVectorShiftIntrinsic(
I,
true);
3941 case Intrinsic::x86_sse2_packsswb_128:
3942 case Intrinsic::x86_sse2_packssdw_128:
3943 case Intrinsic::x86_sse2_packuswb_128:
3944 case Intrinsic::x86_sse41_packusdw:
3945 case Intrinsic::x86_avx2_packsswb:
3946 case Intrinsic::x86_avx2_packssdw:
3947 case Intrinsic::x86_avx2_packuswb:
3948 case Intrinsic::x86_avx2_packusdw:
3949 handleVectorPackIntrinsic(
I);
3952 case Intrinsic::x86_mmx_packsswb:
3953 case Intrinsic::x86_mmx_packuswb:
3954 handleVectorPackIntrinsic(
I, 16);
3957 case Intrinsic::x86_mmx_packssdw:
3958 handleVectorPackIntrinsic(
I, 32);
3961 case Intrinsic::x86_mmx_psad_bw:
3962 case Intrinsic::x86_sse2_psad_bw:
3963 case Intrinsic::x86_avx2_psad_bw:
3964 handleVectorSadIntrinsic(
I);
3967 case Intrinsic::x86_sse2_pmadd_wd:
3968 case Intrinsic::x86_avx2_pmadd_wd:
3969 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3970 case Intrinsic::x86_avx2_pmadd_ub_sw:
3971 handleVectorPmaddIntrinsic(
I);
3974 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3975 handleVectorPmaddIntrinsic(
I, 8);
3978 case Intrinsic::x86_mmx_pmadd_wd:
3979 handleVectorPmaddIntrinsic(
I, 16);
3982 case Intrinsic::x86_sse_cmp_ss:
3983 case Intrinsic::x86_sse2_cmp_sd:
3984 case Intrinsic::x86_sse_comieq_ss:
3985 case Intrinsic::x86_sse_comilt_ss:
3986 case Intrinsic::x86_sse_comile_ss:
3987 case Intrinsic::x86_sse_comigt_ss:
3988 case Intrinsic::x86_sse_comige_ss:
3989 case Intrinsic::x86_sse_comineq_ss:
3990 case Intrinsic::x86_sse_ucomieq_ss:
3991 case Intrinsic::x86_sse_ucomilt_ss:
3992 case Intrinsic::x86_sse_ucomile_ss:
3993 case Intrinsic::x86_sse_ucomigt_ss:
3994 case Intrinsic::x86_sse_ucomige_ss:
3995 case Intrinsic::x86_sse_ucomineq_ss:
3996 case Intrinsic::x86_sse2_comieq_sd:
3997 case Intrinsic::x86_sse2_comilt_sd:
3998 case Intrinsic::x86_sse2_comile_sd:
3999 case Intrinsic::x86_sse2_comigt_sd:
4000 case Intrinsic::x86_sse2_comige_sd:
4001 case Intrinsic::x86_sse2_comineq_sd:
4002 case Intrinsic::x86_sse2_ucomieq_sd:
4003 case Intrinsic::x86_sse2_ucomilt_sd:
4004 case Intrinsic::x86_sse2_ucomile_sd:
4005 case Intrinsic::x86_sse2_ucomigt_sd:
4006 case Intrinsic::x86_sse2_ucomige_sd:
4007 case Intrinsic::x86_sse2_ucomineq_sd:
4008 handleVectorCompareScalarIntrinsic(
I);
4011 case Intrinsic::x86_avx_cmp_pd_256:
4012 case Intrinsic::x86_avx_cmp_ps_256:
4013 case Intrinsic::x86_sse2_cmp_pd:
4014 case Intrinsic::x86_sse_cmp_ps:
4015 handleVectorComparePackedIntrinsic(
I);
4018 case Intrinsic::x86_bmi_bextr_32:
4019 case Intrinsic::x86_bmi_bextr_64:
4020 case Intrinsic::x86_bmi_bzhi_32:
4021 case Intrinsic::x86_bmi_bzhi_64:
4022 case Intrinsic::x86_bmi_pdep_32:
4023 case Intrinsic::x86_bmi_pdep_64:
4024 case Intrinsic::x86_bmi_pext_32:
4025 case Intrinsic::x86_bmi_pext_64:
4026 handleBmiIntrinsic(
I);
4029 case Intrinsic::x86_pclmulqdq:
4030 case Intrinsic::x86_pclmulqdq_256:
4031 case Intrinsic::x86_pclmulqdq_512:
4032 handlePclmulIntrinsic(
I);
4035 case Intrinsic::x86_sse41_round_sd:
4036 case Intrinsic::x86_sse41_round_ss:
4037 handleUnarySdSsIntrinsic(
I);
4039 case Intrinsic::x86_sse2_max_sd:
4040 case Intrinsic::x86_sse_max_ss:
4041 case Intrinsic::x86_sse2_min_sd:
4042 case Intrinsic::x86_sse_min_ss:
4043 handleBinarySdSsIntrinsic(
I);
4046 case Intrinsic::x86_avx_vtestc_pd:
4047 case Intrinsic::x86_avx_vtestc_pd_256:
4048 case Intrinsic::x86_avx_vtestc_ps:
4049 case Intrinsic::x86_avx_vtestc_ps_256:
4050 case Intrinsic::x86_avx_vtestnzc_pd:
4051 case Intrinsic::x86_avx_vtestnzc_pd_256:
4052 case Intrinsic::x86_avx_vtestnzc_ps:
4053 case Intrinsic::x86_avx_vtestnzc_ps_256:
4054 case Intrinsic::x86_avx_vtestz_pd:
4055 case Intrinsic::x86_avx_vtestz_pd_256:
4056 case Intrinsic::x86_avx_vtestz_ps:
4057 case Intrinsic::x86_avx_vtestz_ps_256:
4058 case Intrinsic::x86_avx_ptestc_256:
4059 case Intrinsic::x86_avx_ptestnzc_256:
4060 case Intrinsic::x86_avx_ptestz_256:
4061 case Intrinsic::x86_sse41_ptestc:
4062 case Intrinsic::x86_sse41_ptestnzc:
4063 case Intrinsic::x86_sse41_ptestz:
4064 handleVtestIntrinsic(
I);
4067 case Intrinsic::fshl:
4068 case Intrinsic::fshr:
4069 handleFunnelShift(
I);
4072 case Intrinsic::is_constant:
4074 setShadow(&
I, getCleanShadow(&
I));
4075 setOrigin(&
I, getCleanOrigin());
4079 if (!handleUnknownIntrinsic(
I))
4080 visitInstruction(
I);
4085 void visitLibAtomicLoad(
CallBase &CB) {
4087 assert(isa<CallInst>(CB));
4096 Value *NewOrdering =
4100 NextNodeIRBuilder NextIRB(&CB);
4101 Value *SrcShadowPtr, *SrcOriginPtr;
4102 std::tie(SrcShadowPtr, SrcOriginPtr) =
4103 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4105 Value *DstShadowPtr =
4106 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4110 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4111 if (MS.TrackOrigins) {
4112 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4114 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4115 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4119 void visitLibAtomicStore(
CallBase &CB) {
4126 Value *NewOrdering =
4130 Value *DstShadowPtr =
4148 visitAsmInstruction(CB);
4150 visitInstruction(CB);
4159 case LibFunc_atomic_load:
4160 if (!isa<CallInst>(CB)) {
4161 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4165 visitLibAtomicLoad(CB);
4167 case LibFunc_atomic_store:
4168 visitLibAtomicStore(CB);
4175 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4176 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4184 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4186 Call->removeFnAttrs(
B);
4188 Func->removeFnAttrs(
B);
4194 bool MayCheckCall = MS.EagerChecks;
4198 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4201 unsigned ArgOffset = 0;
4204 if (!
A->getType()->isSized()) {
4205 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4209 if (
A->getType()->isScalableTy()) {
4210 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is vscale: " << CB <<
"\n");
4212 insertShadowCheck(
A, &CB);
4221 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4224 insertShadowCheck(
A, &CB);
4225 Size =
DL.getTypeAllocSize(
A->getType());
4231 Value *ArgShadow = getShadow(
A);
4232 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4234 <<
" Shadow: " << *ArgShadow <<
"\n");
4238 assert(
A->getType()->isPointerTy() &&
4239 "ByVal argument is not a pointer!");
4247 Value *AShadowPtr, *AOriginPtr;
4248 std::tie(AShadowPtr, AOriginPtr) =
4249 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4251 if (!PropagateShadow) {
4258 if (MS.TrackOrigins) {
4259 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4273 Size =
DL.getTypeAllocSize(
A->getType());
4278 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4279 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4281 getOriginPtrForArgument(IRB, ArgOffset));
4285 assert(Store !=
nullptr);
4294 if (FT->isVarArg()) {
4295 VAHelper->visitCallBase(CB, IRB);
4302 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4305 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4306 setShadow(&CB, getCleanShadow(&CB));
4307 setOrigin(&CB, getCleanOrigin());
4313 Value *
Base = getShadowPtrForRetval(IRBBefore);
4314 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4317 if (isa<CallInst>(CB)) {
4321 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4326 setShadow(&CB, getCleanShadow(&CB));
4327 setOrigin(&CB, getCleanOrigin());
4334 "Could not find insertion point for retval shadow load");
4337 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4338 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4340 setShadow(&CB, RetvalShadow);
4341 if (MS.TrackOrigins)
4342 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4343 getOriginPtrForRetval()));
4347 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4348 RetVal =
I->getOperand(0);
4350 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4351 return I->isMustTailCall();
4358 Value *RetVal =
I.getReturnValue();
4364 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4365 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4366 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4369 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4371 Value *Shadow = getShadow(RetVal);
4372 bool StoreOrigin =
true;
4374 insertShadowCheck(RetVal, &
I);
4375 Shadow = getCleanShadow(RetVal);
4376 StoreOrigin =
false;
4383 if (MS.TrackOrigins && StoreOrigin)
4384 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4390 if (!PropagateShadow) {
4391 setShadow(&
I, getCleanShadow(&
I));
4392 setOrigin(&
I, getCleanOrigin());
4396 ShadowPHINodes.push_back(&
I);
4397 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4399 if (MS.TrackOrigins)
4401 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4418 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4420 Value *ShadowBase, *OriginBase;
4421 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4428 if (PoisonStack && MS.TrackOrigins) {
4429 Value *Idptr = getLocalVarIdptr(
I);
4431 Value *Descr = getLocalVarDescription(
I);
4432 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4433 {&I, Len, Idptr, Descr});
4435 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4441 Value *Descr = getLocalVarDescription(
I);
4443 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4445 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4452 NextNodeIRBuilder IRB(InsPoint);
4456 if (
I.isArrayAllocation())
4460 if (MS.CompileKernel)
4461 poisonAllocaKmsan(
I, IRB, Len);
4463 poisonAllocaUserspace(
I, IRB, Len);
4467 setShadow(&
I, getCleanShadow(&
I));
4468 setOrigin(&
I, getCleanOrigin());
4480 Value *Sb = getShadow(
B);
4481 Value *Sc = getShadow(
C);
4482 Value *Sd = getShadow(
D);
4487 if (
I.getType()->isAggregateType()) {
4491 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4499 C = CreateAppToShadowCast(IRB,
C);
4500 D = CreateAppToShadowCast(IRB,
D);
4507 if (MS.TrackOrigins) {
4510 if (
B->getType()->isVectorTy()) {
4511 B = convertToBool(
B, IRB);
4512 Sb = convertToBool(Sb, IRB);
4519 getOrigin(
I.getFalseValue()))));
4526 setShadow(&
I, getCleanShadow(&
I));
4527 setOrigin(&
I, getCleanOrigin());
4531 setShadow(&
I, getCleanShadow(&
I));
4532 setOrigin(&
I, getCleanOrigin());
4536 setShadow(&
I, getCleanShadow(&
I));
4537 setOrigin(&
I, getCleanOrigin());
4544 Value *Agg =
I.getAggregateOperand();
4546 Value *AggShadow = getShadow(Agg);
4550 setShadow(&
I, ResShadow);
4551 setOriginForNaryOp(
I);
4557 Value *AggShadow = getShadow(
I.getAggregateOperand());
4558 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4564 setOriginForNaryOp(
I);
4568 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4569 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4571 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4573 errs() <<
"QQQ " <<
I <<
"\n";
4600 insertShadowCheck(Operand, &
I);
4607 auto Size =
DL.getTypeStoreSize(ElemTy);
4609 if (MS.CompileKernel) {
4610 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4616 auto [ShadowPtr,
_] =
4617 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4628 int NumRetOutputs = 0;
4630 Type *
RetTy = cast<Value>(CB)->getType();
4631 if (!
RetTy->isVoidTy()) {
4633 auto *
ST = dyn_cast<StructType>(
RetTy);
4635 NumRetOutputs =
ST->getNumElements();
4641 switch (
Info.Type) {
4649 return NumOutputs - NumRetOutputs;
4672 int OutputArgs = getNumOutputArgs(IA, CB);
4678 for (
int i = OutputArgs; i < NumOperands; i++) {
4686 for (
int i = 0; i < OutputArgs; i++) {
4692 setShadow(&
I, getCleanShadow(&
I));
4693 setOrigin(&
I, getCleanOrigin());
4698 setShadow(&
I, getCleanShadow(&
I));
4699 setOrigin(&
I, getCleanOrigin());
4707 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4708 Value *Operand =
I.getOperand(i);
4710 insertShadowCheck(Operand, &
I);
4712 setShadow(&
I, getCleanShadow(&
I));
4713 setOrigin(&
I, getCleanOrigin());
4717struct VarArgHelperBase :
public VarArgHelper {
4719 MemorySanitizer &MS;
4720 MemorySanitizerVisitor &MSV;
4722 const unsigned VAListTagSize;
4724 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4725 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4726 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4730 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4735 unsigned ArgOffset) {
4744 unsigned ArgOffset,
unsigned ArgSize) {
4748 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4763 unsigned BaseOffset) {
4772 TailSize,
Align(8));
4777 Value *VAListTag =
I.getArgOperand(0);
4779 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4780 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4783 VAListTagSize, Alignment,
false);
4790 unpoisonVAListTagForInst(
I);
4796 unpoisonVAListTagForInst(
I);
4801struct VarArgAMD64Helper :
public VarArgHelperBase {
4804 static const unsigned AMD64GpEndOffset = 48;
4805 static const unsigned AMD64FpEndOffsetSSE = 176;
4807 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4809 unsigned AMD64FpEndOffset;
4812 Value *VAArgOverflowSize =
nullptr;
4814 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4816 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4817 MemorySanitizerVisitor &MSV)
4818 : VarArgHelperBase(
F, MS, MSV, 24) {
4819 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4820 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4821 if (Attr.isStringAttribute() &&
4822 (Attr.getKindAsString() ==
"target-features")) {
4823 if (Attr.getValueAsString().contains(
"-sse"))
4824 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4830 ArgKind classifyArgument(
Value *arg) {
4833 if (
T->isX86_FP80Ty())
4835 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4836 return AK_FloatingPoint;
4837 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4838 return AK_GeneralPurpose;
4839 if (
T->isPointerTy())
4840 return AK_GeneralPurpose;
4853 unsigned GpOffset = 0;
4854 unsigned FpOffset = AMD64GpEndOffset;
4855 unsigned OverflowOffset = AMD64FpEndOffset;
4860 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4867 assert(
A->getType()->isPointerTy());
4869 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4871 unsigned BaseOffset = OverflowOffset;
4873 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
4874 Value *OriginBase =
nullptr;
4875 if (MS.TrackOrigins)
4876 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4877 OverflowOffset += AlignedSize;
4880 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4884 Value *ShadowPtr, *OriginPtr;
4885 std::tie(ShadowPtr, OriginPtr) =
4890 if (MS.TrackOrigins)
4894 ArgKind AK = classifyArgument(
A);
4895 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4897 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4899 Value *ShadowBase, *OriginBase =
nullptr;
4901 case AK_GeneralPurpose:
4902 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
4903 if (MS.TrackOrigins)
4904 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
4908 case AK_FloatingPoint:
4909 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
4910 if (MS.TrackOrigins)
4911 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
4918 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4920 unsigned BaseOffset = OverflowOffset;
4922 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4923 if (MS.TrackOrigins) {
4924 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4926 OverflowOffset += AlignedSize;
4929 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4938 Value *Shadow = MSV.getShadow(
A);
4940 if (MS.TrackOrigins) {
4941 Value *Origin = MSV.getOrigin(
A);
4943 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4949 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4950 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4953 void finalizeInstrumentation()
override {
4954 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4955 "finalizeInstrumentation called twice");
4956 if (!VAStartInstrumentationList.
empty()) {
4963 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4970 Intrinsic::umin, CopySize,
4974 if (MS.TrackOrigins) {
4984 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4985 CallInst *OrigInst = VAStartInstrumentationList[i];
4986 NextNodeIRBuilder IRB(OrigInst);
4989 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
4992 ConstantInt::get(MS.IntptrTy, 16)),
4993 PointerType::get(RegSaveAreaPtrTy, 0));
4994 Value *RegSaveAreaPtr =
4995 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4996 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4998 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4999 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5001 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5003 if (MS.TrackOrigins)
5004 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5005 Alignment, AMD64FpEndOffset);
5006 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5009 ConstantInt::get(MS.IntptrTy, 8)),
5010 PointerType::get(OverflowArgAreaPtrTy, 0));
5011 Value *OverflowArgAreaPtr =
5012 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5013 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5014 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5015 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5019 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5021 if (MS.TrackOrigins) {
5024 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5033struct VarArgMIPS64Helper :
public VarArgHelperBase {
5035 Value *VAArgSize =
nullptr;
5037 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5038 MemorySanitizerVisitor &MSV)
5039 : VarArgHelperBase(
F, MS, MSV, 8) {}
5042 unsigned VAArgOffset = 0;
5046 Triple TargetTriple(
F.getParent()->getTargetTriple());
5048 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5053 VAArgOffset += (8 - ArgSize);
5055 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5056 VAArgOffset += ArgSize;
5057 VAArgOffset =
alignTo(VAArgOffset, 8);
5066 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5069 void finalizeInstrumentation()
override {
5070 assert(!VAArgSize && !VAArgTLSCopy &&
5071 "finalizeInstrumentation called twice");
5075 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5077 if (!VAStartInstrumentationList.
empty()) {
5086 Intrinsic::umin, CopySize,
5094 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5095 CallInst *OrigInst = VAStartInstrumentationList[i];
5096 NextNodeIRBuilder IRB(OrigInst);
5098 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5099 Value *RegSaveAreaPtrPtr =
5101 PointerType::get(RegSaveAreaPtrTy, 0));
5102 Value *RegSaveAreaPtr =
5103 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5104 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5106 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5107 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5109 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5116struct VarArgAArch64Helper :
public VarArgHelperBase {
5117 static const unsigned kAArch64GrArgSize = 64;
5118 static const unsigned kAArch64VrArgSize = 128;
5120 static const unsigned AArch64GrBegOffset = 0;
5121 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5123 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5124 static const unsigned AArch64VrEndOffset =
5125 AArch64VrBegOffset + kAArch64VrArgSize;
5126 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5129 Value *VAArgOverflowSize =
nullptr;
5131 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5133 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5134 MemorySanitizerVisitor &MSV)
5135 : VarArgHelperBase(
F, MS, MSV, 32) {}
5138 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5139 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5140 return {AK_GeneralPurpose, 1};
5141 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5142 return {AK_FloatingPoint, 1};
5144 if (
T->isArrayTy()) {
5145 auto R = classifyArgument(
T->getArrayElementType());
5146 R.second *=
T->getScalarType()->getArrayNumElements();
5151 auto R = classifyArgument(FV->getScalarType());
5152 R.second *= FV->getNumElements();
5157 return {AK_Memory, 0};
5170 unsigned GrOffset = AArch64GrBegOffset;
5171 unsigned VrOffset = AArch64VrBegOffset;
5172 unsigned OverflowOffset = AArch64VAEndOffset;
5177 auto [AK, RegNum] = classifyArgument(
A->getType());
5178 if (AK == AK_GeneralPurpose &&
5179 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5181 if (AK == AK_FloatingPoint &&
5182 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5186 case AK_GeneralPurpose:
5187 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5188 GrOffset += 8 * RegNum;
5190 case AK_FloatingPoint:
5191 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5192 VrOffset += 16 * RegNum;
5199 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5201 unsigned BaseOffset = OverflowOffset;
5202 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5203 OverflowOffset += AlignedSize;
5206 CleanUnusedTLS(IRB,
Base, BaseOffset);
5218 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5219 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5226 ConstantInt::get(MS.IntptrTy, offset)),
5227 PointerType::get(*MS.C, 0));
5235 ConstantInt::get(MS.IntptrTy, offset)),
5236 PointerType::get(*MS.C, 0));
5238 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5241 void finalizeInstrumentation()
override {
5242 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5243 "finalizeInstrumentation called twice");
5244 if (!VAStartInstrumentationList.
empty()) {
5251 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5258 Intrinsic::umin, CopySize,
5264 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5265 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5269 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5270 CallInst *OrigInst = VAStartInstrumentationList[i];
5271 NextNodeIRBuilder IRB(OrigInst);
5290 Value *StackSaveAreaPtr =
5291 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5294 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5295 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5298 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5301 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5302 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5305 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5311 Value *GrRegSaveAreaShadowPtrOff =
5312 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5314 Value *GrRegSaveAreaShadowPtr =
5315 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5321 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5327 Value *VrRegSaveAreaShadowPtrOff =
5328 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5330 Value *VrRegSaveAreaShadowPtr =
5331 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5338 VrRegSaveAreaShadowPtrOff);
5339 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5345 Value *StackSaveAreaShadowPtr =
5346 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5351 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5354 Align(16), VAArgOverflowSize);
5360struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5362 Value *VAArgSize =
nullptr;
5364 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5365 MemorySanitizerVisitor &MSV)
5366 : VarArgHelperBase(
F, MS, MSV, 8) {}
5376 Triple TargetTriple(
F.getParent()->getTargetTriple());
5384 unsigned VAArgOffset = VAArgBase;
5388 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5390 assert(
A->getType()->isPointerTy());
5392 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5395 ArgAlign =
Align(8);
5396 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5398 Value *
Base = getShadowPtrForVAArgument(
5399 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5401 Value *AShadowPtr, *AOriginPtr;
5402 std::tie(AShadowPtr, AOriginPtr) =
5403 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5413 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5415 if (
A->getType()->isArrayTy()) {
5418 Type *ElementTy =
A->getType()->getArrayElementType();
5420 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5421 }
else if (
A->getType()->isVectorTy()) {
5423 ArgAlign =
Align(ArgSize);
5426 ArgAlign =
Align(8);
5427 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5428 if (
DL.isBigEndian()) {
5432 VAArgOffset += (8 - ArgSize);
5435 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5436 VAArgOffset - VAArgBase, ArgSize);
5440 VAArgOffset += ArgSize;
5444 VAArgBase = VAArgOffset;
5448 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5451 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5454 void finalizeInstrumentation()
override {
5455 assert(!VAArgSize && !VAArgTLSCopy &&
5456 "finalizeInstrumentation called twice");
5460 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5462 if (!VAStartInstrumentationList.
empty()) {
5472 Intrinsic::umin, CopySize,
5480 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5481 CallInst *OrigInst = VAStartInstrumentationList[i];
5482 NextNodeIRBuilder IRB(OrigInst);
5484 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5485 Value *RegSaveAreaPtrPtr =
5487 PointerType::get(RegSaveAreaPtrTy, 0));
5488 Value *RegSaveAreaPtr =
5489 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5490 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5492 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5493 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5495 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5502struct VarArgSystemZHelper :
public VarArgHelperBase {
5503 static const unsigned SystemZGpOffset = 16;
5504 static const unsigned SystemZGpEndOffset = 56;
5505 static const unsigned SystemZFpOffset = 128;
5506 static const unsigned SystemZFpEndOffset = 160;
5507 static const unsigned SystemZMaxVrArgs = 8;
5508 static const unsigned SystemZRegSaveAreaSize = 160;
5509 static const unsigned SystemZOverflowOffset = 160;
5510 static const unsigned SystemZVAListTagSize = 32;
5511 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5512 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5514 bool IsSoftFloatABI;
5517 Value *VAArgOverflowSize =
nullptr;
5519 enum class ArgKind {
5527 enum class ShadowExtension {
None,
Zero, Sign };
5529 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5530 MemorySanitizerVisitor &MSV)
5531 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5532 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5534 ArgKind classifyArgument(
Type *
T) {
5541 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5542 return ArgKind::Indirect;
5543 if (
T->isFloatingPointTy())
5544 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5545 if (
T->isIntegerTy() ||
T->isPointerTy())
5546 return ArgKind::GeneralPurpose;
5547 if (
T->isVectorTy())
5548 return ArgKind::Vector;
5549 return ArgKind::Memory;
5552 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5562 return ShadowExtension::Zero;
5566 return ShadowExtension::Sign;
5568 return ShadowExtension::None;
5572 unsigned GpOffset = SystemZGpOffset;
5573 unsigned FpOffset = SystemZFpOffset;
5574 unsigned VrIndex = 0;
5575 unsigned OverflowOffset = SystemZOverflowOffset;
5582 ArgKind AK = classifyArgument(
T);
5583 if (AK == ArgKind::Indirect) {
5584 T = PointerType::get(
T, 0);
5585 AK = ArgKind::GeneralPurpose;
5587 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5588 AK = ArgKind::Memory;
5589 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5590 AK = ArgKind::Memory;
5591 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5592 AK = ArgKind::Memory;
5593 Value *ShadowBase =
nullptr;
5594 Value *OriginBase =
nullptr;
5595 ShadowExtension SE = ShadowExtension::None;
5597 case ArgKind::GeneralPurpose: {
5602 SE = getShadowExtension(CB, ArgNo);
5604 if (SE == ShadowExtension::None) {
5606 assert(ArgAllocSize <= ArgSize);
5607 GapSize = ArgSize - ArgAllocSize;
5609 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5610 if (MS.TrackOrigins)
5611 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5613 GpOffset += ArgSize;
5619 case ArgKind::FloatingPoint: {
5628 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5629 if (MS.TrackOrigins)
5630 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5632 FpOffset += ArgSize;
5638 case ArgKind::Vector: {
5645 case ArgKind::Memory: {
5653 SE = getShadowExtension(CB, ArgNo);
5655 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5657 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5658 if (MS.TrackOrigins)
5660 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5661 OverflowOffset += ArgSize;
5668 case ArgKind::Indirect:
5671 if (ShadowBase ==
nullptr)
5673 Value *Shadow = MSV.getShadow(
A);
5674 if (SE != ShadowExtension::None)
5675 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5676 SE == ShadowExtension::Sign);
5678 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5680 if (MS.TrackOrigins) {
5681 Value *Origin = MSV.getOrigin(
A);
5683 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5687 Constant *OverflowSize = ConstantInt::get(
5688 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5689 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5693 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5697 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5698 PointerType::get(RegSaveAreaPtrTy, 0));
5699 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5700 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5702 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5703 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5708 unsigned RegSaveAreaSize =
5709 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5710 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5712 if (MS.TrackOrigins)
5713 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5714 Alignment, RegSaveAreaSize);
5720 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5724 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5725 PointerType::get(OverflowArgAreaPtrTy, 0));
5726 Value *OverflowArgAreaPtr =
5727 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5728 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5730 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5731 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5734 SystemZOverflowOffset);
5735 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5737 if (MS.TrackOrigins) {
5739 SystemZOverflowOffset);
5740 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5745 void finalizeInstrumentation()
override {
5746 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5747 "finalizeInstrumentation called twice");
5748 if (!VAStartInstrumentationList.
empty()) {
5755 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5763 Intrinsic::umin, CopySize,
5767 if (MS.TrackOrigins) {
5777 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5778 VaStartNo < VaStartNum; VaStartNo++) {
5779 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5780 NextNodeIRBuilder IRB(OrigInst);
5782 copyRegSaveArea(IRB, VAListTag);
5783 copyOverflowArea(IRB, VAListTag);
5790using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5793struct VarArgNoOpHelper :
public VarArgHelper {
5794 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5795 MemorySanitizerVisitor &MSV) {}
5803 void finalizeInstrumentation()
override {}
5809 MemorySanitizerVisitor &Visitor) {
5812 Triple TargetTriple(Func.getParent()->getTargetTriple());
5814 return new VarArgAMD64Helper(Func, Msan, Visitor);
5816 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5818 return new VarArgAArch64Helper(Func, Msan, Visitor);
5821 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5823 return new VarArgSystemZHelper(Func, Msan, Visitor);
5825 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5827 return new VarArgNoOpHelper(Func, Msan, Visitor);
5834 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5837 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5841 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5844 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.