236#define DEBUG_TYPE "frame-info"
239 cl::desc(
"enable use of redzone on AArch64"),
243 "stack-tagging-merge-settag",
253 cl::desc(
"Emit homogeneous prologue and epilogue for the size "
254 "optimization (default = off)"));
256STATISTIC(NumRedZoneFunctions,
"Number of functions using red zone");
272 int64_t ArgumentPopSize = 0;
273 if (IsTailCallReturn) {
279 ArgumentPopSize = StackAdjust.
getImm();
288 return ArgumentPopSize;
299bool AArch64FrameLowering::homogeneousPrologEpilog(
324 if (AFI->hasSwiftAsyncContext())
331 unsigned NumGPRs = 0;
332 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
334 if (Reg == AArch64::LR) {
335 assert(CSRegs[
I + 1] == AArch64::FP);
336 if (NumGPRs % 2 != 0)
340 if (AArch64::GPR64RegClass.
contains(Reg))
348bool AArch64FrameLowering::producePairRegisters(
MachineFunction &MF)
const {
367 if (
MI.isDebugInstr() ||
MI.isPseudo() ||
368 MI.getOpcode() == AArch64::ADDXri ||
369 MI.getOpcode() == AArch64::ADDSXri)
396 if (!IsWin64 || IsFunclet) {
401 Attribute::SwiftAsync))
406 const unsigned UnwindHelpObject = (MF.
hasEHFunclets() ? 8 : 0);
408 alignTo(VarArgsArea + UnwindHelpObject, 16);
425 const unsigned RedZoneSize =
434 return !(MFI.
hasCalls() ||
hasFP(MF) || NumBytes > RedZoneSize ||
495 unsigned Opc =
I->getOpcode();
496 bool IsDestroy = Opc ==
TII->getCallFrameDestroyOpcode();
497 uint64_t CalleePopAmount = IsDestroy ?
I->getOperand(1).getImm() : 0;
500 int64_t Amount =
I->getOperand(0).getImm();
508 if (CalleePopAmount == 0) {
519 assert(Amount > -0xffffff && Amount < 0xffffff &&
"call frame too large");
530 "non-reserved call frame without var sized objects?");
539 }
else if (CalleePopAmount != 0) {
542 assert(CalleePopAmount < 0xffffff &&
"call frame too large");
549void AArch64FrameLowering::emitCalleeSavedGPRLocations(
563 for (
const auto &Info : CSI) {
567 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
568 unsigned DwarfReg =
TRI.getDwarfRegNum(
Info.getReg(),
true);
580void AArch64FrameLowering::emitCalleeSavedSVELocations(
596 for (
const auto &Info : CSI) {
602 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
637 const MCInstrDesc &CFIDesc =
TII.get(TargetOpcode::CFI_INSTRUCTION);
643 nullptr,
TRI.getDwarfRegNum(AArch64::SP,
true), 0));
647 if (MFI.shouldSignReturnAddress(MF)) {
653 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
655 TRI.getDwarfRegNum(AArch64::X18,
true));
658 const std::vector<CalleeSavedInfo> &CSI =
660 for (
const auto &
Info : CSI) {
661 unsigned Reg =
Info.getReg();
662 if (!
TRI.regNeedsCFI(Reg, Reg))
665 TRI.getDwarfRegNum(Reg,
true));
684 for (
const auto &
Info : CSI) {
689 unsigned Reg =
Info.getReg();
695 nullptr,
TRI.getDwarfRegNum(
Info.getReg(),
true)));
702void AArch64FrameLowering::emitCalleeSavedGPRRestores(
707void AArch64FrameLowering::emitCalleeSavedSVERestores(
715 static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
716 return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE +
Size.getFixed();
719void AArch64FrameLowering::allocateStackSpace(
721 int64_t RealignmentPadding,
StackOffset AllocSize,
bool NeedsWinCFI,
722 bool *HasWinCFI,
bool EmitCFI,
StackOffset InitialOffset,
723 bool FollowupAllocs)
const {
736 const uint64_t AndMask = ~(MaxAlign - 1);
739 Register TargetReg = RealignmentPadding
745 EmitCFI, InitialOffset);
747 if (RealignmentPadding) {
768 if (AllocSize.
getScalable() == 0 && RealignmentPadding == 0) {
770 assert(ScratchReg != AArch64::NoRegister);
780 if (FollowupAllocs) {
797 if (
upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
798 Register ScratchReg = RealignmentPadding
801 assert(ScratchReg != AArch64::NoRegister);
805 EmitCFI, InitialOffset);
806 if (RealignmentPadding) {
814 if (FollowupAllocs ||
upperBound(AllocSize) + RealignmentPadding >
830 assert(TargetReg != AArch64::NoRegister);
834 EmitCFI, InitialOffset);
835 if (RealignmentPadding) {
855 if (RealignmentPadding)
868 case AArch64::W##n: \
869 case AArch64::X##n: \
894 case AArch64::B##n: \
895 case AArch64::H##n: \
896 case AArch64::S##n: \
897 case AArch64::D##n: \
898 case AArch64::Q##n: \
899 return HasSVE ? AArch64::Z##n : AArch64::Q##n
936void AArch64FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
952 bool HasSVE = STI.hasSVE();
954 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
957 GPRsToZero.set(XReg);
958 }
else if (AArch64::FPR128RegClass.
contains(Reg) ||
959 AArch64::FPR64RegClass.
contains(Reg) ||
960 AArch64::FPR32RegClass.
contains(Reg) ||
961 AArch64::FPR16RegClass.
contains(Reg) ||
962 AArch64::FPR8RegClass.
contains(Reg)) {
965 FPRsToZero.set(XReg);
981 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
982 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
983 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
985 if (RegsToZero[PReg])
997 for (
unsigned i = 0; CSRegs[i]; ++i)
998 LiveRegs.
addReg(CSRegs[i]);
1029 for (
unsigned Reg : AArch64::GPR64RegClass) {
1033 return AArch64::NoRegister;
1079 StackSizeInBytes >=
uint64_t(MFI.getStackProbeSize());
1085 F.needsUnwindTableEntry();
1088bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1094 if (homogeneousPrologEpilog(MF))
1117 if (MFI.hasVarSizedObjects())
1120 if (
RegInfo->hasStackRealignment(MF))
1137bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
1139 if (!shouldCombineCSRLocalStackBump(*
MBB.
getParent(), StackBumpBytes))
1149 while (LastI != Begin) {
1151 if (LastI->isTransient())
1156 switch (LastI->getOpcode()) {
1157 case AArch64::STGloop:
1158 case AArch64::STZGloop:
1160 case AArch64::STZGi:
1161 case AArch64::ST2Gi:
1162 case AArch64::STZ2Gi:
1175 unsigned Opc =
MBBI->getOpcode();
1179 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1180 int Imm =
MBBI->getOperand(ImmIdx).getImm();
1188 case AArch64::LDPDpost:
1191 case AArch64::STPDpre: {
1192 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1193 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1194 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFRegP_X))
1201 case AArch64::LDPXpost:
1204 case AArch64::STPXpre: {
1207 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1208 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFPLR_X))
1212 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveRegP_X))
1219 case AArch64::LDRDpost:
1222 case AArch64::STRDpre: {
1223 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1224 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFReg_X))
1230 case AArch64::LDRXpost:
1233 case AArch64::STRXpre: {
1234 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1241 case AArch64::STPDi:
1242 case AArch64::LDPDi: {
1243 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1244 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1252 case AArch64::STPXi:
1253 case AArch64::LDPXi: {
1256 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1268 case AArch64::STRXui:
1269 case AArch64::LDRXui: {
1270 int Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1277 case AArch64::STRDui:
1278 case AArch64::LDRDui: {
1279 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1286 case AArch64::STPQi:
1287 case AArch64::LDPQi: {
1288 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1289 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1290 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQP))
1297 case AArch64::LDPQpost:
1300 case AArch64::STPQpre: {
1301 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1302 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1303 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQPX))
1317 unsigned LocalStackSize) {
1319 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1320 switch (
MBBI->getOpcode()) {
1323 case AArch64::SEH_SaveFPLR:
1324 case AArch64::SEH_SaveRegP:
1325 case AArch64::SEH_SaveReg:
1326 case AArch64::SEH_SaveFRegP:
1327 case AArch64::SEH_SaveFReg:
1328 case AArch64::SEH_SaveAnyRegQP:
1329 case AArch64::SEH_SaveAnyRegQPX:
1330 ImmOpnd = &
MBBI->getOperand(ImmIdx);
1343 bool NeedsWinCFI,
bool *HasWinCFI,
bool EmitCFI,
1345 int CFAOffset = 0) {
1347 switch (
MBBI->getOpcode()) {
1350 case AArch64::STPXi:
1351 NewOpc = AArch64::STPXpre;
1353 case AArch64::STPDi:
1354 NewOpc = AArch64::STPDpre;
1356 case AArch64::STPQi:
1357 NewOpc = AArch64::STPQpre;
1359 case AArch64::STRXui:
1360 NewOpc = AArch64::STRXpre;
1362 case AArch64::STRDui:
1363 NewOpc = AArch64::STRDpre;
1365 case AArch64::STRQui:
1366 NewOpc = AArch64::STRQpre;
1368 case AArch64::LDPXi:
1369 NewOpc = AArch64::LDPXpost;
1371 case AArch64::LDPDi:
1372 NewOpc = AArch64::LDPDpost;
1374 case AArch64::LDPQi:
1375 NewOpc = AArch64::LDPQpost;
1377 case AArch64::LDRXui:
1378 NewOpc = AArch64::LDRXpost;
1380 case AArch64::LDRDui:
1381 NewOpc = AArch64::LDRDpost;
1383 case AArch64::LDRQui:
1384 NewOpc = AArch64::LDRQpost;
1389 auto SEH = std::next(
MBBI);
1391 SEH->eraseFromParent();
1395 int64_t MinOffset, MaxOffset;
1397 NewOpc, Scale, Width, MinOffset, MaxOffset);
1404 if (
MBBI->getOperand(
MBBI->getNumOperands() - 1).getImm() != 0 ||
1405 CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1408 false,
false,
nullptr, EmitCFI,
1411 return std::prev(
MBBI);
1418 unsigned OpndIdx = 0;
1419 for (
unsigned OpndEnd =
MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1421 MIB.
add(
MBBI->getOperand(OpndIdx));
1423 assert(
MBBI->getOperand(OpndIdx).getImm() == 0 &&
1424 "Unexpected immediate offset in first/last callee-save save/restore "
1426 assert(
MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1427 "Unexpected base register in callee-save save/restore instruction!");
1428 assert(CSStackSizeInc % Scale == 0);
1429 MIB.
addImm(CSStackSizeInc / (
int)Scale);
1460 unsigned Opc =
MI.getOpcode();
1463 case AArch64::STPXi:
1464 case AArch64::STRXui:
1465 case AArch64::STPDi:
1466 case AArch64::STRDui:
1467 case AArch64::LDPXi:
1468 case AArch64::LDRXui:
1469 case AArch64::LDPDi:
1470 case AArch64::LDRDui:
1473 case AArch64::STPQi:
1474 case AArch64::STRQui:
1475 case AArch64::LDPQi:
1476 case AArch64::LDRQui:
1483 unsigned OffsetIdx =
MI.getNumExplicitOperands() - 1;
1484 assert(
MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1485 "Unexpected base register in callee-save save/restore instruction!");
1489 assert(LocalStackSize % Scale == 0);
1490 OffsetOpnd.
setImm(OffsetOpnd.
getImm() + LocalStackSize / Scale);
1495 assert(
MBBI !=
MI.getParent()->end() &&
"Expecting a valid instruction");
1497 "Expecting a SEH instruction");
1508 switch (
I->getOpcode()) {
1511 case AArch64::PTRUE_C_B:
1512 case AArch64::LD1B_2Z_IMM:
1513 case AArch64::ST1B_2Z_IMM:
1514 case AArch64::STR_ZXI:
1515 case AArch64::STR_PXI:
1516 case AArch64::LDR_ZXI:
1517 case AArch64::LDR_PXI:
1528 bool NeedsUnwindInfo) {
1544 if (NeedsUnwindInfo) {
1547 static const char CFIInst[] = {
1548 dwarf::DW_CFA_val_expression,
1551 static_cast<char>(
unsigned(dwarf::DW_OP_breg18)),
1552 static_cast<char>(-8) & 0x7f,
1555 nullptr,
StringRef(CFIInst,
sizeof(CFIInst))));
1593 const int OffsetToFirstCalleeSaveFromFP =
1597 unsigned Reg =
TRI->getDwarfRegNum(
FramePtr,
true);
1599 nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1632 bool HasFP =
hasFP(MF);
1634 bool HasWinCFI =
false;
1643 while (NonFrameStart !=
End &&
1648 if (NonFrameStart !=
MBB.
end()) {
1658 if (NonFrameStart ==
MBB.
end())
1663 for (auto &Op : MI.operands())
1664 if (Op.isReg() && Op.isDef())
1665 assert(!LiveRegs.contains(Op.getReg()) &&
1666 "live register clobbered by inserted prologue instructions");
1683 if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
1685 MFnI.needsDwarfUnwindInfo(MF));
1687 if (MFnI.shouldSignReturnAddress(MF)) {
1694 if (EmitCFI && MFnI.isMTETagged()) {
1772 assert(!HasFP &&
"unexpected function without stack frame but with FP");
1774 "unexpected function without stack frame but with SVE objects");
1783 ++NumRedZoneFunctions;
1816 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1817 bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1818 if (CombineSPBump) {
1819 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
1825 }
else if (HomPrologEpilog) {
1827 NumBytes -= PrologueSaveSize;
1828 }
else if (PrologueSaveSize != 0) {
1830 MBB,
MBBI,
DL,
TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
1832 NumBytes -= PrologueSaveSize;
1834 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
1843 NeedsWinCFI, &HasWinCFI);
1848 if (!IsFunclet && HasFP) {
1860 bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1861 if (HaveInitialContext)
1863 Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1879 if (HomPrologEpilog) {
1892 if (NeedsWinCFI && HasWinCFI) {
1897 NeedsWinCFI =
false;
1908 emitCalleeSavedGPRLocations(
MBB,
MBBI);
1911 const bool NeedsRealignment =
1912 NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
1913 const int64_t RealignmentPadding =
1919 uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
1927 if (NumBytes >= (1 << 28))
1929 "unwinding purposes");
1931 uint32_t LowNumWords = NumWords & 0xFFFF;
1938 if ((NumWords & 0xFFFF0000) != 0) {
1941 .
addImm((NumWords & 0xFFFF0000) >> 16)
2012 if (RealignmentPadding > 0) {
2013 if (RealignmentPadding >= 4096) {
2016 .
addImm(RealignmentPadding)
2026 .
addImm(RealignmentPadding)
2043 StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2049 LLVM_DEBUG(
dbgs() <<
"SVECalleeSavedStackSize = " << CalleeSavedSize
2052 CalleeSavesBegin =
MBBI;
2056 CalleeSavesEnd =
MBBI;
2059 SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2066 allocateStackSpace(
MBB, CalleeSavesBegin, 0, SVECalleeSavesSize,
false,
2067 nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
2069 CFAOffset += SVECalleeSavesSize;
2072 emitCalleeSavedSVELocations(
MBB, CalleeSavesEnd);
2077 "Cannot use redzone with stack realignment");
2082 allocateStackSpace(
MBB, CalleeSavesEnd, RealignmentPadding,
2084 NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
2096 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
2108 if (NeedsWinCFI && HasWinCFI) {
2116 if (IsFunclet &&
F.hasPersonalityFn()) {
2126 if (EmitCFI && !EmitAsyncCFI) {
2133 *RegInfo, AArch64::SP, AArch64::SP, TotalSize,
2139 emitCalleeSavedGPRLocations(
MBB,
MBBI);
2140 emitCalleeSavedSVELocations(
MBB,
MBBI);
2145 switch (
MI.getOpcode()) {
2148 case AArch64::CATCHRET:
2149 case AArch64::CLEANUPRET:
2164 bool HasWinCFI =
false;
2165 bool IsFunclet =
false;
2168 DL =
MBBI->getDebugLoc();
2176 BuildMI(MBB, MBB.getFirstTerminator(), DL,
2177 TII->get(AArch64::PAUTH_EPILOGUE))
2178 .setMIFlag(MachineInstr::FrameDestroy);
2188 TII->get(AArch64::SEH_EpilogEnd))
2215 int64_t AfterCSRPopSize = ArgumentStackToRestore;
2223 if (homogeneousPrologEpilog(MF, &
MBB)) {
2227 auto HomogeneousEpilog = std::prev(LastPopI);
2228 if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2229 LastPopI = HomogeneousEpilog;
2239 assert(AfterCSRPopSize == 0);
2242 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(
MBB, NumBytes);
2245 bool CombineAfterCSRBump =
false;
2246 if (!CombineSPBump && PrologueSaveSize != 0) {
2248 while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
2250 Pop = std::prev(Pop);
2253 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
2257 if (OffsetOp.
getImm() == 0 && AfterCSRPopSize >= 0) {
2259 MBB, Pop,
DL,
TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
2266 AfterCSRPopSize += PrologueSaveSize;
2267 CombineAfterCSRBump =
true;
2276 while (LastPopI != Begin) {
2282 }
else if (CombineSPBump)
2284 NeedsWinCFI, &HasWinCFI);
2296 EpilogStartI = LastPopI;
2332 if (CombineSPBump) {
2333 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
2336 if (EmitCFI &&
hasFP(MF)) {
2338 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2353 NumBytes -= PrologueSaveSize;
2354 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
2358 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2361 RestoreBegin = std::prev(RestoreEnd);
2362 while (RestoreBegin !=
MBB.
begin() &&
2371 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2372 DeallocateAfter = CalleeSavedSizeAsOffset;
2394 MBB, RestoreBegin,
DL, AArch64::SP, AArch64::SP,
2396 false,
false,
nullptr, EmitCFI && !
hasFP(MF),
2403 false,
nullptr, EmitCFI && !
hasFP(MF),
2409 false,
nullptr, EmitCFI && !
hasFP(MF),
2414 emitCalleeSavedSVERestores(
MBB, RestoreEnd);
2421 if (RedZone && AfterCSRPopSize == 0)
2428 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2429 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2430 if (NoCalleeSaveRestore)
2431 StackRestoreBytes += AfterCSRPopSize;
2434 MBB, LastPopI,
DL, AArch64::SP, AArch64::SP,
2441 if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2454 MBB, LastPopI,
DL, AArch64::SP, AArch64::FP,
2457 }
else if (NumBytes)
2463 if (EmitCFI &&
hasFP(MF)) {
2465 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2476 if (AfterCSRPopSize) {
2477 assert(AfterCSRPopSize > 0 &&
"attempting to reallocate arg stack that an "
2478 "interrupt may have clobbered");
2483 false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2514 int64_t ObjectOffset) {
2519 unsigned FixedObject =
2528 int64_t ObjectOffset) {
2539 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2546 bool ForSimm)
const {
2549 bool isFixed = MFI.isFixedObjectIndex(FI);
2556 const MachineFunction &MF, int64_t ObjectOffset,
bool isFixed,
bool isSVE,
2557 Register &FrameReg,
bool PreferFP,
bool ForSimm)
const {
2580 PreferFP &= !SVEStackSize;
2588 }
else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2592 assert(
hasFP(MF) &&
"Re-aligned stack must have frame pointer");
2594 }
else if (
hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2599 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2600 PreferFP |=
Offset > -FPOffset && !SVEStackSize;
2602 if (MFI.hasVarSizedObjects()) {
2606 bool CanUseBP = RegInfo->hasBasePointer(MF);
2607 if (FPOffsetFits && CanUseBP)
2614 }
else if (FPOffset >= 0) {
2619 }
else if (MF.
hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2626 "Funclets should only be present on Win64");
2630 if (FPOffsetFits && PreferFP)
2637 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2638 "In the presence of dynamic stack pointer realignment, "
2639 "non-argument/CSR objects cannot be accessed through the frame pointer");
2651 RegInfo->hasStackRealignment(MF))) {
2652 FrameReg = RegInfo->getFrameRegister(MF);
2656 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2662 if (UseFP && !(isFixed || isCSR))
2663 ScalableOffset = -SVEStackSize;
2664 if (!UseFP && (isFixed || isCSR))
2665 ScalableOffset = SVEStackSize;
2668 FrameReg = RegInfo->getFrameRegister(MF);
2673 if (RegInfo->hasBasePointer(MF))
2674 FrameReg = RegInfo->getBaseRegister();
2676 assert(!MFI.hasVarSizedObjects() &&
2677 "Can't use SP when we have var sized objects.");
2678 FrameReg = AArch64::SP;
2704 Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2709 bool NeedsWinCFI,
bool IsFirst,
2718 if (Reg2 == AArch64::FP)
2722 if (
TRI->getEncodingValue(Reg2) ==
TRI->getEncodingValue(Reg1) + 1)
2729 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2730 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2740 bool UsesWinAAPCS,
bool NeedsWinCFI,
2741 bool NeedsFrameRecord,
bool IsFirst,
2749 if (NeedsFrameRecord)
2750 return Reg2 == AArch64::LR;
2758 unsigned Reg1 = AArch64::NoRegister;
2759 unsigned Reg2 = AArch64::NoRegister;
2762 enum RegType { GPR, FPR64, FPR128, PPR, ZPR }
Type;
2764 RegPairInfo() =
default;
2766 bool isPaired()
const {
return Reg2 != AArch64::NoRegister; }
2768 unsigned getScale()
const {
2782 bool isScalable()
const {
return Type == PPR ||
Type == ZPR; }
2788 for (
unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
2789 if (SavedRegs.
test(PReg)) {
2790 unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
2794 return AArch64::NoRegister;
2800 bool NeedsFrameRecord) {
2810 unsigned Count = CSI.
size();
2817 "Odd number of callee-saved regs to spill!");
2819 int StackFillDir = -1;
2821 unsigned FirstReg = 0;
2829 FirstReg = Count - 1;
2835 for (
unsigned i = FirstReg; i < Count; i += RegInc) {
2837 RPI.Reg1 = CSI[i].getReg();
2839 if (AArch64::GPR64RegClass.
contains(RPI.Reg1))
2840 RPI.Type = RegPairInfo::GPR;
2841 else if (AArch64::FPR64RegClass.
contains(RPI.Reg1))
2842 RPI.Type = RegPairInfo::FPR64;
2843 else if (AArch64::FPR128RegClass.
contains(RPI.Reg1))
2844 RPI.Type = RegPairInfo::FPR128;
2845 else if (AArch64::ZPRRegClass.
contains(RPI.Reg1))
2846 RPI.Type = RegPairInfo::ZPR;
2847 else if (AArch64::PPRRegClass.
contains(RPI.Reg1))
2848 RPI.Type = RegPairInfo::PPR;
2853 if (
unsigned(i + RegInc) < Count) {
2854 Register NextReg = CSI[i + RegInc].getReg();
2855 bool IsFirst = i == FirstReg;
2857 case RegPairInfo::GPR:
2858 if (AArch64::GPR64RegClass.
contains(NextReg) &&
2860 NeedsWinCFI, NeedsFrameRecord, IsFirst,
2864 case RegPairInfo::FPR64:
2865 if (AArch64::FPR64RegClass.
contains(NextReg) &&
2870 case RegPairInfo::FPR128:
2871 if (AArch64::FPR128RegClass.
contains(NextReg))
2874 case RegPairInfo::PPR:
2876 case RegPairInfo::ZPR:
2878 if (((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1))
2890 assert((!RPI.isPaired() ||
2891 (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
2892 "Out of order callee saved regs!");
2894 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2895 RPI.Reg1 == AArch64::LR) &&
2896 "FrameRecord must be allocated together with LR");
2899 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2900 RPI.Reg2 == AArch64::LR) &&
2901 "FrameRecord must be allocated together with LR");
2909 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2910 RPI.Reg1 + 1 == RPI.Reg2))) &&
2911 "Callee-save registers not saved as adjacent register pair!");
2913 RPI.FrameIdx = CSI[i].getFrameIdx();
2916 RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
2917 int Scale = RPI.getScale();
2919 int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2920 assert(OffsetPre % Scale == 0);
2922 if (RPI.isScalable())
2923 ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2925 ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2930 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
2931 (IsWindows && RPI.Reg2 == AArch64::LR)))
2932 ByteOffset += StackFillDir * 8;
2936 if (NeedGapToAlignStack && !NeedsWinCFI &&
2937 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2938 !RPI.isPaired() && ByteOffset % 16 != 0) {
2939 ByteOffset += 8 * StackFillDir;
2940 assert(MFI.getObjectAlign(RPI.FrameIdx) <=
Align(16));
2944 MFI.setObjectAlignment(RPI.FrameIdx,
Align(16));
2945 NeedGapToAlignStack =
false;
2948 int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2949 assert(OffsetPost % Scale == 0);
2952 int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
2957 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
2958 (IsWindows && RPI.Reg2 == AArch64::LR)))
2960 RPI.Offset =
Offset / Scale;
2962 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2963 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2964 "Offset out of bounds for LDP/STP immediate");
2968 if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
2969 RPI.Reg2 == AArch64::FP) ||
2970 (IsWindows && RPI.Reg1 == AArch64::FP &&
2971 RPI.Reg2 == AArch64::LR)))
2985 MFI.setObjectAlignment(CSI[0].getFrameIdx(),
Align(16));
2988 std::reverse(RegPairs.
begin(), RegPairs.
end());
3004 if (homogeneousPrologEpilog(MF)) {
3008 for (
auto &RPI : RegPairs) {
3013 if (!
MRI.isReserved(RPI.Reg1))
3015 if (RPI.isPaired() && !
MRI.isReserved(RPI.Reg2))
3020 bool PTrueCreated =
false;
3022 unsigned Reg1 = RPI.Reg1;
3023 unsigned Reg2 = RPI.Reg2;
3039 case RegPairInfo::GPR:
3040 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
3042 Alignment =
Align(8);
3044 case RegPairInfo::FPR64:
3045 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
3047 Alignment =
Align(8);
3049 case RegPairInfo::FPR128:
3050 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
3052 Alignment =
Align(16);
3054 case RegPairInfo::ZPR:
3055 StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
3057 Alignment =
Align(16);
3059 case RegPairInfo::PPR:
3060 StrOpc = AArch64::STR_PXI;
3062 Alignment =
Align(2);
3067 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3068 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3071 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
3072 "Windows unwdinding requires a consecutive (FP,LR) pair");
3076 unsigned FrameIdxReg1 = RPI.FrameIdx;
3077 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3078 if (NeedsWinCFI && RPI.isPaired()) {
3083 if (RPI.isPaired() && RPI.isScalable()) {
3088 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3089 "Expects SVE2.1 or SME2 target and a predicate register");
3090#ifdef EXPENSIVE_CHECKS
3091 auto IsPPR = [](
const RegPairInfo &c) {
3092 return c.Reg1 == RegPairInfo::PPR;
3094 auto PPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsPPR);
3095 auto IsZPR = [](
const RegPairInfo &c) {
3096 return c.Type == RegPairInfo::ZPR;
3098 auto ZPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsZPR);
3099 assert(!(PPRBegin < ZPRBegin) &&
3100 "Expected callee save predicate to be handled first");
3102 if (!PTrueCreated) {
3103 PTrueCreated =
true;
3108 if (!
MRI.isReserved(Reg1))
3110 if (!
MRI.isReserved(Reg2))
3112 MIB.
addReg( AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
3128 if (!
MRI.isReserved(Reg1))
3130 if (RPI.isPaired()) {
3131 if (!
MRI.isReserved(Reg2))
3151 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
3170 DL =
MBBI->getDebugLoc();
3173 if (homogeneousPrologEpilog(MF, &
MBB)) {
3176 for (
auto &RPI : RegPairs) {
3184 auto IsPPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::PPR; };
3185 auto PPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsPPR);
3186 auto PPREnd = std::find_if_not(PPRBegin, RegPairs.
end(), IsPPR);
3187 std::reverse(PPRBegin, PPREnd);
3188 auto IsZPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::ZPR; };
3189 auto ZPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsZPR);
3190 auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.
end(), IsZPR);
3191 std::reverse(ZPRBegin, ZPREnd);
3193 bool PTrueCreated =
false;
3194 for (
const RegPairInfo &RPI : RegPairs) {
3195 unsigned Reg1 = RPI.Reg1;
3196 unsigned Reg2 = RPI.Reg2;
3210 case RegPairInfo::GPR:
3211 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
3213 Alignment =
Align(8);
3215 case RegPairInfo::FPR64:
3216 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
3218 Alignment =
Align(8);
3220 case RegPairInfo::FPR128:
3221 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
3223 Alignment =
Align(16);
3225 case RegPairInfo::ZPR:
3226 LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
3228 Alignment =
Align(16);
3230 case RegPairInfo::PPR:
3231 LdrOpc = AArch64::LDR_PXI;
3233 Alignment =
Align(2);
3238 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3239 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3245 unsigned FrameIdxReg1 = RPI.FrameIdx;
3246 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3247 if (NeedsWinCFI && RPI.isPaired()) {
3253 if (RPI.isPaired() && RPI.isScalable()) {
3257 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3258 "Expects SVE2.1 or SME2 target and a predicate register");
3259#ifdef EXPENSIVE_CHECKS
3260 assert(!(PPRBegin < ZPRBegin) &&
3261 "Expected callee save predicate to be handled first");
3263 if (!PTrueCreated) {
3264 PTrueCreated =
true;
3269 MIB.
addReg( AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
3286 if (RPI.isPaired()) {
3320 unsigned UnspilledCSGPR = AArch64::NoRegister;
3321 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
3330 unsigned ExtraCSSpill = 0;
3331 bool HasUnpairedGPR64 =
false;
3332 bool HasPairZReg =
false;
3334 for (
unsigned i = 0; CSRegs[i]; ++i) {
3335 const unsigned Reg = CSRegs[i];
3338 if (Reg == BasePointerReg)
3341 bool RegUsed = SavedRegs.
test(Reg);
3342 unsigned PairedReg = AArch64::NoRegister;
3343 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
3344 if (RegIsGPR64 || AArch64::FPR64RegClass.
contains(Reg) ||
3345 AArch64::FPR128RegClass.contains(Reg)) {
3348 if (HasUnpairedGPR64)
3349 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
3351 PairedReg = CSRegs[i ^ 1];
3358 if (RegIsGPR64 && !AArch64::GPR64RegClass.
contains(PairedReg)) {
3359 PairedReg = AArch64::NoRegister;
3360 HasUnpairedGPR64 =
true;
3362 assert(PairedReg == AArch64::NoRegister ||
3363 AArch64::GPR64RegClass.
contains(Reg, PairedReg) ||
3364 AArch64::FPR64RegClass.
contains(Reg, PairedReg) ||
3365 AArch64::FPR128RegClass.
contains(Reg, PairedReg));
3368 if (AArch64::GPR64RegClass.
contains(Reg) &&
3370 UnspilledCSGPR = Reg;
3371 UnspilledCSGPRPaired = PairedReg;
3379 if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
3380 !SavedRegs.
test(PairedReg)) {
3381 SavedRegs.
set(PairedReg);
3382 if (AArch64::GPR64RegClass.
contains(PairedReg) &&
3384 ExtraCSSpill = PairedReg;
3387 HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
3388 SavedRegs.
test(CSRegs[i ^ 1]));
3391 if (HasPairZReg && (Subtarget.hasSVE2p1() || Subtarget.hasSME2())) {
3396 if (PnReg != AArch64::NoRegister)
3402 SavedRegs.
set(AArch64::P8);
3407 "Predicate cannot be a reserved register");
3417 SavedRegs.
set(AArch64::X18);
3421 unsigned CSStackSize = 0;
3422 unsigned SVECSStackSize = 0;
3425 for (
unsigned Reg : SavedRegs.
set_bits()) {
3427 if (AArch64::PPRRegClass.
contains(Reg) ||
3428 AArch64::ZPRRegClass.
contains(Reg))
3435 unsigned NumSavedRegs = SavedRegs.
count();
3441 SavedRegs.
set(AArch64::FP);
3442 SavedRegs.
set(AArch64::LR);
3452 int64_t SVEStackSize =
3453 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
3454 bool CanEliminateFrame = (SavedRegs.
count() == 0) && !SVEStackSize;
3463 int64_t CalleeStackUsed = 0;
3466 if (FixedOff > CalleeStackUsed) CalleeStackUsed = FixedOff;
3470 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
3471 CalleeStackUsed) > EstimatedStackSizeLimit;
3473 AFI->setHasStackFrame(
true);
3482 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
3484 <<
" to get a scratch register.\n");
3485 SavedRegs.
set(UnspilledCSGPR);
3486 ExtraCSSpill = UnspilledCSGPR;
3491 if (producePairRegisters(MF)) {
3492 if (UnspilledCSGPRPaired == AArch64::NoRegister) {
3495 SavedRegs.
reset(UnspilledCSGPR);
3496 ExtraCSSpill = AArch64::NoRegister;
3499 SavedRegs.
set(UnspilledCSGPRPaired);
3508 unsigned Size =
TRI->getSpillSize(RC);
3509 Align Alignment =
TRI->getSpillAlign(RC);
3512 LLVM_DEBUG(
dbgs() <<
"No available CS registers, allocated fi#" << FI
3513 <<
" as the emergency spill slot.\n");
3518 CSStackSize += 8 * (SavedRegs.
count() - NumSavedRegs);
3522 if (
hasFP(MF) && AFI->hasSwiftAsyncContext())
3527 << EstimatedStackSize + AlignedCSStackSize
3531 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3532 "Should not invalidate callee saved info");
3536 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
3537 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3538 AFI->setSVECalleeSavedStackSize(
alignTo(SVECSStackSize, 16));
3543 std::vector<CalleeSavedInfo> &CSI,
unsigned &MinCSFrameIndex,
3544 unsigned &MaxCSFrameIndex)
const {
3552 std::reverse(CSI.begin(), CSI.end());
3566 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3567 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3570 for (
auto &CS : CSI) {
3577 CS.setFrameIdx(FrameIdx);
3579 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3580 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3584 Reg == AArch64::FP) {
3587 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3588 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3608 int &Min,
int &Max) {
3609 Min = std::numeric_limits<int>::max();
3610 Max = std::numeric_limits<int>::min();
3616 for (
auto &CS : CSI) {
3617 if (AArch64::ZPRRegClass.
contains(CS.getReg()) ||
3618 AArch64::PPRRegClass.contains(CS.getReg())) {
3619 assert((Max == std::numeric_limits<int>::min() ||
3620 Max + 1 == CS.getFrameIdx()) &&
3621 "SVE CalleeSaves are not consecutive");
3623 Min = std::min(Min, CS.getFrameIdx());
3624 Max = std::max(Max, CS.getFrameIdx());
3627 return Min != std::numeric_limits<int>::max();
3636 int &MinCSFrameIndex,
3637 int &MaxCSFrameIndex,
3638 bool AssignOffsets) {
3643 "SVE vectors should never be passed on the stack by value, only by "
3647 auto Assign = [&MFI](
int FI, int64_t
Offset) {
3657 for (
int I = MinCSFrameIndex;
I <= MaxCSFrameIndex; ++
I) {
3673 int StackProtectorFI = -1;
3677 ObjectsToAllocate.
push_back(StackProtectorFI);
3683 if (
I == StackProtectorFI)
3685 if (MaxCSFrameIndex >=
I &&
I >= MinCSFrameIndex)
3694 for (
unsigned FI : ObjectsToAllocate) {
3699 if (Alignment >
Align(16))
3701 "Alignment of scalable vectors > 16 bytes is not yet supported");
3711int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3713 int MinCSFrameIndex, MaxCSFrameIndex;
3717int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3728 "Upwards growing stack unsupported");
3730 int MinCSFrameIndex, MaxCSFrameIndex;
3731 int64_t SVEStackSize =
3732 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
3752 int64_t FixedObject =
3765 assert(DstReg &&
"There must be a free register after frame setup");
3774struct TagStoreInstr {
3797 std::optional<int64_t> FrameRegUpdate;
3799 unsigned FrameRegUpdateFlags;
3810 :
MBB(
MBB), ZeroData(ZeroData) {
3816 void addInstruction(TagStoreInstr
I) {
3818 TagStores.
back().Offset + TagStores.
back().Size ==
I.Offset) &&
3819 "Non-adjacent tag store instructions.");
3834 const int64_t kMinOffset = -256 * 16;
3835 const int64_t kMaxOffset = 255 * 16;
3838 int64_t BaseRegOffsetBytes = FrameRegOffset.
getFixed();
3839 if (BaseRegOffsetBytes < kMinOffset ||
3840 BaseRegOffsetBytes + (
Size -
Size % 32) > kMaxOffset ||
3844 BaseRegOffsetBytes % 16 != 0) {
3845 Register ScratchReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3848 BaseReg = ScratchReg;
3849 BaseRegOffsetBytes = 0;
3854 int64_t InstrSize = (
Size > 16) ? 32 : 16;
3857 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
3858 : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
3859 assert(BaseRegOffsetBytes % 16 == 0);
3863 .
addImm(BaseRegOffsetBytes / 16)
3867 if (BaseRegOffsetBytes == 0)
3869 BaseRegOffsetBytes += InstrSize;
3883 :
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3884 Register SizeReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3888 int64_t LoopSize =
Size;
3891 if (FrameRegUpdate && *FrameRegUpdate)
3892 LoopSize -= LoopSize % 32;
3894 TII->get(ZeroData ? AArch64::STZGloop_wback
3895 : AArch64::STGloop_wback))
3902 LoopI->
setFlags(FrameRegUpdateFlags);
3904 int64_t ExtraBaseRegUpdate =
3905 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.
getFixed() -
Size) : 0;
3906 if (LoopSize <
Size) {
3911 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3915 .
addImm(1 + ExtraBaseRegUpdate / 16)
3918 }
else if (ExtraBaseRegUpdate) {
3922 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
3925 .
addImm(std::abs(ExtraBaseRegUpdate))
3935 int64_t
Size, int64_t *TotalOffset) {
3937 if ((
MI.getOpcode() == AArch64::ADDXri ||
3938 MI.getOpcode() == AArch64::SUBXri) &&
3939 MI.getOperand(0).getReg() == Reg &&
MI.getOperand(1).getReg() == Reg) {
3941 int64_t
Offset =
MI.getOperand(2).getImm() << Shift;
3942 if (
MI.getOpcode() == AArch64::SUBXri)
3944 int64_t AbsPostOffset = std::abs(
Offset -
Size);
3945 const int64_t kMaxOffset =
3947 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
3958 for (
auto &TS : TSE) {
3962 if (
MI->memoperands_empty()) {
3966 MemRefs.
append(
MI->memoperands_begin(),
MI->memoperands_end());
3972 bool TryMergeSPUpdate) {
3973 if (TagStores.
empty())
3975 TagStoreInstr &FirstTagStore = TagStores[0];
3976 TagStoreInstr &LastTagStore = TagStores[TagStores.
size() - 1];
3977 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3978 DL = TagStores[0].MI->getDebugLoc();
3982 *MF, FirstTagStore.Offset,
false ,
false , Reg,
3985 FrameRegUpdate = std::nullopt;
3987 mergeMemRefs(TagStores, CombinedMemRefs);
3990 for (
const auto &Instr
3991 : TagStores) {
dbgs() <<
" " << *
Instr.MI; });
3997 if (TagStores.size() < 2)
3999 emitUnrolled(InsertI);
4002 int64_t TotalOffset = 0;
4003 if (TryMergeSPUpdate) {
4009 if (InsertI !=
MBB->
end() &&
4010 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.
getFixed() +
Size,
4012 UpdateInstr = &*InsertI++;
4018 if (!UpdateInstr && TagStores.size() < 2)
4022 FrameRegUpdate = TotalOffset;
4023 FrameRegUpdateFlags = UpdateInstr->
getFlags();
4030 for (
auto &TS : TagStores)
4031 TS.MI->eraseFromParent();
4035 int64_t &
Size,
bool &ZeroData) {
4039 unsigned Opcode =
MI.getOpcode();
4040 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
4041 Opcode == AArch64::STZ2Gi);
4043 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
4044 if (!
MI.getOperand(0).isDead() || !
MI.getOperand(1).isDead())
4046 if (!
MI.getOperand(2).isImm() || !
MI.getOperand(3).isFI())
4049 Size =
MI.getOperand(2).getImm();
4053 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
4055 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
4060 if (
MI.getOperand(0).getReg() != AArch64::SP || !
MI.getOperand(1).isFI())
4064 16 *
MI.getOperand(2).getImm();
4084 if (!isMergeableStackTaggingInstruction(
MI,
Offset,
Size, FirstZeroData))
4090 constexpr int kScanLimit = 10;
4093 NextI != E && Count < kScanLimit; ++NextI) {
4102 if (isMergeableStackTaggingInstruction(
MI,
Offset,
Size, ZeroData)) {
4103 if (ZeroData != FirstZeroData)
4111 if (!
MI.isTransient())
4120 if (
MI.mayLoadOrStore() ||
MI.hasUnmodeledSideEffects())
4136 LiveRegs.addLiveOuts(*
MBB);
4141 LiveRegs.stepBackward(*
I);
4144 if (LiveRegs.contains(AArch64::NZCV))
4148 [](
const TagStoreInstr &
Left,
const TagStoreInstr &
Right) {
4153 int64_t CurOffset = Instrs[0].Offset;
4154 for (
auto &Instr : Instrs) {
4155 if (CurOffset >
Instr.Offset)
4162 TagStoreEdit TSE(
MBB, FirstZeroData);
4163 std::optional<int64_t> EndOffset;
4164 for (
auto &Instr : Instrs) {
4165 if (EndOffset && *EndOffset !=
Instr.Offset) {
4167 TSE.emitCode(InsertI, TFI,
false);
4171 TSE.addInstruction(Instr);
4190 II = tryMergeAdjacentSTG(II,
this, RS);
4198 bool IgnoreSPUpdates)
const {
4200 if (IgnoreSPUpdates) {
4203 FrameReg = AArch64::SP;
4213 FrameReg = AArch64::SP;
4238 bool IsValid =
false;
4240 int ObjectIndex = 0;
4242 int GroupIndex = -1;
4244 bool ObjectFirst =
false;
4247 bool GroupFirst =
false;
4252 int NextGroupIndex = 0;
4253 std::vector<FrameObject> &Objects;
4256 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
4258 void EndCurrentGroup() {
4259 if (CurrentMembers.
size() > 1) {
4264 for (
int Index : CurrentMembers) {
4265 Objects[
Index].GroupIndex = NextGroupIndex;
4271 CurrentMembers.clear();
4275bool FrameObjectCompare(
const FrameObject &
A,
const FrameObject &
B) {
4293 return std::make_tuple(!
A.IsValid,
A.ObjectFirst,
A.GroupFirst,
A.GroupIndex,
4295 std::make_tuple(!
B.IsValid,
B.ObjectFirst,
B.GroupFirst,
B.GroupIndex,
4307 for (
auto &Obj : ObjectsToAllocate) {
4308 FrameObjects[Obj].IsValid =
true;
4309 FrameObjects[Obj].ObjectIndex = Obj;
4313 GroupBuilder GB(FrameObjects);
4314 for (
auto &
MBB : MF) {
4315 for (
auto &
MI :
MBB) {
4316 if (
MI.isDebugInstr())
4319 switch (
MI.getOpcode()) {
4320 case AArch64::STGloop:
4321 case AArch64::STZGloop:
4325 case AArch64::STZGi:
4326 case AArch64::ST2Gi:
4327 case AArch64::STZ2Gi:
4340 FrameObjects[FI].IsValid)
4348 GB.AddMember(TaggedFI);
4350 GB.EndCurrentGroup();
4353 GB.EndCurrentGroup();
4363 FrameObjects[*TBPI].ObjectFirst =
true;
4364 FrameObjects[*TBPI].GroupFirst =
true;
4365 int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4366 if (FirstGroupIndex >= 0)
4367 for (FrameObject &Object : FrameObjects)
4368 if (Object.GroupIndex == FirstGroupIndex)
4369 Object.GroupFirst =
true;
4375 for (
auto &Obj : FrameObjects) {
4379 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4386 dbgs() <<
" " << Obj.ObjectIndex <<
": group " << Obj.GroupIndex;
4387 if (Obj.ObjectFirst)
4388 dbgs() <<
", first";
4390 dbgs() <<
", group-first";
4400AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
4411 MF.
insert(MBBInsertPoint, LoopMBB);
4413 MF.
insert(MBBInsertPoint, ExitMBB);
4448 return ExitMBB->
begin();
4451void AArch64FrameLowering::inlineStackProbeFixed(
4464 int64_t NumBlocks = FrameSize / ProbeSize;
4465 int64_t ResidualSize = FrameSize % ProbeSize;
4467 LLVM_DEBUG(
dbgs() <<
"Stack probing: total " << FrameSize <<
" bytes, "
4468 << NumBlocks <<
" blocks of " << ProbeSize
4469 <<
" bytes, plus " << ResidualSize <<
" bytes\n");
4474 for (
int i = 0; i < NumBlocks; ++i) {
4480 EmitAsyncCFI && !HasFP, CFAOffset);
4489 }
else if (NumBlocks != 0) {
4495 EmitAsyncCFI && !HasFP, CFAOffset);
4497 MBBI = inlineStackProbeLoopExactMultiple(
MBBI, ProbeSize, ScratchReg);
4499 if (EmitAsyncCFI && !HasFP) {
4503 unsigned Reg =
RegInfo.getDwarfRegNum(AArch64::SP,
true);
4512 if (ResidualSize != 0) {
4518 EmitAsyncCFI && !HasFP, CFAOffset);
4537 if (
MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
4538 MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
4542 if (
MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
4543 Register ScratchReg =
MI->getOperand(0).getReg();
4544 int64_t FrameSize =
MI->getOperand(1).getImm();
4546 MI->getOperand(3).getImm());
4547 inlineStackProbeFixed(
MI->getIterator(), ScratchReg, FrameSize,
4550 assert(
MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
4551 "Stack probe pseudo-instruction expected");
4554 Register TargetReg =
MI->getOperand(0).getReg();
4555 (void)
TII->probedStackAlloc(
MI->getIterator(), TargetReg,
true);
4557 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB)
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL)
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
static void emitCalleeSavedRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool SVE)
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool NeedsFrameRecord)
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned FixedObject)
static bool needsWinCFI(const MachineFunction &MF)
static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, unsigned DwarfReg)
static cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
static bool produceCompactUnwindFrame(MachineFunction &MF)
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex, bool AssignOffsets)
static cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
static bool windowsRequiresStackProbe(MachineFunction &MF, uint64_t StackSizeInBytes)
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst, const TargetRegisterInfo *TRI)
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, MachineInstr::MIFlag FrameFlag=MachineInstr::FrameSetup, int CFAOffset=0)
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, LivePhysRegs &LiveRegs)
Collect live registers from the end of MI's parent up to (including) MI in LiveRegs.
cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
static bool IsSVECalleeSave(MachineBasicBlock::iterator I)
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst, const TargetRegisterInfo *TRI)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
unsigned findFreePredicateReg(BitVector &SavedRegs)
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
static bool isTargetWindows(const MachineFunction &MF)
static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset)
static int64_t upperBound(StackOffset Size)
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool NeedsWinCFI, bool NeedsUnwindInfo)
static unsigned getFixedObjectSize(const MachineFunction &MF, const AArch64FunctionInfo *AFI, bool IsWin64, bool IsFunclet)
Returns the size of the fixed object area (allocated next to sp on entry) On Win64 this may include a...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const int kSetTagLoopThreshold
This file contains the simple types necessary to represent the attributes associated with functions a...
#define CASE(ATTRNAME, AANAME,...)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static void clear(coro::Shape &Shape)
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static const unsigned FramePtr
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
bool enableCFIFixup(MachineFunction &MF) const override
Returns true if we may need to fix the unwind information for the function.
void resetCFIToInitialState(MachineBasicBlock &MBB) const override
Emit CFI instructions that recreate the state of the unwind information upon fucntion entry.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE, Register &FrameReg, bool PreferFP, bool ForSimm) const
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const
void setSwiftAsyncContextFrameIdx(int FI)
unsigned getTailCallReservedStack() const
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
bool hasStackProbing() const
unsigned getArgumentStackToRestore() const
void setLocalStackSize(uint64_t Size)
int getCalleeSaveBaseToFrameRecordOffset() const
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setPredicateRegForFillSpill(unsigned Reg)
int64_t getStackProbeSize() const
uint64_t getStackSizeSVE() const
void setHasRedZone(bool s)
bool hasStackFrame() const
std::optional< int > getTaggedBasePointerIndex() const
uint64_t getLocalStackSize() const
void setStackRealigned(bool s)
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
unsigned getVarArgsGPRSize() const
void setStackSizeSVE(uint64_t S)
bool isStackRealigned() const
bool hasSwiftAsyncContext() const
void setTaggedBasePointerOffset(unsigned Offset)
unsigned getPredicateRegForFillSpill() const
unsigned getSVECalleeSavedStackSize() const
bool needsAsyncDwarfUnwindInfo(const MachineFunction &MF) const
void setMinMaxSVECSFrameIndex(int Min, int Max)
bool hasCalleeSaveStackFreeSpace() const
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
bool hasBasePointer(const MachineFunction &MF) const
bool cannotEliminateFrame(const MachineFunction &MF) const
unsigned getBaseRegister() const
bool isTargetWindows() const
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
bool isTargetMachO() const
const Triple & getTargetTriple() const
bool isCallingConvWin64(CallingConv::ID CC) const
const char * getChkStkName() const
bool swiftAsyncContextIsDynamicallySet() const
Return whether FrameLowering should always set the "extended frame present" bit in FP,...
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
unsigned getRedZoneSize(const Function &F) const
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value.
bool test(unsigned Idx) const
size_type count() const
count - Returns the number of bits which are set.
iterator_range< const_set_bits_iterator > set_bits() const
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void stepBackward(const MachineInstr &MI)
Simulates liveness when stepping backwards over an instruction(bundle).
void removeReg(MCPhysReg Reg)
Removes a physical register, all its sub-registers, and all its super-registers from the set.
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds all live-out registers of basic block MBB.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
bool usesWindowsCFI() const
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_def_cfa_register modifies a rule for computing CFA.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
static MCCFIInstruction createNegateRAState(MCSymbol *L, SMLoc Loc={})
.cfi_negate_ra_state AArch64 negate RA state.
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_same_value Current value of Register is the same as in the previous frame.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
Wrapper class representing physical registers. Should be passed by value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator instr_begin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MachineInstr & instr_back()
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
reverse_iterator rbegin()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
int getStackProtectorIndex() const
Return the index for the stack protector object.
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackProtectorIndex() const
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
unsigned addFrameInst(const MCCFIInstruction &Inst)
void setHasWinCFI(bool v)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
bool hasEHFunclets() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
void setFlags(unsigned flags)
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
uint32_t getFlags() const
Return the MI flags bitvector.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
static MachineOperand CreateImm(int64_t Val)
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool isLiveIn(Register Reg) const
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void backward()
Update internal register state and move MBB iterator backwards.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Wrapper class representing virtual and physical registers.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
StringRef - Represent a constant reference to a string, i.e.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool enableCFIFixup(MachineFunction &MF) const
Returns true if we may need to fix the unwind information for the function.
TargetInstrInfo - Interface to description of machine instruction set.
CodeModel::Model getCodeModel() const
Returns the code model.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
SwiftAsyncFramePointerMode SwiftAsyncFramePointer
Control when and how the Swift async frame pointer bit should be set.
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
StringRef getArchName() const
Get the architecture (first) component of the triple.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
const unsigned StackProbeMaxLoopUnroll
Maximum number of iterations to unroll for a constant size probing loop.
const unsigned StackProbeMaxUnprobedStack
Maximum allowed number of unprobed bytes above SP at an ABI boundary.
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
NodeAddr< InstrNode * > Instr
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
auto reverse(ContainerTy &&C)
@ Always
Always set the bit.
@ Never
Never set the bit.
@ DeploymentBased
Determine whether to set the bit statically or dynamically based on the deployment target.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Description of the encoding of one expression Op.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.