llvm.org GIT mirror llvm / cee9af9
XRay: Add entry and exit sleds Summary: In this patch we implement the following parts of XRay: - Supporting a function attribute named 'function-instrument' which currently only supports 'xray-always'. We should be able to use this attribute for other instrumentation approaches. - Supporting a function attribute named 'xray-instruction-threshold' used to determine whether a function is instrumented with a minimum number of instructions (IR instruction counts). - X86-specific nop sleds as described in the white paper. - A machine function pass that adds the different instrumentation marker instructions at a very late stage. - A way of identifying which return opcode is considered "normal" for each architecture. There are some caveats here: 1) We don't handle PATCHABLE_RET in platforms other than x86_64 yet -- this means if IR used PATCHABLE_RET directly instead of a normal ret, instruction lowering for that platform might do the wrong thing. We think this should be handled at instruction selection time to by default be unpacked for platforms where XRay is not availble yet. 2) The generated section for X86 is different from what is described from the white paper for the sole reason that LLVM allows us to do this neatly. We're taking the opportunity to deviate from the white paper from this perspective to allow us to get richer information from the runtime library. Reviewers: sanjoy, eugenis, kcc, pcc, echristo, rnk Subscribers: niravd, majnemer, atrick, rnk, emaste, bmakam, mcrosier, mehdi_amini, llvm-commits Differential Revision: http://reviews.llvm.org/D19904 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@275367 91177308-0d34-0410-b5e6-96231b3b80d8 Dean Michael Berris 3 years ago
18 changed file(s) with 317 addition(s) and 12 deletion(s). Raw diff Collapse all Expand all
261261
262262 /// \brief This pass lays out funclets contiguously.
263263 extern char &FuncletLayoutID;
264
265 /// This pass inserts the XRay instrumentation sleds if they are supported by
266 /// the target platform.
267 extern char &XRayInstrumentationID;
264268
265269 /// \brief This pass implements the "patchable-function" attribute.
266270 extern char &PatchableFunctionID;
334334 void initializeWholeProgramDevirtPass(PassRegistry &);
335335 void initializeWinEHPreparePass(PassRegistry&);
336336 void initializeWriteBitcodePassPass(PassRegistry &);
337 void initializeXRayInstrumentationPass(PassRegistry &);
337338 }
338339
339340 #endif
945945 let mayStore = 1;
946946 let hasSideEffects = 1;
947947 }
948 def PATCHABLE_FUNCTION_ENTER : Instruction {
949 let OutOperandList = (outs);
950 let InOperandList = (ins);
951 let AsmString = "# XRay Function Enter.";
952 let usesCustomInserter = 1;
953 let hasSideEffects = 0;
954 }
955 def PATCHABLE_RET : Instruction {
956 let OutOperandList = (outs unknown:$dst);
957 let InOperandList = (ins variable_ops);
958 let AsmString = "# XRay Function Exit.";
959 let usesCustomInserter = 1;
960 let hasSideEffects = 1;
961 let isReturn = 1;
962 }
948963
949964 // Generic opcodes used in GlobalISel.
950965 include "llvm/Target/GenericOpcodes.td"
5454 void operator=(const TargetInstrInfo &) = delete;
5555 public:
5656 TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u,
57 unsigned CatchRetOpcode = ~0u)
57 unsigned CatchRetOpcode = ~0u, unsigned ReturnOpcode = ~0u)
5858 : CallFrameSetupOpcode(CFSetupOpcode),
5959 CallFrameDestroyOpcode(CFDestroyOpcode),
60 CatchRetOpcode(CatchRetOpcode) {}
60 CatchRetOpcode(CatchRetOpcode),
61 ReturnOpcode(ReturnOpcode) {}
6162
6263 virtual ~TargetInstrInfo();
6364
150151 unsigned getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; }
151152
152153 unsigned getCatchReturnOpcode() const { return CatchRetOpcode; }
154 unsigned getReturnOpcode() const { return ReturnOpcode; }
153155
154156 /// Returns the actual stack pointer adjustment made by an instruction
155157 /// as part of a call sequence. By default, only call frame setup/destroy
14391441 private:
14401442 unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode;
14411443 unsigned CatchRetOpcode;
1444 unsigned ReturnOpcode;
14421445 };
14431446
14441447 /// \brief Provide DenseMapInfo for TargetInstrInfo::RegSubRegPair.
141141 /// original instruction.
142142 HANDLE_TARGET_OPCODE(PATCHABLE_OP, 23)
143143
144 /// This is a marker instruction which gets translated into a nop sled, useful
145 /// for inserting instrumentation instructions at runtime.
146 HANDLE_TARGET_OPCODE(PATCHABLE_FUNCTION_ENTER, 24)
147
148 /// Wraps a return instruction and its operands to enable adding nop sleds
149 /// either before or after the return. The nop sleds are useful for inserting
150 /// instrumentation instructions at runtime.
151 HANDLE_TARGET_OPCODE(PATCHABLE_RET, 25)
152
144153 /// The following generic opcodes are not supposed to appear after ISel.
145154 /// This is something we might want to relax, but for now, this is convenient
146155 /// to produce diagnostics.
147156
148157 /// Generic ADD instruction. This is an integer add.
149 HANDLE_TARGET_OPCODE(G_ADD, 24)
158 HANDLE_TARGET_OPCODE(G_ADD, 26)
150159 HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_START, G_ADD)
151160
152161 /// Generic Bitwise-OR instruction.
153162 HANDLE_TARGET_OPCODE(G_OR, 25)
154163
155164 /// Generic BRANCH instruction. This is an unconditional branch.
156 HANDLE_TARGET_OPCODE(G_BR, 26)
165 HANDLE_TARGET_OPCODE(G_BR, 27)
157166
158167 // TODO: Add more generic opcodes as we move along.
159168
135135 UnreachableBlockElim.cpp
136136 VirtRegMap.cpp
137137 WinEHPrepare.cpp
138 XRayInstrumentation.cpp
138139
139140 ADDITIONAL_HEADER_DIRS
140141 ${LLVM_MAIN_INCLUDE_DIR}/llvm/CodeGen
5656 initializeMachineSchedulerPass(Registry);
5757 initializeMachineSinkingPass(Registry);
5858 initializeMachineVerifierPassPass(Registry);
59 initializeXRayInstrumentationPass(Registry);
5960 initializePatchableFunctionPass(Registry);
6061 initializeOptimizePHIsPass(Registry);
6162 initializePEIPass(Registry);
654654 addPass(&StackMapLivenessID, false);
655655 addPass(&LiveDebugValuesID, false);
656656
657 addPass(&XRayInstrumentationID, false);
657658 addPass(&PatchableFunctionID, false);
658659
659660 AddingMachinePasses = false;
0 //===-- XRayInstrumentation.cpp - Adds XRay instrumentation to functions. -===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a MachineFunctionPass that inserts the appropriate
10 // XRay instrumentation instructions. We look for XRay-specific attributes
11 // on the function to determine whether we should insert the replacement
12 // operations.
13 //
14 //===---------------------------------------------------------------------===//
15
16 #include "llvm/CodeGen/Analysis.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/MachineFunctionPass.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/Passes.h"
21 #include "llvm/Support/TargetRegistry.h"
22 #include "llvm/Target/TargetInstrInfo.h"
23 #include "llvm/Target/TargetSubtargetInfo.h"
24
25 using namespace llvm;
26
27 namespace {
28 struct XRayInstrumentation : public MachineFunctionPass {
29 static char ID;
30
31 XRayInstrumentation() : MachineFunctionPass(ID) {
32 initializeXRayInstrumentationPass(*PassRegistry::getPassRegistry());
33 }
34
35 bool runOnMachineFunction(MachineFunction &MF) override;
36 };
37 }
38
39 bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
40 auto &F = *MF.getFunction();
41 auto InstrAttr = F.getFnAttribute("function-instrument");
42 bool AlwaysInstrument = !InstrAttr.hasAttribute(Attribute::None) &&
43 InstrAttr.isStringAttribute() &&
44 InstrAttr.getValueAsString() == "xray-always";
45 Attribute Attr = F.getFnAttribute("xray-instruction-threshold");
46 unsigned XRayThreshold = 0;
47 if (!AlwaysInstrument) {
48 if (Attr.hasAttribute(Attribute::None) || !Attr.isStringAttribute())
49 return false; // XRay threshold attribute not found.
50 if (Attr.getValueAsString().getAsInteger(10, XRayThreshold))
51 return false; // Invalid value for threshold.
52 if (F.size() < XRayThreshold)
53 return false; // Function is too small.
54 }
55
56 // FIXME: Do the loop triviality analysis here or in an earlier pass.
57
58 // First, insert an PATCHABLE_FUNCTION_ENTER as the first instruction of the
59 // MachineFunction.
60 auto &FirstMBB = *MF.begin();
61 auto &FirstMI = *FirstMBB.begin();
62 auto *TII = MF.getSubtarget().getInstrInfo();
63 BuildMI(FirstMBB, FirstMI, FirstMI.getDebugLoc(),
64 TII->get(TargetOpcode::PATCHABLE_FUNCTION_ENTER));
65
66 // Then we look for *all* terminators and returns, then replace those with
67 // PATCHABLE_RET instructions.
68 SmallVector Terminators;
69 for (auto &MBB : MF) {
70 for (auto &T : MBB.terminators()) {
71 // FIXME: Handle tail calls here too?
72 if (T.isReturn() && T.getOpcode() == TII->getReturnOpcode()) {
73 // Replace return instructions with:
74 // PATCHABLE_RET , ...
75 auto MIB = BuildMI(MBB, T, T.getDebugLoc(),
76 TII->get(TargetOpcode::PATCHABLE_RET))
77 .addImm(T.getOpcode());
78 for (auto &MO : T.operands())
79 MIB.addOperand(MO);
80 Terminators.push_back(&T);
81 break;
82 }
83 }
84 }
85
86 for (auto &I : Terminators)
87 I->eraseFromParent();
88
89 return true;
90 }
91
92 char XRayInstrumentation::ID = 0;
93 char &llvm::XRayInstrumentationID = XRayInstrumentation::ID;
94 INITIALIZE_PASS(XRayInstrumentation, "xray-instrumentation", "Insert XRay ops",
95 false, false);
6868 // Emit the rest of the function body.
6969 EmitFunctionBody();
7070
71 // Emit the XRay table for this function.
72 EmitXRayTable();
73
7174 // We didn't modify anything.
7275 return false;
7376 }
7070
7171 StackMapShadowTracker SMShadowTracker;
7272
73 // This describes the kind of sled we're storing in the XRay table.
74 enum class SledKind : uint8_t {
75 FUNCTION_ENTER = 0,
76 FUNCTION_EXIT = 1,
77 TAIL_CALL = 2,
78 };
79
80 // The table will contain these structs that point to the sled, the function
81 // containing the sled, and what kind of sled (and whether they should always
82 // be instrumented).
83 struct XRayFunctionEntry {
84 const MCSymbol *Sled;
85 const MCSymbol *Function;
86 SledKind Kind;
87 bool AlwaysInstrument;
88 const class Function *Fn;
89 };
90
91 // All the sleds to be emitted.
92 std::vector Sleds;
93
7394 // All instructions emitted by the X86AsmPrinter should use this helper
7495 // method.
7596 //
85106
86107 void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI);
87108
88 public:
89 explicit X86AsmPrinter(TargetMachine &TM,
90 std::unique_ptr Streamer)
91 : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this) {}
109 // XRay-specific lowering for X86.
110 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
111 X86MCInstLower &MCIL);
112 void LowerPATCHABLE_RET(const MachineInstr &MI, X86MCInstLower &MCIL);
113 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL);
114
115 // Helper function that emits the XRay sleds we've collected for a particular
116 // function.
117 void EmitXRayTable();
118
119 // Helper function to record a given XRay sled.
120 void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind);
121 public:
122 explicit X86AsmPrinter(TargetMachine &TM,
123 std::unique_ptr Streamer)
124 : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this) {}
92125
93126 const char *getPassName() const override {
94127 return "X86 Assembly / Object Emitter";
158158 unsigned Opc = MBBI->getOpcode();
159159 switch (Opc) {
160160 default: return 0;
161 case TargetOpcode::PATCHABLE_RET:
161162 case X86::RET:
162163 case X86::RETL:
163164 case X86::RETQ:
116116 : X86::ADJCALLSTACKDOWN32),
117117 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
118118 : X86::ADJCALLSTACKUP32),
119 X86::CATCHRET),
119 X86::CATCHRET,
120 (STI.is64Bit() ? X86::RETQ : X86::RETL)),
120121 Subtarget(STI), RI(STI.getTargetTriple()) {
121122
122123 static const X86MemoryFoldTableEntry MemoryFoldTable2Addr[] = {
3535 #include "llvm/MC/MCFixup.h"
3636 #include "llvm/MC/MCInst.h"
3737 #include "llvm/MC/MCInstBuilder.h"
38 #include "llvm/MC/MCSection.h"
3839 #include "llvm/MC/MCStreamer.h"
3940 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/MC/MCSymbolELF.h"
42 #include "llvm/MC/MCSectionELF.h"
4043 #include "llvm/Support/TargetRegistry.h"
44 #include "llvm/Support/ELF.h"
45 #include "llvm/Target/TargetLoweringObjectFile.h"
46
4147 using namespace llvm;
4248
4349 namespace {
10171023 getSubtargetInfo());
10181024 }
10191025
1026 void X86AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
1027 SledKind Kind) {
1028 auto Fn = MI.getParent()->getParent()->getFunction();
1029 auto Attr = Fn->getFnAttribute("function-instrument");
1030 bool AlwaysInstrument =
1031 Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always";
1032 Sleds.emplace_back(
1033 XRayFunctionEntry{Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn});
1034 }
1035
1036 void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1037 X86MCInstLower &MCIL) {
1038 // We want to emit the following pattern:
1039 //
1040 // .Lxray_sled_N:
1041 // .palign 2, ...
1042 // jmp .tmpN
1043 // # 9 bytes worth of noops
1044 // .tmpN
1045 //
1046 // We need the 9 bytes because at runtime, we'd be patching over the full 11
1047 // bytes with the following pattern:
1048 //
1049 // mov %r10, // 6 bytes
1050 // call // 5 bytes
1051 //
1052 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1053 OutStreamer->EmitLabel(CurSled);
1054 OutStreamer->EmitCodeAlignment(4);
1055 auto Target = OutContext.createTempSymbol();
1056
1057 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1058 // an operand (computed as an offset from the jmp instruction).
1059 // FIXME: Find another less hacky way do force the relative jump.
1060 OutStreamer->EmitBytes("\xeb\x09");
1061 EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo());
1062 OutStreamer->EmitLabel(Target);
1063 recordSled(CurSled, MI, SledKind::FUNCTION_ENTER);
1064 }
1065
1066 void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1067 X86MCInstLower &MCIL) {
1068 // Since PATCHABLE_RET takes the opcode of the return statement as an
1069 // argument, we use that to emit the correct form of the RET that we want.
1070 // i.e. when we see this:
1071 //
1072 // PATCHABLE_RET X86::RET ...
1073 //
1074 // We should emit the RET followed by sleds.
1075 //
1076 // .Lxray_sled_N:
1077 // ret # or equivalent instruction
1078 // # 10 bytes worth of noops
1079 //
1080 // This just makes sure that the alignment for the next instruction is 2.
1081 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1082 OutStreamer->EmitLabel(CurSled);
1083 unsigned OpCode = MI.getOperand(0).getImm();
1084 MCInst Ret;
1085 Ret.setOpcode(OpCode);
1086 for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end()))
1087 if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1088 Ret.addOperand(MaybeOperand.getValue());
1089 OutStreamer->EmitInstruction(Ret, getSubtargetInfo());
1090 EmitNops(*OutStreamer, 10, Subtarget->is64Bit(), getSubtargetInfo());
1091 recordSled(CurSled, MI, SledKind::FUNCTION_EXIT);
1092 }
1093
1094 void X86AsmPrinter::EmitXRayTable() {
1095 if (Sleds.empty())
1096 return;
1097 if (Subtarget->isTargetELF()) {
1098 auto *Section = OutContext.getELFSection(
1099 "xray_instr_map", ELF::SHT_PROGBITS,
1100 ELF::SHF_ALLOC | ELF::SHF_GROUP | ELF::SHF_MERGE, 0,
1101 CurrentFnSym->getName());
1102 auto PrevSection = OutStreamer->getCurrentSectionOnly();
1103 OutStreamer->SwitchSection(Section);
1104 for (const auto &Sled : Sleds) {
1105 OutStreamer->EmitSymbolValue(Sled.Sled, 8);
1106 OutStreamer->EmitSymbolValue(CurrentFnSym, 8);
1107 auto Kind = static_cast(Sled.Kind);
1108 OutStreamer->EmitBytes(
1109 StringRef(reinterpret_cast(&Kind), 1));
1110 OutStreamer->EmitBytes(
1111 StringRef(reinterpret_cast(&Sled.AlwaysInstrument), 1));
1112 OutStreamer->EmitZeros(14);
1113 }
1114 OutStreamer->SwitchSection(PrevSection);
1115 }
1116 Sleds.clear();
1117 }
1118
10201119 // Returns instruction preceding MBBI in MachineFunction.
10211120 // If MBBI is the first instruction of the first basic block, returns null.
10221121 static MachineBasicBlock::const_iterator
12571356
12581357 case TargetOpcode::PATCHPOINT:
12591358 return LowerPATCHPOINT(*MI, MCInstLowering);
1359
1360 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1361 return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
1362
1363 case TargetOpcode::PATCHABLE_RET:
1364 return LowerPATCHABLE_RET(*MI, MCInstLowering);
12601365
12611366 case X86::MORESTACK_RET:
12621367 EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
0 ; RUN: llc -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
1
2 define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" {
3 ; CHECK-LABEL: Lxray_sled_0:
4 ; CHECK-NEXT: .p2align 2, 0x90
5 ; CHECK-NEXT: .ascii "\353\t"
6 ; CHECK-NEXT: nopw 512(%rax,%rax)
7 ; CHECK-LABEL: Ltmp0:
8 ret i32 0
9 ; CHECK-LABEL: Lxray_sled_1:
10 ; CHECK-NEXT: retq
11 ; CHECK-NEXT: nopw %cs:512(%rax,%rax)
12 }
0 ; RUN: llc -mcpu=nehalem < %s | not grep xray_sled_
1
2 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
3 target triple = "x86_64-apple-darwin8"
4
5 define i32 @foo() nounwind uwtable "xray-instruction-threshold"="3" {
6 entry:
7 ret i32 0
8 }
0 ; RUN: llc -mcpu=nehalem < %s | grep xray_sled_
1
2 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
3 target triple = "x86_64-apple-darwin8"
4
5 define i32 @foo() nounwind uwtable "xray-instruction-threshold"="1" {
6 entry:
7 ret i32 0
8 }
427427 OS << "namespace llvm {\n";
428428 OS << "struct " << ClassName << " : public TargetInstrInfo {\n"
429429 << " explicit " << ClassName
430 << "(int CFSetupOpcode = -1, int CFDestroyOpcode = -1, int CatchRetOpcode = -1);\n"
430 << "(int CFSetupOpcode = -1, int CFDestroyOpcode = -1, int CatchRetOpcode = -1, int ReturnOpcode = -1);\n"
431431 << " ~" << ClassName << "() override {}\n"
432432 << "};\n";
433433 OS << "} // end llvm namespace\n";
442442 OS << "extern const unsigned " << TargetName << "InstrNameIndices[];\n";
443443 OS << "extern const char " << TargetName << "InstrNameData[];\n";
444444 OS << ClassName << "::" << ClassName
445 << "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode)\n"
446 << " : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode) {\n"
445 << "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode, int ReturnOpcode)\n"
446 << " : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, ReturnOpcode) {\n"
447447 << " InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
448448 << "InstrNameIndices, " << TargetName << "InstrNameData, "
449449 << NumberedInstructions.size() << ");\n}\n";