llvm.org GIT mirror llvm / 35f9e1a
[X86] Add support for Read Time Stamp Counter x86 builtin intrinsics. This patch: - Adds two new X86 builtin intrinsics ('int_x86_rdtsc' and 'int_x86_rdtscp') as GCCBuiltin intrinsics; - Teaches the backend how to lower the two new builtins; - Introduces a common function to lower READCYCLECOUNTER dag nodes and the two new rdtsc/rdtscp intrinsics; - Improves (and extends) the existing x86 test 'rdtsc.ll'; now test 'rdtsc.ll' correctly verifies that both READCYCLECOUNTER and the two new intrinsics work fine for both 64bit and 32bit Subtargets. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@207127 91177308-0d34-0410-b5e6-96231b3b80d8 Andrea Di Biagio 5 years ago
6 changed file(s) with 160 addition(s) and 39 deletion(s). Raw diff Collapse all Expand all
1414 // Interrupt traps
1515 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
1616 def int_x86_int : Intrinsic<[], [llvm_i8_ty]>;
17 }
18
19 //===----------------------------------------------------------------------===//
20 // Read Time Stamp Counter.
21 let TargetPrefix = "x86" in {
22 def int_x86_rdtsc : GCCBuiltin<"__builtin_ia32_rdtsc">,
23 Intrinsic<[llvm_i64_ty], [], []>;
24 def int_x86_rdtscp : GCCBuiltin<"__builtin_ia32_rdtscp">,
25 Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrReadWriteArgMem]>;
1726 }
1827
1928 //===----------------------------------------------------------------------===//
14711471 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
14721472 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
14731473 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1474 if (!Subtarget->is64Bit())
1475 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
14741476
14751477 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
14761478 // handle type legalization for these operations here.
1226012262 return SDValue(Res, 1);
1226112263 }
1226212264
12265 // getReadTimeStampCounter - Handles the lowering of builtin intrinsics that
12266 // read the time stamp counter (x86_rdtsc and x86_rdtscp). This function is
12267 // also used to custom lower READCYCLECOUNTER nodes.
12268 static void getReadTimeStampCounter(SDNode *N, SDLoc DL, unsigned Opcode,
12269 SelectionDAG &DAG, const X86Subtarget *Subtarget,
12270 SmallVectorImpl &Results) {
12271 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
12272 SDValue TheChain = N->getOperand(0);
12273 SDValue rd = DAG.getNode(Opcode, DL, Tys, &TheChain, 1);
12274 SDValue LO, HI;
12275
12276 // The processor's time-stamp counter (a 64-bit MSR) is stored into the
12277 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
12278 // and the EAX register is loaded with the low-order 32 bits.
12279 if (Subtarget->is64Bit()) {
12280 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
12281 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
12282 LO.getValue(2));
12283 } else {
12284 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
12285 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
12286 LO.getValue(2));
12287 }
12288 SDValue Chain = HI.getValue(1);
12289
12290 if (Opcode == X86ISD::RDTSCP_DAG) {
12291 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
12292
12293 // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
12294 // the ECX register. Add 'ecx' explicitly to the chain.
12295 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32,
12296 HI.getValue(2));
12297 // Explicitly store the content of ECX at the location passed in input
12298 // to the 'rdtscp' intrinsic.
12299 Chain = DAG.getStore(ecx.getValue(1), DL, ecx, N->getOperand(2),
12300 MachinePointerInfo(), false, false, 0);
12301 }
12302
12303 if (Subtarget->is64Bit()) {
12304 // The EDX register is loaded with the high-order 32 bits of the MSR, and
12305 // the EAX register is loaded with the low-order 32 bits.
12306 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
12307 DAG.getConstant(32, MVT::i8));
12308 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
12309 Results.push_back(Chain);
12310 return;
12311 }
12312
12313 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
12314 SDValue Ops[] = { LO, HI };
12315 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops,
12316 array_lengthof(Ops));
12317 Results.push_back(Pair);
12318 Results.push_back(Chain);
12319 }
12320
12321 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
12322 SelectionDAG &DAG) {
12323 SmallVector Results;
12324 SDLoc DL(Op);
12325 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
12326 Results);
12327 return DAG.getMergeValues(&Results[0], Results.size(), DL);
12328 }
12329
1226312330 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
1226412331 SelectionDAG &DAG) {
1226512332 SDLoc dl(Op);
1243312500 SDValue Src = Op.getOperand(5);
1243412501 SDValue Scale = Op.getOperand(6);
1243512502 return getMScatterNode(Opc, Op, DAG, Src, Mask, Base, Index, Scale, Chain);
12503 }
12504 // Read Time Stamp Counter (RDTSC).
12505 case Intrinsic::x86_rdtsc:
12506 // Read Time Stamp Counter and Processor ID (RDTSCP).
12507 case Intrinsic::x86_rdtscp: {
12508 unsigned Opc;
12509 switch (IntNo) {
12510 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
12511 case Intrinsic::x86_rdtsc:
12512 Opc = X86ISD::RDTSC_DAG; break;
12513 case Intrinsic::x86_rdtscp:
12514 Opc = X86ISD::RDTSCP_DAG; break;
12515 }
12516 SmallVector Results;
12517 getReadTimeStampCounter(Op.getNode(), dl, Opc, DAG, Subtarget, Results);
12518 return DAG.getMergeValues(&Results[0], Results.size(), dl);
1243612519 }
1243712520 // XTEST intrinsics.
1243812521 case Intrinsic::x86_xtest: {
1380213885 SDValue cpOut =
1380313886 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
1380413887 return cpOut;
13805 }
13806
13807 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
13808 SelectionDAG &DAG) {
13809 assert(Subtarget->is64Bit() && "Result not type legalized?");
13810 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
13811 SDValue TheChain = Op.getOperand(0);
13812 SDLoc dl(Op);
13813 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
13814 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1));
13815 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64,
13816 rax.getValue(2));
13817 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx,
13818 DAG.getConstant(32, MVT::i8));
13819 SDValue Ops[] = {
13820 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp),
13821 rdx.getValue(1)
13822 };
13823 return DAG.getMergeValues(Ops, array_lengthof(Ops), dl);
1382413888 }
1382513889
1382613890 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget *Subtarget,
1415714221 Results.push_back(V);
1415814222 return;
1415914223 }
14224 case ISD::INTRINSIC_W_CHAIN: {
14225 unsigned IntNo = cast(N->getOperand(1))->getZExtValue();
14226 switch (IntNo) {
14227 default : llvm_unreachable("Do not know how to custom type "
14228 "legalize this intrinsic operation!");
14229 case Intrinsic::x86_rdtsc:
14230 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
14231 Results);
14232 case Intrinsic::x86_rdtscp:
14233 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
14234 Results);
14235 }
14236 }
1416014237 case ISD::READCYCLECOUNTER: {
14161 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
14162 SDValue TheChain = N->getOperand(0);
14163 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
14164 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32,
14165 rd.getValue(1));
14166 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32,
14167 eax.getValue(2));
14168 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
14169 SDValue Ops[] = { eax, edx };
14170 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops,
14171 array_lengthof(Ops)));
14172 Results.push_back(edx.getValue(1));
14173 return;
14238 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
14239 Results);
1417414240 }
1417514241 case ISD::ATOMIC_CMP_SWAP: {
1417614242 EVT T = N->getValueType(0);
8181 /// RDTSC_DAG - This operation implements the lowering for
8282 /// readcyclecounter
8383 RDTSC_DAG,
84
85 /// X86 Read Time-Stamp Counter and Processor ID.
86 RDTSCP_DAG,
8487
8588 /// X86 compare and logical compare instructions.
8689 CMP, COMI, UCOMI,
204204 SDNPMayLoad]>;
205205
206206 def X86rdtsc : SDNode<"X86ISD::RDTSC_DAG", SDTX86Void,
207 [SDNPHasChain, SDNPOutGlue, SDNPSideEffect]>;
208 def X86rdtscp : SDNode<"X86ISD::RDTSCP_DAG", SDTX86Void,
207209 [SDNPHasChain, SDNPOutGlue, SDNPSideEffect]>;
208210
209211 def X86Wrapper : SDNode<"X86ISD::Wrapper", SDTX86Wrapper>;
1818 TB;
1919
2020 let Defs = [RAX, RCX, RDX] in
21 def RDTSCP : I<0x01, MRM_F9, (outs), (ins), "rdtscp", []>, TB;
21 def RDTSCP : I<0x01, MRM_F9, (outs), (ins), "rdtscp", [(X86rdtscp)]>, TB;
2222
2323 // CPU flow control instructions
2424
None ; RUN: llc < %s -march=x86 | grep rdtsc
1 ; RUN: llc < %s -march=x86-64 | grep rdtsc
0 ; RUN: llc < %s -march=x86-64 -mcpu=generic | FileCheck %s
1 ; RUN: llc < %s -march=x86 -mcpu=generic | FileCheck %s --check-prefix=CHECK --check-prefix=X86
2
3 ; Verify that we correctly lower ISD::READCYCLECOUNTER.
4
5
6 define i64 @test_builtin_readcyclecounter() {
7 %1 = tail call i64 @llvm.readcyclecounter()
8 ret i64 %1
9 }
10 ; CHECK-LABEL: test_builtin_readcyclecounter
11 ; CHECK: rdtsc
12 ; X86-NOT: shlq
13 ; X86-NOT: or
14 ; CHECK-NOT: mov
15 ; CHECK: ret
16
17
18 ; Verify that we correctly lower the Read Cycle Counter GCC x86 builtins
19 ; (i.e. RDTSC and RDTSCP).
20
21 define i64 @test_builtin_rdtsc() {
22 %1 = tail call i64 @llvm.x86.rdtsc()
23 ret i64 %1
24 }
25 ; CHECK-LABEL: test_builtin_rdtsc
26 ; CHECK: rdtsc
27 ; X86-NOT: shlq
28 ; X86-NOT: or
29 ; CHECK-NOT: mov
30 ; CHECK: ret
31
32
33 define i64 @test_builtin_rdtscp(i8* %A) {
34 %1 = tail call i64 @llvm.x86.rdtscp(i8* %A)
35 ret i64 %1
36 }
37 ; CHECK-LABEL: test_builtin_rdtscp
38 ; CHECK: rdtscp
39 ; X86-NOT: shlq
40 ; CHECK: movl %ecx, (%{{[a-z]+}})
41 ; X86-NOT: shlq
42 ; CHECK: ret
43
44
245 declare i64 @llvm.readcyclecounter()
46 declare i64 @llvm.x86.rdtscp(i8*)
47 declare i64 @llvm.x86.rdtsc()
348
4 define i64 @foo() {
5 %tmp.1 = call i64 @llvm.readcyclecounter( ) ; [#uses=1]
6 ret i64 %tmp.1
7 }