llvm.org GIT mirror llvm / d2b1fb1
[x86/SLH] Add a real Clang flag and LLVM IR attribute for Speculative Load Hardening. Wires up the existing pass to work with a proper IR attribute rather than just a hidden/internal flag. The internal flag continues to work for now, but I'll likely remove it soon. Most of the churn here is adding the IR attribute. I talked about this Kristof Beyls and he seemed at least initially OK with this direction. The idea of using a full attribute here is that we *do* expect at least some forms of this for other architectures. There isn't anything *inherently* x86-specific about this technique, just that we only have an implementation for x86 at the moment. While we could potentially expose this as a Clang-level attribute as well, that seems like a good question to defer for the moment as it isn't 100% clear whether that or some other programmer interface (or both?) would be best. We'll defer the programmer interface side of this for now, but at least get to the point where the feature can be enabled without relying on implementation details. This also allows us to do something that was really hard before: we can enable *just* the indirect call retpolines when using SLH. For x86, we don't have any other way to mitigate indirect calls. Other architectures may take a different approach of course, and none of this is surfaced to user-level flags. Differential Revision: https://reviews.llvm.org/D51157 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@341363 91177308-0d34-0410-b5e6-96231b3b80d8 Chandler Carruth 2 years ago
19 changed file(s) with 105 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
16351635 This attribute indicates that HWAddressSanitizer checks
16361636 (dynamic address safety analysis based on tagged pointers) are enabled for
16371637 this function.
1638 ``speculative_load_hardening``
1639 This attribute indicates that
1640 `Speculative Load Hardening `_
1641 should be enabled for the function body. This is a best-effort attempt to
1642 mitigate all known speculative execution information leak vulnerabilities
1643 that are based on the fundamental principles of modern processors'
1644 speculative execution. These vulnerabilities are classified as "Spectre
1645 variant #1" vulnerabilities typically. Notably, this does not attempt to
1646 mitigate any vulnerabilities where the speculative execution and/or
1647 prediction devices of specific processors can be *completely* undermined
1648 (such as "Branch Target Injection", a.k.a, "Spectre variant #2"). Instead,
1649 this is a target-independent request to harden against the completely
1650 generic risk posed by speculative execution to incorrectly load secret data,
1651 making it available to some micro-architectural side-channel for information
1652 leak. For a processor without any speculative execution or predictors, this
1653 is expected to be a no-op.
1654
1655 When inlining, the attribute is sticky. Inlining a function that carries
1656 this attribute will cause the caller to gain the attribute. This is intended
1657 to provide a maximally conservative model where the code in a function
1658 annotated with this attribute will always (even after inlining) end up
1659 hardened.
16381660 ``speculatable``
16391661 This function attribute indicates that the function does not have any
16401662 effects besides calculating its result and does not have undefined behavior.
590590 ATTR_KIND_NOCF_CHECK = 56,
591591 ATTR_KIND_OPT_FOR_FUZZING = 57,
592592 ATTR_KIND_SHADOWCALLSTACK = 58,
593 ATTR_KIND_SPECULATIVE_LOAD_HARDENING = 59,
593594 };
594595
595596 enum ComdatSelectionKindCodes {
174174
175175 /// HWAddressSanitizer is on.
176176 def SanitizeHWAddress : EnumAttr<"sanitize_hwaddress">;
177
178 /// Speculative Load Hardening is enabled.
179 ///
180 /// Note that this uses the default compatibility (always compatible during
181 /// inlining) and a conservative merge strategy where inlining an attributed
182 /// body will add the attribute to the caller. This ensures that code carrying
183 /// this attribute will always be lowered with hardening enabled.
184 def SpeculativeLoadHardening : EnumAttr<"speculative_load_hardening">;
177185
178186 /// Argument is swift error.
179187 def SwiftError : EnumAttr<"swifterror">;
231239 def : MergeRule<"setOR">;
232240 def : MergeRule<"setOR">;
233241 def : MergeRule<"setOR">;
242 def : MergeRule<"setOR">;
234243 def : MergeRule<"adjustCallerSSPLevel">;
235244 def : MergeRule<"adjustCallerStackProbes">;
236245 def : MergeRule<"adjustCallerStackProbeSize">;
677677 KEYWORD(sanitize_hwaddress);
678678 KEYWORD(sanitize_thread);
679679 KEYWORD(sanitize_memory);
680 KEYWORD(speculative_load_hardening);
680681 KEYWORD(swifterror);
681682 KEYWORD(swiftself);
682683 KEYWORD(uwtable);
12751275 B.addAttribute(Attribute::SanitizeThread); break;
12761276 case lltok::kw_sanitize_memory:
12771277 B.addAttribute(Attribute::SanitizeMemory); break;
1278 case lltok::kw_speculative_load_hardening:
1279 B.addAttribute(Attribute::SpeculativeLoadHardening);
1280 break;
12781281 case lltok::kw_strictfp: B.addAttribute(Attribute::StrictFP); break;
12791282 case lltok::kw_uwtable: B.addAttribute(Attribute::UWTable); break;
12801283 case lltok::kw_writeonly: B.addAttribute(Attribute::WriteOnly); break;
16181621 case lltok::kw_sanitize_hwaddress:
16191622 case lltok::kw_sanitize_memory:
16201623 case lltok::kw_sanitize_thread:
1624 case lltok::kw_speculative_load_hardening:
16211625 case lltok::kw_ssp:
16221626 case lltok::kw_sspreq:
16231627 case lltok::kw_sspstrong:
17141718 case lltok::kw_sanitize_hwaddress:
17151719 case lltok::kw_sanitize_memory:
17161720 case lltok::kw_sanitize_thread:
1721 case lltok::kw_speculative_load_hardening:
17171722 case lltok::kw_ssp:
17181723 case lltok::kw_sspreq:
17191724 case lltok::kw_sspstrong:
218218 kw_sret,
219219 kw_sanitize_thread,
220220 kw_sanitize_memory,
221 kw_speculative_load_hardening,
221222 kw_strictfp,
222223 kw_swifterror,
223224 kw_swiftself,
11641164 case Attribute::NoCfCheck: return 1ULL << 57;
11651165 case Attribute::OptForFuzzing: return 1ULL << 58;
11661166 case Attribute::ShadowCallStack: return 1ULL << 59;
1167 case Attribute::SpeculativeLoadHardening:
1168 return 1ULL << 60;
11671169 case Attribute::Dereferenceable:
11681170 llvm_unreachable("dereferenceable attribute not supported in raw format");
11691171 break;
13881390 return Attribute::SanitizeThread;
13891391 case bitc::ATTR_KIND_SANITIZE_MEMORY:
13901392 return Attribute::SanitizeMemory;
1393 case bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING:
1394 return Attribute::SpeculativeLoadHardening;
13911395 case bitc::ATTR_KIND_SWIFT_ERROR:
13921396 return Attribute::SwiftError;
13931397 case bitc::ATTR_KIND_SWIFT_SELF:
689689 return bitc::ATTR_KIND_SANITIZE_THREAD;
690690 case Attribute::SanitizeMemory:
691691 return bitc::ATTR_KIND_SANITIZE_MEMORY;
692 case Attribute::SpeculativeLoadHardening:
693 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
692694 case Attribute::SwiftError:
693695 return bitc::ATTR_KIND_SWIFT_ERROR;
694696 case Attribute::SwiftSelf:
322322 return "returns_twice";
323323 if (hasAttribute(Attribute::SExt))
324324 return "signext";
325 if (hasAttribute(Attribute::SpeculativeLoadHardening))
326 return "speculative_load_hardening";
325327 if (hasAttribute(Attribute::Speculatable))
326328 return "speculatable";
327329 if (hasAttribute(Attribute::StackProtect))
14771477 case Attribute::InaccessibleMemOnly:
14781478 case Attribute::InaccessibleMemOrArgMemOnly:
14791479 case Attribute::AllocSize:
1480 case Attribute::SpeculativeLoadHardening:
14801481 case Attribute::Speculatable:
14811482 case Attribute::StrictFP:
14821483 return true;
7474 STATISTIC(NumInstsInserted, "Number of instructions inserted");
7575 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
7676
77 static cl::opt EnableSpeculativeLoadHardening(
78 "x86-speculative-load-hardening",
79 cl::desc("Force enable speculative load hardening"), cl::init(false),
80 cl::Hidden);
81
7782 static cl::opt HardenEdgesWithLFENCE(
7883 PASS_KEY "-lfence",
7984 cl::desc(
402407 MachineFunction &MF) {
403408 LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
404409 << " **********\n");
410
411 // Only run if this pass is forced enabled or we detect the relevant function
412 // attribute requesting SLH.
413 if (!EnableSpeculativeLoadHardening &&
414 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
415 return false;
405416
406417 Subtarget = &MF.getSubtarget();
407418 MRI = &MF.getRegInfo();
5353 cl::desc("Enable the machine combiner pass"),
5454 cl::init(true), cl::Hidden);
5555
56 static cl::opt EnableSpeculativeLoadHardening(
57 "x86-speculative-load-hardening",
58 cl::desc("Enable speculative load hardening"), cl::init(false), cl::Hidden);
59
6056 namespace llvm {
6157
6258 void initializeWinEHStatePassPass(PassRegistry &);
474470 addPass(createX86AvoidStoreForwardingBlocks());
475471 }
476472
477 if (EnableSpeculativeLoadHardening)
478 addPass(createX86SpeculativeLoadHardeningPass());
479
473 addPass(createX86SpeculativeLoadHardeningPass());
480474 addPass(createX86FlagsCopyLoweringPass());
481475 addPass(createX86WinAllocaExpander());
482476 }
5757 .Case("sanitize_hwaddress", Attribute::SanitizeHWAddress)
5858 .Case("sanitize_memory", Attribute::SanitizeMemory)
5959 .Case("sanitize_thread", Attribute::SanitizeThread)
60 .Case("speculative_load_hardening", Attribute::SpeculativeLoadHardening)
6061 .Case("ssp", Attribute::StackProtect)
6162 .Case("sspreq", Attribute::StackProtectReq)
6263 .Case("sspstrong", Attribute::StackProtectStrong)
752752 case Attribute::SanitizeMemory:
753753 case Attribute::SanitizeThread:
754754 case Attribute::SanitizeHWAddress:
755 case Attribute::SpeculativeLoadHardening:
755756 case Attribute::StackProtect:
756757 case Attribute::StackProtectReq:
757758 case Attribute::StackProtectStrong:
3636 ; CHECK-NEXT: X86 PIC Global Base Reg Initialization
3737 ; CHECK-NEXT: Expand ISel Pseudo-instructions
3838 ; CHECK-NEXT: Local Stack Slot Allocation
39 ; CHECK-NEXT: X86 speculative load hardening
3940 ; CHECK-NEXT: MachineDominator Tree Construction
4041 ; CHECK-NEXT: X86 EFLAGS copy lowering
4142 ; CHECK-NEXT: X86 WinAlloca Expander
8989 ; CHECK-NEXT: X86 LEA Optimize
9090 ; CHECK-NEXT: X86 Optimize Call Frame
9191 ; CHECK-NEXT: X86 Avoid Store Forwarding Block
92 ; CHECK-NEXT: X86 speculative load hardening
9293 ; CHECK-NEXT: MachineDominator Tree Construction
9394 ; CHECK-NEXT: X86 EFLAGS copy lowering
9495 ; CHECK-NEXT: X86 WinAlloca Expander
0 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
1 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -x86-speculative-load-hardening | FileCheck %s
1 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
22
33 declare <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float>, i8*, <4 x i32>, <4 x float>, i8)
44
949949 ret <4 x i64> %v
950950 }
951951
952 attributes #0 = { nounwind "target-features"="+avx2" }
953 attributes #1 = { nounwind "target-features"="+avx512f" }
954 attributes #2 = { nounwind "target-features"="+avx512vl" }
952 attributes #0 = { nounwind speculative_load_hardening "target-features"="+avx2" }
953 attributes #1 = { nounwind speculative_load_hardening "target-features"="+avx512f" }
954 attributes #2 = { nounwind speculative_load_hardening "target-features"="+avx512vl" }
0 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
1 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -x86-speculative-load-hardening | FileCheck %s --check-prefix=X64
2 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -x86-speculative-load-hardening -x86-slh-lfence | FileCheck %s --check-prefix=X64-LFENCE
1 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=X64
2 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -x86-slh-lfence | FileCheck %s --check-prefix=X64-LFENCE
33 ;
44 ; FIXME: Add support for 32-bit and other EH ABIs.
55
77
88 declare void @sink(i32)
99
10 define i32 @test_trivial_entry_load(i32* %ptr) {
10 define i32 @test_trivial_entry_load(i32* %ptr) speculative_load_hardening {
1111 ; X64-LABEL: test_trivial_entry_load:
1212 ; X64: # %bb.0: # %entry
1313 ; X64-NEXT: movq %rsp, %rcx
2828 ret i32 %v
2929 }
3030
31 define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %ptr2, i32** %ptr3) {
31 define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %ptr2, i32** %ptr3) speculative_load_hardening {
3232 ; X64-LABEL: test_basic_conditions:
3333 ; X64: # %bb.0: # %entry
3434 ; X64-NEXT: pushq %r15
192192 ret void
193193 }
194194
195 define void @test_basic_loop(i32 %a, i32 %b, i32* %ptr1, i32* %ptr2) nounwind {
195 define void @test_basic_loop(i32 %a, i32 %b, i32* %ptr1, i32* %ptr2) nounwind speculative_load_hardening {
196196 ; X64-LABEL: test_basic_loop:
197197 ; X64: # %bb.0: # %entry
198198 ; X64-NEXT: pushq %rbp
300300 ret void
301301 }
302302
303 define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %ptr2) nounwind {
303 define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, i32* %ptr1, i32* %ptr2) nounwind speculative_load_hardening {
304304 ; X64-LABEL: test_basic_nested_loop:
305305 ; X64: # %bb.0: # %entry
306306 ; X64-NEXT: pushq %rbp
496496
497497 declare void @__cxa_throw(i8*, i8*, i8*) local_unnamed_addr
498498
499 define void @test_basic_eh(i32 %a, i32* %ptr1, i32* %ptr2) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
499 define void @test_basic_eh(i32 %a, i32* %ptr1, i32* %ptr2) speculative_load_hardening personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
500500 ; X64-LABEL: test_basic_eh:
501501 ; X64: # %bb.0: # %entry
502502 ; X64-NEXT: pushq %rbp
667667 declare void @sink_double(double)
668668
669669 ; Test direct and converting loads of floating point values.
670 define void @test_fp_loads(float* %fptr, double* %dptr, i32* %i32ptr, i64* %i64ptr) nounwind {
670 define void @test_fp_loads(float* %fptr, double* %dptr, i32* %i32ptr, i64* %i64ptr) nounwind speculative_load_hardening {
671671 ; X64-LABEL: test_fp_loads:
672672 ; X64: # %bb.0: # %entry
673673 ; X64-NEXT: pushq %r15
854854 declare void @sink_v2i64(<2 x i64>)
855855
856856 ; Test loads of vectors.
857 define void @test_vec_loads(<4 x float>* %v4f32ptr, <2 x double>* %v2f64ptr, <16 x i8>* %v16i8ptr, <8 x i16>* %v8i16ptr, <4 x i32>* %v4i32ptr, <2 x i64>* %v2i64ptr) nounwind {
857 define void @test_vec_loads(<4 x float>* %v4f32ptr, <2 x double>* %v2f64ptr, <16 x i8>* %v16i8ptr, <8 x i16>* %v8i16ptr, <4 x i32>* %v4i32ptr, <2 x i64>* %v2i64ptr) nounwind speculative_load_hardening {
858858 ; X64-LABEL: test_vec_loads:
859859 ; X64: # %bb.0: # %entry
860860 ; X64-NEXT: pushq %rbp
995995 ret void
996996 }
997997
998 define void @test_deferred_hardening(i32* %ptr1, i32* %ptr2, i32 %x) nounwind {
998 define void @test_deferred_hardening(i32* %ptr1, i32* %ptr2, i32 %x) nounwind speculative_load_hardening {
999999 ; X64-LABEL: test_deferred_hardening:
10001000 ; X64: # %bb.0: # %entry
10011001 ; X64-NEXT: pushq %r15
2222 }
2323
2424 define i32 @safestack_callee(i32 %i) safestack {
25 ret i32 %i
26 }
27
28 define i32 @slh_callee(i32 %i) speculative_load_hardening {
2529 ret i32 %i
2630 }
2731
158162 ; CHECK-LABEL: @test_safestack(
159163 ; CHECK-NEXT: @noattr_callee
160164 ; CHECK-NEXT: ret i32
165 }
166
167 ; Can inline a normal function into an SLH'ed function.
168 define i32 @test_caller_slh(i32 %i) speculative_load_hardening {
169 ; CHECK-LABEL: @test_caller_slh(
170 ; CHECK-SAME: ) [[SLH:.*]] {
171 ; CHECK-NOT: call
172 ; CHECK: ret i32
173 entry:
174 %callee = call i32 @noattr_callee(i32 %i)
175 ret i32 %callee
176 }
177
178 ; Can inline a SLH'ed function into a normal one, propagating SLH.
179 define i32 @test_callee_slh(i32 %i) {
180 ; CHECK-LABEL: @test_callee_slh(
181 ; CHECK-SAME: ) [[SLH:.*]] {
182 ; CHECK-NOT: call
183 ; CHECK: ret i32
184 entry:
185 %callee = call i32 @slh_callee(i32 %i)
186 ret i32 %callee
161187 }
162188
163189 ; Check that a function doesn't get inlined if target-cpu strings don't match
383409 ; CHECK-NEXT: ret i32
384410 }
385411
412 ; CHECK: attributes [[SLH]] = { speculative_load_hardening }
386413 ; CHECK: attributes [[FPMAD_FALSE]] = { "less-precise-fpmad"="false" }
387414 ; CHECK: attributes [[FPMAD_TRUE]] = { "less-precise-fpmad"="true" }
388415 ; CHECK: attributes [[NOIMPLICITFLOAT]] = { noimplicitfloat }