llvm.org GIT mirror llvm / aebcb9c
Disable loop peeling during full unrolling pass. Summary: Peeling should not occur during the full unrolling invocation early in the pipeline, but rather later with partial and runtime loop unrolling. The later loop unrolling invocation will also eventually utilize profile summary and branch frequency information, which we would like to use to control peeling. And for ThinLTO we want to delay peeling until the backend (post thin link) phase, just as we do for most types of unrolling. Ensure peeling doesn't occur during the full unrolling invocation by adding a parameter to the shared implementation function, similar to the way partial and runtime loop unrolling are disabled. Performance results for ThinLTO suggest this has a neutral to positive effect on some internal benchmarks. Reviewers: chandlerc, davidxl Subscribers: mzolotukhin, llvm-commits, mehdi_amini Differential Revision: https://reviews.llvm.org/D36258 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@309966 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson 2 years ago
3 changed file(s) with 33 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
183183 //
184184 Pass *createLoopUnrollPass(int OptLevel = 2, int Threshold = -1, int Count = -1,
185185 int AllowPartial = -1, int Runtime = -1,
186 int UpperBound = -1);
186 int UpperBound = -1, int AllowPeeling = -1);
187187 // Create an unrolling pass for full unrolling that uses exact trip count only.
188188 Pass *createSimpleLoopUnrollPass(int OptLevel = 2);
189189
133133 Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, int OptLevel,
134134 Optional UserThreshold, Optional UserCount,
135135 Optional UserAllowPartial, Optional UserRuntime,
136 Optional UserUpperBound) {
136 Optional UserUpperBound, Optional UserAllowPeeling) {
137137 TargetTransformInfo::UnrollingPreferences UP;
138138
139139 // Set up the defaults
200200 UP.Runtime = *UserRuntime;
201201 if (UserUpperBound.hasValue())
202202 UP.UpperBound = *UserUpperBound;
203 if (UserAllowPeeling.hasValue())
204 UP.AllowPeeling = *UserAllowPeeling;
203205
204206 return UP;
205207 }
926928 return ExplicitUnroll;
927929 }
928930
929 static bool tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
930 ScalarEvolution &SE, const TargetTransformInfo &TTI,
931 AssumptionCache &AC, OptimizationRemarkEmitter &ORE,
932 bool PreserveLCSSA, int OptLevel,
933 Optional ProvidedCount,
934 Optional ProvidedThreshold,
935 Optional ProvidedAllowPartial,
936 Optional ProvidedRuntime,
937 Optional ProvidedUpperBound) {
931 static bool tryToUnrollLoop(
932 Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
933 const TargetTransformInfo &TTI, AssumptionCache &AC,
934 OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, int OptLevel,
935 Optional ProvidedCount, Optional ProvidedThreshold,
936 Optional ProvidedAllowPartial, Optional ProvidedRuntime,
937 Optional ProvidedUpperBound, Optional ProvidedAllowPeeling) {
938938 DEBUG(dbgs() << "Loop Unroll: F[" << L->getHeader()->getParent()->getName()
939939 << "] Loop %" << L->getHeader()->getName() << "\n");
940940 if (HasUnrollDisablePragma(L))
950950 bool Convergent;
951951 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
952952 L, SE, TTI, OptLevel, ProvidedThreshold, ProvidedCount,
953 ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound);
953 ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
954 ProvidedAllowPeeling);
954955 // Exit early if unrolling is disabled.
955956 if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0))
956957 return false;
10521053 LoopUnroll(int OptLevel = 2, Optional Threshold = None,
10531054 Optional Count = None,
10541055 Optional AllowPartial = None, Optional Runtime = None,
1055 Optional UpperBound = None)
1056 Optional UpperBound = None,
1057 Optional AllowPeeling = None)
10561058 : LoopPass(ID), OptLevel(OptLevel), ProvidedCount(std::move(Count)),
10571059 ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial),
1058 ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound) {
1060 ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound),
1061 ProvidedAllowPeeling(AllowPeeling) {
10591062 initializeLoopUnrollPass(*PassRegistry::getPassRegistry());
10601063 }
10611064
10651068 Optional ProvidedAllowPartial;
10661069 Optional ProvidedRuntime;
10671070 Optional ProvidedUpperBound;
1071 Optional ProvidedAllowPeeling;
10681072
10691073 bool runOnLoop(Loop *L, LPPassManager &) override {
10701074 if (skipLoop(L))
10871091 return tryToUnrollLoop(L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel,
10881092 ProvidedCount, ProvidedThreshold,
10891093 ProvidedAllowPartial, ProvidedRuntime,
1090 ProvidedUpperBound);
1094 ProvidedUpperBound, ProvidedAllowPeeling);
10911095 }
10921096
10931097 /// This transformation requires natural loop information & requires that
11111115 INITIALIZE_PASS_END(LoopUnroll, "loop-unroll", "Unroll loops", false, false)
11121116
11131117 Pass *llvm::createLoopUnrollPass(int OptLevel, int Threshold, int Count,
1114 int AllowPartial, int Runtime,
1115 int UpperBound) {
1118 int AllowPartial, int Runtime, int UpperBound,
1119 int AllowPeeling) {
11161120 // TODO: It would make more sense for this function to take the optionals
11171121 // directly, but that's dangerous since it would silently break out of tree
11181122 // callers.
11211125 Count == -1 ? None : Optional(Count),
11221126 AllowPartial == -1 ? None : Optional(AllowPartial),
11231127 Runtime == -1 ? None : Optional(Runtime),
1124 UpperBound == -1 ? None : Optional(UpperBound));
1128 UpperBound == -1 ? None : Optional(UpperBound),
1129 AllowPeeling == -1 ? None : Optional(AllowPeeling));
11251130 }
11261131
11271132 Pass *llvm::createSimpleLoopUnrollPass(int OptLevel) {
1128 return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0);
1133 return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0, 0);
11291134 }
11301135
11311136 PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM,
11551160 tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE,
11561161 /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
11571162 /*Threshold*/ None, /*AllowPartial*/ false,
1158 /*Runtime*/ false, /*UpperBound*/ false);
1163 /*Runtime*/ false, /*UpperBound*/ false,
1164 /*AllowPeeling*/ false);
11591165 if (!Changed)
11601166 return PreservedAnalyses::all();
11611167
12771283 bool CurChanged = tryToUnrollLoop(
12781284 &L, DT, &LI, SE, TTI, AC, ORE,
12791285 /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
1280 /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam);
1286 /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam,
1287 /*AllowPeeling*/ None);
12811288 Changed |= CurChanged;
12821289
12831290 // The parent must not be damaged by unrolling!
0 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s
1 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,unroll' 2>&1 | FileCheck %s
12 ; REQUIRES: asserts
23
34 ; Make sure we use the profile information correctly to peel-off 3 iterations
78 ; CHECK: PEELING loop %for.body with iteration count 3!
89 ; CHECK: Loop Unroll: F[optsize]
910 ; CHECK-NOT: PEELING
11
12 ; Confirm that no peeling occurs when we are performing full unrolling.
13 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=FULLUNROLL
14 ; FULLUNROLL-NOT: PEELING
1015
1116 ; CHECK-LABEL: @basic
1217 ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !1