llvm.org GIT mirror llvm / 49baa9f
Switch lowering: use profile info to build weight-balanced binary search trees This will cause hot nodes to appear closer to the root. The literature says building the tree like this makes it a near-optimal (in terms of search time given key frequencies) binary search tree. In LLVM's case, we can do up to 3 comparisons in each leaf node, so it might be better to opt for lower tree height in some cases; that's something to look into in the future. Differential Revision: http://reviews.llvm.org/D9318 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@236192 91177308-0d34-0410-b5e6-96231b3b80d8 Hans Wennborg 5 years ago
3 changed file(s) with 158 addition(s) and 9 deletion(s). Raw diff Collapse all Expand all
79857985 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
79867986 assert(NumClusters >= 2 && "Too small to split!");
79877987
7988 // FIXME: When we have profile info, we might want to balance the tree based
7989 // on weights instead of node count.
7990
7991 CaseClusterIt PivotCluster = W.FirstCluster + NumClusters / 2;
7988 // Balance the tree based on branch weights to create a near-optimal (in terms
7989 // of search time given key frequency) binary search tree. See e.g. Kurt
7990 // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
7991 CaseClusterIt LastLeft = W.FirstCluster;
7992 CaseClusterIt FirstRight = W.LastCluster;
7993 uint32_t LeftWeight = LastLeft->Weight;
7994 uint32_t RightWeight = FirstRight->Weight;
7995
7996 // Move LastLeft and FirstRight towards each other from opposite directions to
7997 // find a partitioning of the clusters which balances the weight on both
7998 // sides.
7999 while (LastLeft + 1 < FirstRight) {
8000 // Zero-weight nodes would cause skewed trees since they don't affect
8001 // LeftWeight or RightWeight.
8002 assert(LastLeft->Weight != 0);
8003 assert(FirstRight->Weight != 0);
8004
8005 if (LeftWeight < RightWeight)
8006 LeftWeight += (++LastLeft)->Weight;
8007 else
8008 RightWeight += (--FirstRight)->Weight;
8009 }
8010 assert(LastLeft + 1 == FirstRight);
8011 assert(LastLeft >= W.FirstCluster);
8012 assert(FirstRight <= W.LastCluster);
8013
8014 // Use the first element on the right as pivot since we will make less-than
8015 // comparisons against it.
8016 CaseClusterIt PivotCluster = FirstRight;
8017 assert(PivotCluster > W.FirstCluster);
8018 assert(PivotCluster <= W.LastCluster);
8019
79928020 CaseClusterIt FirstLeft = W.FirstCluster;
7993 CaseClusterIt LastLeft = PivotCluster - 1;
7994 CaseClusterIt FirstRight = PivotCluster;
79958021 CaseClusterIt LastRight = W.LastCluster;
8022
79968023 const ConstantInt *Pivot = PivotCluster->Low;
79978024
79988025 // New blocks will be inserted immediately after the current one.
80318058 }
80328059
80338060 // Create the CaseBlock record that will be used to lower the branch.
8034 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB);
8061 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
8062 LeftWeight, RightWeight);
80358063
80368064 if (W.MBB == SwitchMBB)
80378065 visitSwitchCase(CB, SwitchMBB);
80478075 for (auto I : SI.cases()) {
80488076 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
80498077 const ConstantInt *CaseVal = I.getCaseValue();
8050 uint32_t Weight = 0; // FIXME: Use 1 instead?
8078 uint32_t Weight = 1;
80518079 if (BPI) {
80528080 Weight = BPI->getEdgeWeight(SI.getParent(), I.getSuccessorIndex());
80538081 assert(Weight <= UINT32_MAX / SI.getNumSuccessors());
44
55 ; Make sure we have the correct weight attached to each successor.
66 define i32 @test2(i32 %x) nounwind uwtable readnone ssp {
7 ; CHECK: Machine code for function test2:
7 ; CHECK-LABEL: Machine code for function test2:
88 entry:
99 %conv = sext i32 %x to i64
1010 switch i64 %conv, label %return [
3232 }
3333
3434 !0 = !{!"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}
35
36
37 declare void @g(i32)
38 define void @left_leaning_weight_balanced_tree(i32 %x) {
39 entry:
40 switch i32 %x, label %return [
41 i32 0, label %bb0
42 i32 10, label %bb1
43 i32 20, label %bb2
44 i32 30, label %bb3
45 i32 40, label %bb4
46 i32 50, label %bb5
47 ], !prof !1
48 bb0: tail call void @g(i32 0) br label %return
49 bb1: tail call void @g(i32 1) br label %return
50 bb2: tail call void @g(i32 2) br label %return
51 bb3: tail call void @g(i32 3) br label %return
52 bb4: tail call void @g(i32 4) br label %return
53 bb5: tail call void @g(i32 5) br label %return
54 return: ret void
55
56 ; Check that we set branch weights on the pivot cmp instruction correctly.
57 ; Cases {0,10,20,30} go on the left with weight 13; cases {40,50} go on the
58 ; right with weight 20.
59 ;
60 ; CHECK-LABEL: Machine code for function left_leaning_weight_balanced_tree:
61 ; CHECK: BB#0: derived from LLVM BB %entry
62 ; CHECK-NOT: Successors
63 ; CHECK: Successors according to CFG: BB#8(13) BB#9(20)
64 }
65
66 !1 = !{!"branch_weights",
67 ; Default:
68 i32 1,
69 ; Case 0, 10, 20:
70 i32 10, i32 1, i32 1,
71 ; Case 30, 40, 50:
72 i32 1, i32 10, i32 10}
441441 i32 1000,
442442 ; Case 300:
443443 i32 10}
444
445
446 define void @zero_weight_tree(i32 %x) {
447 entry:
448 switch i32 %x, label %return [
449 i32 0, label %bb0
450 i32 10, label %bb1
451 i32 20, label %bb2
452 i32 30, label %bb3
453 i32 40, label %bb4
454 i32 50, label %bb5
455 ], !prof !3
456 bb0: tail call void @g(i32 0) br label %return
457 bb1: tail call void @g(i32 1) br label %return
458 bb2: tail call void @g(i32 2) br label %return
459 bb3: tail call void @g(i32 3) br label %return
460 bb4: tail call void @g(i32 4) br label %return
461 bb5: tail call void @g(i32 5) br label %return
462 return: ret void
463
464 ; Make sure to pick a pivot in the middle also with zero-weight cases.
465 ; CHECK-LABEL: zero_weight_tree
466 ; CHECK-NOT: cmpl
467 ; CHECK: cmpl $29
468 }
469
470 !3 = !{!"branch_weights", i32 1, i32 10, i32 0, i32 0, i32 0, i32 0, i32 10}
471
472
473 define void @left_leaning_weight_balanced_tree(i32 %x) {
474 entry:
475 switch i32 %x, label %return [
476 i32 0, label %bb0
477 i32 10, label %bb1
478 i32 20, label %bb2
479 i32 30, label %bb3
480 i32 40, label %bb4
481 i32 50, label %bb5
482 ], !prof !4
483 bb0: tail call void @g(i32 0) br label %return
484 bb1: tail call void @g(i32 1) br label %return
485 bb2: tail call void @g(i32 2) br label %return
486 bb3: tail call void @g(i32 3) br label %return
487 bb4: tail call void @g(i32 4) br label %return
488 bb5: tail call void @g(i32 5) br label %return
489 return: ret void
490
491 ; To balance the tree by weight, the pivot is shifted to the right, moving hot
492 ; cases closer to the root.
493 ; CHECK-LABEL: left_leaning_weight_balanced_tree
494 ; CHECK-NOT: cmpl
495 ; CHECK: cmpl $39
496 }
497
498 !4 = !{!"branch_weights", i32 1, i32 10, i32 1, i32 1, i32 1, i32 10, i32 10}
499
500
501 define void @jump_table_affects_balance(i32 %x) {
502 entry:
503 switch i32 %x, label %return [
504 ; Jump table:
505 i32 0, label %bb0
506 i32 1, label %bb1
507 i32 2, label %bb2
508 i32 3, label %bb3
509
510 i32 100, label %bb0
511 i32 200, label %bb1
512 i32 300, label %bb2
513 ]
514 bb0: tail call void @g(i32 0) br label %return
515 bb1: tail call void @g(i32 1) br label %return
516 bb2: tail call void @g(i32 2) br label %return
517 bb3: tail call void @g(i32 3) br label %return
518 return: ret void
519
520 ; CHECK-LABEL: jump_table_affects_balance
521 ; If the tree were balanced based on number of clusters, {0-3,100} would go on
522 ; the left and {200,300} on the right. However, the jump table weights as much
523 ; as its components, so 100 is selected as the pivot.
524 ; CHECK-NOT: cmpl
525 ; CHECK: cmpl $99
526 }