[gicombiner] Add GIMatchTree and use it for the code generation

Summary:
GIMatchTree's job is to build a decision tree by zipping all the
GIMatchDag's together.

Each DAG is added to the tree builder as a leaf and partitioners are used
to subdivide each node until there are no more partitioners to apply. At
this point, the code generator is responsible for testing any untested
predicates and following any unvisited traversals (there shouldn't be any
of the latter as the getVRegDef partitioner handles them all).

Note that the leaves don't always fit into partitions cleanly and the
partitions may overlap as a result. This is resolved by cloning the leaf
into every partition it belongs to. One example of this is a rule that can
match one of N opcodes. The leaf for this rule would end up in N partitions
when processed by the opcode partitioner. A similar example is the
getVRegDef partitioner where having rules (add $a, $b), and (add ($a, $b), $c)
will result in the former being in the partition for successfully
following the vreg-def and failing to do so as it doesn't care which
happens.

Depends on D69151

Reviewers: bogner, volkan

Reviewed By: volkan

Subscribers: lkail, mgorny, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D69152

llvm/include/llvm/Target/GlobalISel/Combine.td

@@ -89,6 +89,10 @@ def match;
 class GIMatchKind;
 class GIMatchKindWithArgs;
 
+/// In lieu of having proper macro support. Trivial one-off opcode checks can be
+/// performed with this.
+def wip_match_opcode : GIMatchKindWithArgs;
+
 /// The operator at the root of a GICombineRule.Apply dag.
 def apply;
 /// All arguments of the apply operator must be subclasses of GIApplyKind, or
@@ -99,26 +103,30 @@ class GIApplyKindWithArgs;
 
 def copy_prop : GICombineRule<
   (defs root:$d),
-  (match [{ return Helper.matchCombineCopy(${d}); }]),
-  (apply [{ Helper.applyCombineCopy(${d}); }])>;
+  (match (COPY $d, $s):$mi,
+         [{ return Helper.matchCombineCopy(*${mi}); }]),
+  (apply [{ Helper.applyCombineCopy(*${mi}); }])>;
 def trivial_combines : GICombineGroup<[copy_prop]>;
 
 def extending_loads : GICombineRule<
   (defs root:$root, extending_load_matchdata:$matchinfo),
-  (match [{ return Helper.matchCombineExtendingLoads(${root}, ${matchinfo}); }]),
-  (apply [{ Helper.applyCombineExtendingLoads(${root}, ${matchinfo}); }])>;
+  (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root,
+         [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]),
+  (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>;
 def combines_for_extload: GICombineGroup<[extending_loads]>;
 
 def combine_indexed_load_store : GICombineRule<
   (defs root:$root, indexed_load_store_matchdata:$matchinfo),
-  (match [{ return Helper.matchCombineIndexedLoadStore(${root}, ${matchinfo}); }]),
-  (apply [{ Helper.applyCombineIndexedLoadStore(${root}, ${matchinfo}); }])>;
+  (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD, G_STORE):$root,
+         [{ return Helper.matchCombineIndexedLoadStore(*${root}, ${matchinfo}); }]),
+  (apply [{ Helper.applyCombineIndexedLoadStore(*${root}, ${matchinfo}); }])>;
 
 // FIXME: Is there a reason this wasn't in tryCombine? I've left it out of
 //        all_combines because it wasn't there.
 def elide_br_by_inverting_cond : GICombineRule<
-  (defs root:$d),
-  (match [{ return Helper.matchElideBrByInvertingCond(${d}); }]),
-  (apply [{ Helper.applyElideBrByInvertingCond(${d}); }])>;
+  (defs root:$root),
+  (match (wip_match_opcode G_BR):$root,
+         [{ return Helper.matchElideBrByInvertingCond(*${root}); }]),
+  (apply [{ Helper.applyElideBrByInvertingCond(*${root}); }])>;
 
 def all_combines : GICombineGroup<[trivial_combines, combines_for_extload, combine_indexed_load_store]>;

llvm/test/TableGen/GICombinerEmitter/match-tree.td

@@ -0,0 +1,142 @@
+// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN:     -combiners=MyCombinerHelper -gicombiner-stop-after-build %s \
+// RUN:     -o /dev/null -debug 2>&1 | FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def dummy;
+
+def R0 : Register<"r0"> { let Namespace = "MyTarget"; }
+def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
+class I<dag OOps, dag IOps, list<dag> Pat>
+  : Instruction {
+  let Namespace = "MyTarget";
+  let OutOperandList = OOps;
+  let InOperandList = IOps;
+  let Pattern = Pat;
+}
+def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), []>;
+def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), []>;
+def SUB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), []>;
+def MUL : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), []>;
+def TRUNC : I<(outs GPR32:$dst), (ins GPR32:$src1), []>;
+def SEXT : I<(outs GPR32:$dst), (ins GPR32:$src1), []>;
+def ZEXT : I<(outs GPR32:$dst), (ins GPR32:$src1), []>;
+
+def Rule0 : GICombineRule<
+  (defs root:$d),
+  (match (MUL $t, $s1, $s2),
+         (SUB $d, $t, $s3)),
+  (apply [{ APPLY }])>;
+
+def Rule1 : GICombineRule<
+  (defs root:$d),
+  (match (MOV $s1, $s2),
+         (MOV $d, $s1)),
+  (apply [{ APPLY }])>;
+
+def Rule2 : GICombineRule<
+  (defs root:$d),
+  (match (MOV $d, $s)),
+  (apply [{ APPLY }])>;
+
+def Rule3 : GICombineRule<
+  (defs root:$d),
+  (match (MUL $t, $s1, $s2),
+         (ADD $d, $t, $s3), [{ A }]),
+  (apply [{ APPLY }])>;
+
+def Rule4 : GICombineRule<
+  (defs root:$d),
+  (match (ADD $d, $s1, $s2)),
+  (apply [{ APPLY }])>;
+
+def Rule5 : GICombineRule<
+  (defs root:$d),
+  (match (SUB $d, $s1, $s2)),
+  (apply [{ APPLY }])>;
+
+def Rule6 : GICombineRule<
+  (defs root:$d),
+  (match (SEXT $t, $s1),
+         (TRUNC $d, $t)),
+  (apply [{ APPLY }])>;
+
+def Rule7 : GICombineRule<
+  (defs root:$d),
+  (match (ZEXT $t, $s1),
+         (TRUNC $d, $t)),
+  (apply [{ APPLY }])>;
+
+def MyCombinerHelper: GICombinerHelper<"GenMyCombinerHelper", [
+  Rule0,
+  Rule1,
+  Rule2,
+  Rule3,
+  Rule4,
+  Rule5,
+  Rule6,
+  Rule7
+]>;
+
+// CHECK-LABEL: digraph "matchtree" {
+// CHECK-DAG:   Node[[N0:0x[0-9a-f]+]] [shape=record,label="{MI[0].getOpcode()|4 partitions|Rule0,Rule1,Rule2,Rule3,Rule4,Rule5,Rule6,Rule7}"]
+// CHECK-DAG:   Node[[N1:0x[0-9a-f]+]] [shape=record,label="{MI[1] = getVRegDef(MI[0].getOperand(1))|2 partitions|Rule0,Rule5}"]
+// CHECK-DAG:   Node[[N2:0x[0-9a-f]+]] [shape=record,label="{MI[1].getOpcode()|2 partitions|Rule0,Rule5}"]
+// CHECK-DAG:   Node[[N3:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule0}"]
+// CHECK-DAG:   Node[[N4:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule5}"]
+// CHECK-DAG:   Node[[N5:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule5}"]
+// CHECK-DAG:   Node[[N6:0x[0-9a-f]+]] [shape=record,label="{MI[1] = getVRegDef(MI[0].getOperand(1))|2 partitions|Rule1,Rule2}"]
+// CHECK-DAG:   Node[[N7:0x[0-9a-f]+]] [shape=record,label="{MI[1].getOpcode()|2 partitions|Rule1,Rule2}"]
+// CHECK-DAG:   Node[[N8:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule1}"]
+// CHECK-DAG:   Node[[N9:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule2}"]
+// CHECK-DAG:   Node[[N10:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule2}"]
+// CHECK-DAG:   Node[[N11:0x[0-9a-f]+]] [shape=record,label="{MI[1] = getVRegDef(MI[0].getOperand(1))|2 partitions|Rule3,Rule4}"]
+// CHECK-DAG:   Node[[N12:0x[0-9a-f]+]] [shape=record,label="{MI[1].getOpcode()|2 partitions|Rule3,Rule4}"]
+// CHECK-DAG:   Node[[N13:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule3,Rule4}",color=red]
+// CHECK-DAG:   Node[[N14:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule4}"]
+// CHECK-DAG:   Node[[N15:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule4}"]
+// CHECK-DAG:   Node[[N16:0x[0-9a-f]+]] [shape=record,label="{MI[1] = getVRegDef(MI[0].getOperand(1))|1 partitions|Rule6,Rule7}"]
+// CHECK-DAG:   Node[[N17:0x[0-9a-f]+]] [shape=record,label="{MI[1].getOpcode()|2 partitions|Rule6,Rule7}"]
+// CHECK-DAG:   Node[[N18:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule6}"]
+// CHECK-DAG:   Node[[N19:0x[0-9a-f]+]] [shape=record,label="{No partitioner|Rule7}"]
+
+// The most important partitioner is on the first opcode:
+// CHECK-DAG:   Node[[N0]] -> Node[[N1]] [label="#0 MyTarget::SUB"]
+// CHECK-DAG:   Node[[N0]] -> Node[[N6]] [label="#1 MyTarget::MOV"]
+// CHECK-DAG:   Node[[N0]] -> Node[[N11]] [label="#2 MyTarget::ADD"]
+// CHECK-DAG:   Node[[N0]] -> Node[[N16]] [label="#3 MyTarget::TRUNC"]
+
+// For, MI[0].getOpcode() == SUB, then has to determine whether it has a reg
+// operand and follow that link. If it can't then Rule5 is the only choice as
+// that rule is not constrained to a reg.
+// CHECK-DAG:   Node[[N1]] -> Node[[N2]] [label="#0 true"]
+// CHECK-DAG:   Node[[N1]] -> Node[[N5]] [label="#1 false"]
+
+// For, MI[0].getOpcode() == SUB && MI[0].getOperand(1).isReg(), if MI[1] is a
+// MUL then it must be either Rule0 or Rule5. Rule0 is fully tested so Rule5 is
+// unreachable. If it's not MUL then it must be Rule5.
+// CHECK-DAG:   Node[[N2]] -> Node[[N3]] [label="#0 MyTarget::MUL"]
+// CHECK-DAG:   Node[[N2]] -> Node[[N4]] [label="#1 * or nullptr"]
+
+// CHECK-DAG:   Node[[N6]] -> Node[[N7]] [label="#0 true"]
+// CHECK-DAG:   Node[[N6]] -> Node[[N10]] [label="#1 false"]
+
+// CHECK-DAG:   Node[[N7]] -> Node[[N8]] [label="#0 MyTarget::MOV"]
+// CHECK-DAG:   Node[[N7]] -> Node[[N9]] [label="#1 * or nullptr"]
+
+// CHECK-DAG:   Node[[N11]] -> Node[[N12]] [label="#0 true"]
+// CHECK-DAG:   Node[[N11]] -> Node[[N15]] [label="#1 false"]
+
+// CHECK-DAG:   Node[[N12]] -> Node[[N13]] [label="#0 MyTarget::MUL"]
+// CHECK-DAG:   Node[[N12]] -> Node[[N14]] [label="#1 * or nullptr"]
+
+// CHECK-DAG:   Node[[N16]] -> Node[[N17]] [label="#0 true"]
+
+// CHECK-DAG:   Node[[N17]] -> Node[[N18]] [label="#0 MyTarget::SEXT"]
+// CHECK-DAG:   Node[[N17]] -> Node[[N19]] [label="#1 MyTarget::ZEXT"]
+// CHECK-LABEL: {{^}$}}

llvm/utils/TableGen/GICombinerEmitter.cpp

@@ -24,6 +24,7 @@
 #include "GlobalISel/CodeExpander.h"
 #include "GlobalISel/CodeExpansions.h"
 #include "GlobalISel/GIMatchDag.h"
+#include "GlobalISel/GIMatchTree.h"
 #include <cstdint>
 
 using namespace llvm;
@@ -47,6 +48,10 @@ static cl::opt<bool> StopAfterParse(
     "gicombiner-stop-after-parse",
     cl::desc("Stop processing after parsing rules and dump state"),
     cl::cat(GICombinerEmitterCat));
+static cl::opt<bool> StopAfterBuild(
+    "gicombiner-stop-after-build",
+    cl::desc("Stop processing after building the match tree"),
+    cl::cat(GICombinerEmitterCat));
 
 namespace {
 typedef uint64_t RuleID;
@@ -62,6 +67,22 @@ StringRef insertStrTab(StringRef S) {
   return StrTab.insert(S).first->first();
 }
 
+class format_partition_name {
+  const GIMatchTree &Tree;
+  unsigned Idx;
+
+public:
+  format_partition_name(const GIMatchTree &Tree, unsigned Idx)
+      : Tree(Tree), Idx(Idx) {}
+  void print(raw_ostream &OS) const {
+    Tree.getPartitioner()->emitPartitionName(OS, Idx);
+  }
+};
+raw_ostream &operator<<(raw_ostream &OS, const format_partition_name &Fmt) {
+  Fmt.print(OS);
+  return OS;
+}
+
 /// Declares data that is passed from the match stage to the apply stage.
 class MatchDataInfo {
   /// The symbol used in the tablegen patterns
@@ -162,6 +183,8 @@ protected:
                                const Init &Arg,
                                StringMap<std::vector<VarInfo>> &NamedEdgeDefs,
                                StringMap<std::vector<VarInfo>> &NamedEdgeUses);
+  bool parseWipMatchOpcodeMatcher(const CodeGenTarget &Target,
+                                  StringInit *ArgName, const Init &Arg);
 
 public:
   CombineRule(const CodeGenTarget &Target, GIMatchDagContext &Ctx, RuleID ID,
@@ -275,6 +298,20 @@ static Record *getDefOfSubClass(const Init &N, StringRef Cls) {
   return nullptr;
 }
 
+/// A convenience function to check that an Init refers to a dag whose operator
+/// is a specific def and coerce it to a dag if it is. This is primarily useful
+/// for testing for subclasses of GIMatchKind and similar in DagInit's since
+/// DagInit's support any type inside them.
+static const DagInit *getDagWithSpecificOperator(const Init &N,
+                                                 StringRef Name) {
+  if (const DagInit *I = dyn_cast<DagInit>(&N))
+    if (I->getNumArgs() > 0)
+      if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator()))
+        if (OpI->getDef()->getName() == Name)
+          return I;
+  return nullptr;
+}
+
 /// A convenience function to check that an Init refers to a dag whose operator
 /// is a def that is a subclass of the given class and coerce it to a dag if it
 /// is. This is primarily useful for testing for subclasses of GIMatchKind and
@@ -412,6 +449,44 @@ bool CombineRule::parseInstructionMatcher(
   return false;
 }
 
+// Parse the wip_match_opcode placeholder that's temporarily present in lieu of
+// implementing macros or choices between two matchers.
+bool CombineRule::parseWipMatchOpcodeMatcher(const CodeGenTarget &Target,
+                                             StringInit *ArgName,
+                                             const Init &Arg) {
+  if (const DagInit *Matcher =
+          getDagWithSpecificOperator(Arg, "wip_match_opcode")) {
+    StringRef Name = ArgName ? ArgName->getValue() : "";
+
+    GIMatchDagInstr *N =
+        MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name),
+                              MatchDag.getContext().makeEmptyOperandList());
+
+    if (find_if(Roots, [&](const RootInfo &X) {
+          return ArgName && X.getPatternSymbol() == ArgName->getValue();
+        }) != Roots.end()) {
+      N->setMatchRoot();
+    }
+
+    const auto &P = MatchDag.addPredicateNode<GIMatchDagOneOfOpcodesPredicate>(
+        makeNameForAnonPredicate(*this));
+    MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]);
+    // Each argument is an opcode that will pass this predicate. Add them all to
+    // the predicate implementation
+    for (const auto &Arg : Matcher->getArgs()) {
+      Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction");
+      if (OpcodeDef) {
+        P->addOpcode(&Target.getInstruction(OpcodeDef));
+        continue;
+      }
+      PrintError(TheDef.getLoc(),
+                 "Arguments to wip_match_opcode must be instructions");
+      return false;
+    }
+    return true;
+  }
+  return false;
+}
 bool CombineRule::parseMatcher(const CodeGenTarget &Target) {
   NamedRegionTimer T("parseMatcher", "Time spent parsing the matcher",
                      "Rule Parsing", "Time spent on rule parsing", TimeRegions);
@@ -437,12 +512,17 @@ bool CombineRule::parseMatcher(const CodeGenTarget &Target) {
                                 NamedEdgeUses))
       continue;
 
+    if (parseWipMatchOpcodeMatcher(Target, Matchers->getArgName(I),
+                                   *Matchers->getArg(I)))
+      continue;
+
 
     // Parse arbitrary C++ code we have in lieu of supporting MIR matching
     if (const CodeInit *CodeI = dyn_cast<CodeInit>(Matchers->getArg(I))) {
       assert(!MatchingFixupCode &&
              "Only one block of arbitrary code is currently permitted");
       MatchingFixupCode = CodeI;
+      MatchDag.setHasPostMatchPredicate(true);
       continue;
     }
 
@@ -537,7 +617,9 @@ public:
   /// Emit the name matcher (guarded by #ifndef NDEBUG) used to disable rules in
   /// response to the generated cl::opt.
   void emitNameMatcher(raw_ostream &OS) const;
-  void generateCodeForRule(raw_ostream &OS, const CombineRule *Rule,
+
+  void generateDeclarationsCodeForTree(raw_ostream &OS, const GIMatchTree &Tree) const;
+  void generateCodeForTree(raw_ostream &OS, const GIMatchTree &Tree,
                            StringRef Indent) const;
 };
 
@@ -592,6 +674,25 @@ GICombinerEmitter::makeCombineRule(const Record &TheDef) {
   if (StopAfterParse)
     return Rule;
 
+  // For now, don't support traversing from def to use. We'll come back to
+  // this later once we have the algorithm changes to support it.
+  bool EmittedDefToUseError = false;
+  for (const auto &E : Rule->getMatchDag().edges()) {
+    if (E->isDefToUse()) {
+      if (!EmittedDefToUseError) {
+        PrintError(
+            TheDef.getLoc(),
+            "Generated state machine cannot lookup uses from a def (yet)");
+        EmittedDefToUseError = true;
+      }
+      PrintNote("Node " + to_string(*E->getFromMI()));
+      PrintNote("Node " + to_string(*E->getToMI()));
+      PrintNote("Edge " + to_string(*E));
+    }
+  }
+  if (EmittedDefToUseError)
+    return nullptr;
+
   // For now, don't support multi-root rules. We'll come back to this later
   // once we have the algorithm changes to support it.
   if (Rule->getNumRoots() > 1) {
@@ -619,18 +720,40 @@ void GICombinerEmitter::gatherRules(
   }
 }
 
-void GICombinerEmitter::generateCodeForRule(raw_ostream &OS,
-                                            const CombineRule *Rule,
+void GICombinerEmitter::generateCodeForTree(raw_ostream &OS,
+                                            const GIMatchTree &Tree,
                                             StringRef Indent) const {
-  {
+  if (Tree.getPartitioner() != nullptr) {
+    Tree.getPartitioner()->generatePartitionSelectorCode(OS, Indent);
+    for (const auto &EnumChildren : enumerate(Tree.children())) {
+      OS << Indent << "if (Partition == " << EnumChildren.index() << " /* "
+         << format_partition_name(Tree, EnumChildren.index()) << " */) {\n";
+      generateCodeForTree(OS, EnumChildren.value(), (Indent + "  ").str());
+      OS << Indent << "}\n";
+    }
+    return;
+  }
+
+  bool AnyFullyTested = false;
+  for (const auto &Leaf : Tree.possible_leaves()) {
+    OS << Indent << "// Leaf name: " << Leaf.getName() << "\n";
+
+    const CombineRule *Rule = Leaf.getTargetData<CombineRule>();
     const Record &RuleDef = Rule->getDef();
 
     OS << Indent << "// Rule: " << RuleDef.getName() << "\n"
        << Indent << "if (!isRuleDisabled(" << Rule->getID() << ")) {\n";
 
     CodeExpansions Expansions;
-    for (const RootInfo &Root : Rule->roots()) {
-      Expansions.declare(Root.getPatternSymbol(), "MI");
+    for (const auto &VarBinding : Leaf.var_bindings()) {
+      if (VarBinding.isInstr())
+        Expansions.declare(VarBinding.getName(),
+                           "MIs[" + to_string(VarBinding.getInstrID()) + "]");
+      else
+        Expansions.declare(VarBinding.getName(),
+                           "MIs[" + to_string(VarBinding.getInstrID()) +
+                               "]->getOperand(" +
+                               to_string(VarBinding.getOpIdx()) + ")");
     }
     Rule->declareExpansions(Expansions);
 
@@ -643,6 +766,29 @@ void GICombinerEmitter::generateCodeForRule(raw_ostream &OS,
 
     OS << Indent << "  if (1\n";
 
+    // Attempt to emit code for any untested predicates left over. Note that
+    // isFullyTested() will remain false even if we succeed here and therefore
+    // combine rule elision will not be performed. This is because we do not
+    // know if there's any connection between the predicates for each leaf and
+    // therefore can't tell if one makes another unreachable. Ideally, the
+    // partitioner(s) would be sufficiently complete to prevent us from having
+    // untested predicates left over.
+    for (const GIMatchDagPredicate *Predicate : Leaf.untested_predicates()) {
+      if (Predicate->generateCheckCode(OS, (Indent + "      ").str(),
+                                       Expansions))
+        continue;
+      PrintError(RuleDef.getLoc(),
+                 "Unable to test predicate used in rule");
+      PrintNote(SMLoc(),
+                "This indicates an incomplete implementation in tablegen");
+      Predicate->print(errs());
+      errs() << "\n";
+      OS << Indent
+         << "llvm_unreachable(\"TableGen did not emit complete code for this "
+            "path\");\n";
+      break;
+    }
+
     if (Rule->getMatchingFixupCode() &&
         !Rule->getMatchingFixupCode()->getValue().empty()) {
       // FIXME: Single-use lambda's like this are a serious compile-time
@@ -674,7 +820,24 @@ void GICombinerEmitter::generateCodeForRule(raw_ostream &OS,
     }
 
     OS << Indent << "}\n";
+
+    assert(Leaf.isFullyTraversed());
+
+    // If we didn't have any predicates left over and we're not using the
+    // trap-door we have to support arbitrary C++ code while we're migrating to
+    // the declarative style then we know that subsequent leaves are
+    // unreachable.
+    if (Leaf.isFullyTested() &&
+        (!Rule->getMatchingFixupCode() ||
+         Rule->getMatchingFixupCode()->getValue().empty())) {
+      AnyFullyTested = true;
+      OS << Indent
+         << "llvm_unreachable(\"Combine rule elision was incorrect\");\n"
+         << Indent << "return false;\n";
+    }
   }
+  if (!AnyFullyTested)
+    OS << Indent << "return false;\n";
 }
 
 void GICombinerEmitter::run(raw_ostream &OS) {
@@ -687,6 +850,33 @@ void GICombinerEmitter::run(raw_ostream &OS) {
   }
   if (ErrorsPrinted)
     PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules");
+  LLVM_DEBUG(dbgs() << "Optimizing tree for " << Rules.size() << " rules\n");
+  std::unique_ptr<GIMatchTree> Tree;
+  {
+    NamedRegionTimer T("Optimize", "Time spent optimizing the combiner",
+                       "Code Generation", "Time spent generating code",
+                       TimeRegions);
+
+    GIMatchTreeBuilder TreeBuilder(0);
+    for (const auto &Rule : Rules) {
+      bool HadARoot = false;
+      for (const auto &Root : enumerate(Rule->getMatchDag().roots())) {
+        TreeBuilder.addLeaf(Rule->getName(), Root.index(), Rule->getMatchDag(),
+                            Rule.get());
+        HadARoot = true;
+      }
+      if (!HadARoot)
+        PrintFatalError(Rule->getDef().getLoc(), "All rules must have a root");
+    }
+
+    Tree = TreeBuilder.run();
+  }
+  if (StopAfterBuild) {
+    Tree->writeDOTGraph(outs());
+    PrintNote(Combiner->getLoc(),
+              "Terminating due to -gicombiner-stop-after-build");
+    return;
+  }
 
   NamedRegionTimer T("Emit", "Time spent emitting the combiner",
                      "Code Generation", "Time spent generating code",
@@ -772,6 +962,7 @@ void GICombinerEmitter::run(raw_ostream &OS) {
      << "  MachineBasicBlock *MBB = MI.getParent();\n"
      << "  MachineFunction *MF = MBB->getParent();\n"
      << "  MachineRegisterInfo &MRI = MF->getRegInfo();\n"
+     << "  SmallVector<MachineInstr *, 8> MIs = { &MI };\n\n"
      << "  (void)MBB; (void)MF; (void)MRI;\n\n";
 
   OS << "  // Match data\n";
@@ -780,8 +971,8 @@ void GICombinerEmitter::run(raw_ostream &OS) {
       OS << "  " << I.getType() << " " << I.getVariableName() << ";\n";
   OS << "\n";
 
-  for (const auto &Rule : Rules)
-    generateCodeForRule(OS, Rule.get(), "  ");
+  OS << "  int Partition = -1;\n";
+  generateCodeForTree(OS, *Tree, "  ");
   OS << "\n  return false;\n"
      << "}\n"
      << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n";

llvm/utils/TableGen/GlobalISel/CMakeLists.txt

@@ -10,4 +10,5 @@ llvm_add_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB
                  GIMatchDagOperands.cpp
                  GIMatchDagPredicate.cpp
                  GIMatchDagPredicateDependencyEdge.cpp
+                 GIMatchTree.cpp
                 )

llvm/utils/TableGen/GlobalISel/GIMatchDag.h

@@ -52,14 +52,25 @@ public:
 class GIMatchDag {
 public:
   using InstrNodesVec = std::vector<std::unique_ptr<GIMatchDagInstr>>;
+  using instr_node_iterator = raw_pointer_iterator<InstrNodesVec::iterator>;
+  using const_instr_node_iterator =
+      raw_pointer_iterator<InstrNodesVec::const_iterator>;
+
   using EdgesVec = std::vector<std::unique_ptr<GIMatchDagEdge>>;
   using edge_iterator = raw_pointer_iterator<EdgesVec::iterator>;
   using const_edge_iterator = raw_pointer_iterator<EdgesVec::const_iterator>;
 
   using PredicateNodesVec = std::vector<std::unique_ptr<GIMatchDagPredicate>>;
+  using predicate_iterator = raw_pointer_iterator<PredicateNodesVec::iterator>;
+  using const_predicate_iterator =
+      raw_pointer_iterator<PredicateNodesVec::const_iterator>;
 
   using PredicateDependencyEdgesVec =
       std::vector<std::unique_ptr<GIMatchDagPredicateDependencyEdge>>;
+  using predicate_edge_iterator =
+      raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>;
+  using const_predicate_edge_iterator =
+      raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>;
 
 protected:
   GIMatchDagContext &Ctx;
@@ -68,6 +79,9 @@ protected:
   EdgesVec Edges;
   PredicateDependencyEdgesVec PredicateDependencies;
   std::vector<GIMatchDagInstr *> MatchRoots;
+  // FIXME: This is a temporary measure while we still accept arbitrary code
+  //        blocks to fix up the matcher while it's being developed.
+  bool HasPostMatchPredicate = false;
 
 public:
   GIMatchDag(GIMatchDagContext &Ctx)
@@ -101,6 +115,71 @@ public:
     return make_range(MatchRoots.begin(), MatchRoots.end());
   }
 
+  instr_node_iterator instr_nodes_begin() {
+    return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.begin());
+  }
+  instr_node_iterator instr_nodes_end() {
+    return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.end());
+  }
+  const_instr_node_iterator instr_nodes_begin() const {
+    return raw_pointer_iterator<InstrNodesVec::const_iterator>(
+        InstrNodes.begin());
+  }
+  const_instr_node_iterator instr_nodes_end() const {
+    return raw_pointer_iterator<InstrNodesVec::const_iterator>(
+        InstrNodes.end());
+  }
+  iterator_range<instr_node_iterator> instr_nodes() {
+    return make_range(instr_nodes_begin(), instr_nodes_end());
+  }
+  iterator_range<const_instr_node_iterator> instr_nodes() const {
+    return make_range(instr_nodes_begin(), instr_nodes_end());
+  }
+  predicate_edge_iterator predicate_edges_begin() {
+    return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>(
+        PredicateDependencies.begin());
+  }
+  predicate_edge_iterator predicate_edges_end() {
+    return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>(
+        PredicateDependencies.end());
+  }
+  const_predicate_edge_iterator predicate_edges_begin() const {
+    return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>(
+        PredicateDependencies.begin());
+  }
+  const_predicate_edge_iterator predicate_edges_end() const {
+    return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>(
+        PredicateDependencies.end());
+  }
+  iterator_range<predicate_edge_iterator> predicate_edges() {
+    return make_range(predicate_edges_begin(), predicate_edges_end());
+  }
+  iterator_range<const_predicate_edge_iterator> predicate_edges() const {
+    return make_range(predicate_edges_begin(), predicate_edges_end());
+  }
+  predicate_iterator predicates_begin() {
+    return raw_pointer_iterator<PredicateNodesVec::iterator>(
+        PredicateNodes.begin());
+  }
+  predicate_iterator predicates_end() {
+    return raw_pointer_iterator<PredicateNodesVec::iterator>(
+        PredicateNodes.end());
+  }
+  const_predicate_iterator predicates_begin() const {
+    return raw_pointer_iterator<PredicateNodesVec::const_iterator>(
+        PredicateNodes.begin());
+  }
+  const_predicate_iterator predicates_end() const {
+    return raw_pointer_iterator<PredicateNodesVec::const_iterator>(
+        PredicateNodes.end());
+  }
+  iterator_range<predicate_iterator> predicates() {
+    return make_range(predicates_begin(), predicates_end());
+  }
+  iterator_range<const_predicate_iterator> predicates() const {
+    return make_range(predicates_begin(), predicates_end());
+  }
+
   template <class... Args> GIMatchDagInstr *addInstrNode(Args &&... args) {
     auto Obj =
         std::make_unique<GIMatchDagInstr>(*this, std::forward<Args>(args)...);
@@ -133,6 +212,19 @@ public:
     return ObjRaw;
   }
 
+  size_t getInstrNodeIdx(instr_node_iterator I) {
+    return std::distance(instr_nodes_begin(), I);
+  }
+  size_t getInstrNodeIdx(const_instr_node_iterator I) const {
+    return std::distance(instr_nodes_begin(), I);
+  }
+  size_t getNumInstrNodes() const { return InstrNodes.size(); }
+  size_t getNumEdges() const { return Edges.size(); }
+  size_t getNumPredicates() const { return PredicateNodes.size(); }
+
+  void setHasPostMatchPredicate(bool V) { HasPostMatchPredicate = V; }
+  bool hasPostMatchPredicate() const { return HasPostMatchPredicate; }
+
   void addMatchRoot(GIMatchDagInstr *N) { MatchRoots.push_back(N); }
 
   LLVM_DUMP_METHOD void print(raw_ostream &OS) const;

llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp

@@ -18,8 +18,21 @@ LLVM_DUMP_METHOD void GIMatchDagEdge::print(raw_ostream &OS) const {
      << "]";
 }
 
+bool GIMatchDagEdge::isDefToUse() const {
+  // Def -> Def is invalid so we only need to check FromMO.
+  return FromMO->isDef();
+}
+
 void GIMatchDagEdge::reverse() {
   std::swap(FromMI, ToMI);
   std::swap(FromMO, ToMO);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void GIMatchDagEdge::dump() const { print(errs()); }
+#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagEdge &E) {
+  E.print(OS);
+  return OS;
+}

llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h

@@ -54,6 +54,9 @@ public:
   /// Flip the direction of the edge.
   void reverse();
 
+  /// Does this edge run from a def to (one of many) uses?
+  bool isDefToUse() const;
+
   LLVM_DUMP_METHOD void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h

@@ -74,6 +74,7 @@ public:
 
   const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; }
   StringRef getName() const { return Name; }
+  StringRef getUserAssignedName() const { return UserAssignedName; }
   void assignNameToOperand(unsigned Idx, StringRef Name) {
     assert(UserAssignedNamesForOperands[Idx].empty() && "Cannot assign twice");
     UserAssignedNamesForOperands[Idx] = Name;

llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "GIMatchDagPredicate.h"
+
 #include "llvm/TableGen/Record.h"
 
 #include "GIMatchDagOperands.h"
@@ -32,6 +33,21 @@ void GIMatchDagOpcodePredicate::printDescription(raw_ostream &OS) const {
   OS << "$mi.getOpcode() == " << Instr.TheDef->getName();
 }
 
+GIMatchDagOneOfOpcodesPredicate::GIMatchDagOneOfOpcodesPredicate(
+    GIMatchDagContext &Ctx, StringRef Name)
+    : GIMatchDagPredicate(GIMatchDagPredicateKind_OneOfOpcodes, Name,
+                          Ctx.makeMIPredicateOperandList()) {}
+
+void GIMatchDagOneOfOpcodesPredicate::printDescription(raw_ostream &OS) const {
+  OS << "$mi.getOpcode() == oneof(";
+  StringRef Separator = "";
+  for (const CodeGenInstruction *Instr : Instrs) {
+    OS << Separator << Instr->TheDef->getName();
+    Separator = ",";
+  }
+  OS << ")";
+}
+
 GIMatchDagSameMOPredicate::GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx,
                                                      StringRef Name)
     : GIMatchDagPredicate(GIMatchDagPredicateKind_SameMO, Name,

llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h

@@ -11,7 +11,9 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "GIMatchDag.h"
+
 namespace llvm {
+class CodeExpansions;
 class CodeGenInstruction;
 class GIMatchDagOperandList;
 class GIMatchDagContext;
@@ -26,6 +28,7 @@ class GIMatchDagPredicate {
 public:
   enum GIMatchDagPredicateKind {
     GIMatchDagPredicateKind_Opcode,
+    GIMatchDagPredicateKind_OneOfOpcodes,
     GIMatchDagPredicateKind_SameMO,
   };
 
@@ -55,6 +58,16 @@ public:
   StringRef getName() const { return Name; }
   const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; }
 
+  // Generate C++ code to check this predicate. If a partitioner has already
+  // tested this predicate then this function won't be called. If this function
+  // is called, it must emit code and return true to indicate that it did so. If
+  // it ever returns false, then the caller will abort due to an untested
+  // predicate.
+  virtual bool generateCheckCode(raw_ostream &OS, StringRef Indent,
+                                 const CodeExpansions &Expansions) const {
+    return false;
+  }
+
   virtual void print(raw_ostream &OS) const;
   virtual void printDescription(raw_ostream &OS) const;
 
@@ -83,6 +96,29 @@ public:
 #endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 };
 
+class GIMatchDagOneOfOpcodesPredicate : public GIMatchDagPredicate {
+  SmallVector<const CodeGenInstruction *, 4> Instrs;
+
+public:
+  GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name);
+
+  void addOpcode(const CodeGenInstruction *Instr) { Instrs.push_back(Instr); }
+
+  static bool classof(const GIMatchDagPredicate *P) {
+    return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes;
+  }
+
+  const SmallVectorImpl<const CodeGenInstruction *> &getInstrs() const {
+    return Instrs;
+  }
+
+  void printDescription(raw_ostream &OS) const override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); }
+#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+};
+
 class GIMatchDagSameMOPredicate : public GIMatchDagPredicate {
 public:
   GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name);

llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp

@@ -11,6 +11,8 @@
 #include "GIMatchDagInstr.h"
 #include "GIMatchDagPredicate.h"
 
+#include "llvm/Support/raw_ostream.h"
+
 using namespace llvm;
 
 LLVM_DUMP_METHOD void