Bug 905973 part 1 - Add a function to read simple dependency makefiles, and make makeutil.Rule faster. r=gps

python/mozbuild/mozbuild/makeutil.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+import re
 from types import StringTypes
 from collections import Iterable
 
@@ -53,6 +54,38 @@ class Makefile(object):
             guard.dump(fh)
 
 
+class _SimpleOrderedSet(object):
+    '''
+    Simple ordered set, specialized for used in Rule below only.
+    It doesn't expose a complete API, and normalizes path separators
+    at insertion.
+    '''
+    def __init__(self):
+        self._list = []
+        self._set = set()
+
+    def __nonzero__(self):
+        return bool(self._set)
+
+    def __iter__(self):
+        return iter(self._list)
+
+    def __contains__(self, key):
+        return key in self._set
+
+    def update(self, iterable):
+        def _add(iterable):
+            emitted = set()
+            for i in iterable:
+                i = i.replace(os.sep, '/')
+                if i not in self._set and i not in emitted:
+                    yield i
+                    emitted.add(i)
+        added = list(_add(iterable))
+        self._set.update(added)
+        self._list.extend(added)
+
+
 class Rule(object):
     '''Class handling simple rules in the form:
            target1 target2 ... : dep1 dep2 ...
@@ -61,23 +94,21 @@ class Rule(object):
                    ...
     '''
     def __init__(self, targets=[]):
-        self._targets = []
-        self._dependencies = []
+        self._targets = _SimpleOrderedSet()
+        self._dependencies = _SimpleOrderedSet()
         self._commands = []
         self.add_targets(targets)
 
     def add_targets(self, targets):
         '''Add additional targets to the rule.'''
         assert isinstance(targets, Iterable) and not isinstance(targets, StringTypes)
-        self._targets.extend(t.replace(os.sep, '/') for t in targets
-                             if not t in self._targets)
+        self._targets.update(targets)
         return self
 
     def add_dependencies(self, deps):
         '''Add dependencies to the rule.'''
         assert isinstance(deps, Iterable) and not isinstance(deps, StringTypes)
-        self._dependencies.extend(d.replace(os.sep, '/') for d in deps
-                                  if not d in self._dependencies)
+        self._dependencies.update(deps)
         return self
 
     def add_commands(self, commands):
@@ -94,7 +125,7 @@ class Rule(object):
 
     def dependencies(self):
         '''Return an iterator on the rule dependencies.'''
-        return iter(self._dependencies)
+        return iter(d for d in self._dependencies if not d in self._targets)
 
     def commands(self):
         '''Return an iterator on the rule commands.'''
@@ -108,7 +139,36 @@ class Rule(object):
             return
         fh.write('%s:' % ' '.join(self._targets))
         if self._dependencies:
-            fh.write(' %s' % ' '.join(self._dependencies))
+            fh.write(' %s' % ' '.join(self.dependencies()))
         fh.write('\n')
         for cmd in self._commands:
             fh.write('\t%s\n' % cmd)
+
+
+# colon followed by anything except a slash (Windows path detection)
+_depfilesplitter = re.compile(r':(?![\\/])')
+
+
+def read_dep_makefile(fh):
+    """
+    Read the file handler containing a dep makefile (simple makefile only
+    containing dependencies) and returns an iterator of the corresponding Rules
+    it contains. Ignores removal guard rules.
+    """
+
+    rule = ''
+    for line in fh.readlines():
+        assert not line.startswith('\t')
+        line = line.strip()
+        if line.endswith('\\'):
+            rule += line[:-1]
+        else:
+            rule += line
+            split_rule = _depfilesplitter.split(rule, 1)
+            if len(split_rule) > 1 and split_rule[1].strip():
+                yield Rule(split_rule[0].strip().split()) \
+                      .add_dependencies(split_rule[1].strip().split())
+            rule = ''
+
+    if rule:
+        raise Exception('Makefile finishes with a backslash. Expected more input.')

python/mozbuild/mozbuild/test/test_makeutil.py

@@ -4,6 +4,7 @@
 
 from mozbuild.makeutil import (
     Makefile,
+    read_dep_makefile,
     Rule,
 )
 from mozunit import main
@@ -41,6 +42,47 @@ class TestMakefile(unittest.TestCase):
             '\techo $@\n' +
             '\t$(BAZ) -o $@ $<\n' +
             '\t$(TOUCH) $@\n')
+        out.truncate(0)
+
+        rule = Rule(['foo'])
+        rule.add_dependencies(['bar', 'foo', 'baz'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo: bar baz\n')
+        out.truncate(0)
+
+        rule.add_targets(['bar'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz\n')
+        out.truncate(0)
+
+        rule.add_targets(['bar'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz\n')
+        out.truncate(0)
+
+        rule.add_dependencies(['bar'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz\n')
+        out.truncate(0)
+
+        rule.add_dependencies(['qux'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz qux\n')
+        out.truncate(0)
+
+        rule.add_dependencies(['qux'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz qux\n')
+        out.truncate(0)
+
+        rule.add_dependencies(['hoge', 'hoge'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar: baz qux hoge\n')
+        out.truncate(0)
+
+        rule.add_targets(['fuga', 'fuga'])
+        rule.dump(out)
+        self.assertEqual(out.getvalue(), 'foo bar fuga: baz qux hoge\n')
 
     def test_makefile(self):
         out = StringIO()
@@ -95,5 +137,21 @@ class TestMakefile(unittest.TestCase):
             '\techo c:\\foo\n' +
             'c:/bar c:/baz/qux:\n')
 
+    def test_read_dep_makefile(self):
+        input = StringIO(
+            os.path.abspath('foo') + ': bar\n' +
+            'baz qux: \\ \n' +
+            'hoge \\\n' +
+            'piyo \\\n' +
+            'fuga\n' +
+            'fuga:\n'
+        )
+        result = list(read_dep_makefile(input))
+        self.assertEqual(len(result), 2)
+        self.assertEqual(list(result[0].targets()), [os.path.abspath('foo').replace(os.sep, '/')])
+        self.assertEqual(list(result[0].dependencies()), ['bar'])
+        self.assertEqual(list(result[1].targets()), ['baz', 'qux'])
+        self.assertEqual(list(result[1].dependencies()), ['hoge', 'piyo', 'fuga'])
+
 if __name__ == '__main__':
     main()