From 5c826129bac578a0bf1cd7ba9ef9e12578ac25c6 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 2 Oct 2013 08:53:22 +0900 Subject: [PATCH] Bug 905973 part 1 - Add a function to read simple dependency makefiles, and make makeutil.Rule faster. r=gps --- python/mozbuild/mozbuild/makeutil.py | 76 +++++++++++++++++-- .../mozbuild/mozbuild/test/test_makeutil.py | 58 ++++++++++++++ 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py index 280b80c7613..570a7215603 100644 --- a/python/mozbuild/mozbuild/makeutil.py +++ b/python/mozbuild/mozbuild/makeutil.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os +import re from types import StringTypes from collections import Iterable @@ -53,6 +54,38 @@ class Makefile(object): guard.dump(fh) +class _SimpleOrderedSet(object): + ''' + Simple ordered set, specialized for used in Rule below only. + It doesn't expose a complete API, and normalizes path separators + at insertion. + ''' + def __init__(self): + self._list = [] + self._set = set() + + def __nonzero__(self): + return bool(self._set) + + def __iter__(self): + return iter(self._list) + + def __contains__(self, key): + return key in self._set + + def update(self, iterable): + def _add(iterable): + emitted = set() + for i in iterable: + i = i.replace(os.sep, '/') + if i not in self._set and i not in emitted: + yield i + emitted.add(i) + added = list(_add(iterable)) + self._set.update(added) + self._list.extend(added) + + class Rule(object): '''Class handling simple rules in the form: target1 target2 ... : dep1 dep2 ... @@ -61,23 +94,21 @@ class Rule(object): ... ''' def __init__(self, targets=[]): - self._targets = [] - self._dependencies = [] + self._targets = _SimpleOrderedSet() + self._dependencies = _SimpleOrderedSet() self._commands = [] self.add_targets(targets) def add_targets(self, targets): '''Add additional targets to the rule.''' assert isinstance(targets, Iterable) and not isinstance(targets, StringTypes) - self._targets.extend(t.replace(os.sep, '/') for t in targets - if not t in self._targets) + self._targets.update(targets) return self def add_dependencies(self, deps): '''Add dependencies to the rule.''' assert isinstance(deps, Iterable) and not isinstance(deps, StringTypes) - self._dependencies.extend(d.replace(os.sep, '/') for d in deps - if not d in self._dependencies) + self._dependencies.update(deps) return self def add_commands(self, commands): @@ -94,7 +125,7 @@ class Rule(object): def dependencies(self): '''Return an iterator on the rule dependencies.''' - return iter(self._dependencies) + return iter(d for d in self._dependencies if not d in self._targets) def commands(self): '''Return an iterator on the rule commands.''' @@ -108,7 +139,36 @@ class Rule(object): return fh.write('%s:' % ' '.join(self._targets)) if self._dependencies: - fh.write(' %s' % ' '.join(self._dependencies)) + fh.write(' %s' % ' '.join(self.dependencies())) fh.write('\n') for cmd in self._commands: fh.write('\t%s\n' % cmd) + + +# colon followed by anything except a slash (Windows path detection) +_depfilesplitter = re.compile(r':(?![\\/])') + + +def read_dep_makefile(fh): + """ + Read the file handler containing a dep makefile (simple makefile only + containing dependencies) and returns an iterator of the corresponding Rules + it contains. Ignores removal guard rules. + """ + + rule = '' + for line in fh.readlines(): + assert not line.startswith('\t') + line = line.strip() + if line.endswith('\\'): + rule += line[:-1] + else: + rule += line + split_rule = _depfilesplitter.split(rule, 1) + if len(split_rule) > 1 and split_rule[1].strip(): + yield Rule(split_rule[0].strip().split()) \ + .add_dependencies(split_rule[1].strip().split()) + rule = '' + + if rule: + raise Exception('Makefile finishes with a backslash. Expected more input.') diff --git a/python/mozbuild/mozbuild/test/test_makeutil.py b/python/mozbuild/mozbuild/test/test_makeutil.py index 38ecdb34243..791b0bd46b2 100644 --- a/python/mozbuild/mozbuild/test/test_makeutil.py +++ b/python/mozbuild/mozbuild/test/test_makeutil.py @@ -4,6 +4,7 @@ from mozbuild.makeutil import ( Makefile, + read_dep_makefile, Rule, ) from mozunit import main @@ -41,6 +42,47 @@ class TestMakefile(unittest.TestCase): '\techo $@\n' + '\t$(BAZ) -o $@ $<\n' + '\t$(TOUCH) $@\n') + out.truncate(0) + + rule = Rule(['foo']) + rule.add_dependencies(['bar', 'foo', 'baz']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo: bar baz\n') + out.truncate(0) + + rule.add_targets(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_targets(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_dependencies(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_dependencies(['qux']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux\n') + out.truncate(0) + + rule.add_dependencies(['qux']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux\n') + out.truncate(0) + + rule.add_dependencies(['hoge', 'hoge']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux hoge\n') + out.truncate(0) + + rule.add_targets(['fuga', 'fuga']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar fuga: baz qux hoge\n') def test_makefile(self): out = StringIO() @@ -95,5 +137,21 @@ class TestMakefile(unittest.TestCase): '\techo c:\\foo\n' + 'c:/bar c:/baz/qux:\n') + def test_read_dep_makefile(self): + input = StringIO( + os.path.abspath('foo') + ': bar\n' + + 'baz qux: \\ \n' + + 'hoge \\\n' + + 'piyo \\\n' + + 'fuga\n' + + 'fuga:\n' + ) + result = list(read_dep_makefile(input)) + self.assertEqual(len(result), 2) + self.assertEqual(list(result[0].targets()), [os.path.abspath('foo').replace(os.sep, '/')]) + self.assertEqual(list(result[0].dependencies()), ['bar']) + self.assertEqual(list(result[1].targets()), ['baz', 'qux']) + self.assertEqual(list(result[1].dependencies()), ['hoge', 'piyo', 'fuga']) + if __name__ == '__main__': main()