Bug 1065306 - Part 3: Extract DotProperties helper. r=lucasr,mshal

The only substantive change here is to stop stripping the
'browser.suggestedsites.' prefix from each line when reading
region.properties.

python/moz.build

@@ -43,6 +43,7 @@ PYTHON_UNIT_TESTS += [
     'mozbuild/mozbuild/test/frontend/test_sandbox.py',
     'mozbuild/mozbuild/test/test_base.py',
     'mozbuild/mozbuild/test/test_containers.py',
+    'mozbuild/mozbuild/test/test_dotproperties.py',
     'mozbuild/mozbuild/test/test_expression.py',
     'mozbuild/mozbuild/test/test_jarmaker.py',
     'mozbuild/mozbuild/test/test_line_endings.py',

python/mozbuild/mozbuild/action/generate_suggestedsites.py

@@ -29,10 +29,12 @@ from __future__ import print_function
 
 import argparse
 import json
-import re
 import sys
 import os
 
+from mozbuild.dotproperties import (
+    DotProperties,
+)
 from mozbuild.util import (
     FileAvoidWrite,
 )
@@ -42,59 +44,19 @@ from mozpack.files import (
 import mozpack.path as mozpath
 
 
-def read_properties_file(filename):
-    """Reads a properties file into a dict.
-
-    Ignores empty, comment lines, and keys not starting with the prefix for
-    suggested sites ('browser.suggestedsites'). Removes the prefix from all
-    matching keys i.e. turns 'browser.suggestedsites.foo' into simply 'foo'
-    """
-    prefix = 'browser.suggestedsites.'
-    properties = {}
-    for l in open(filename, 'rt').readlines():
-        line = l.strip()
-        if not line.startswith(prefix):
-            continue
-        (k, v) = re.split('\s*=\s*', line, 1)
-        properties[k[len(prefix):]] = v
-    return properties
-
-
 def merge_properties(filename, srcdirs):
     """Merges properties from the given file in the given source directories."""
-    properties = {}
+    properties = DotProperties()
     for srcdir in srcdirs:
         path = mozpath.join(srcdir, filename)
         try:
-            properties.update(read_properties_file(path))
+            properties.update(path)
-        except IOError, e:
+        except IOError:
             # Ignore non-existing files
             continue
     return properties
 
 
-def get_site_list_from_properties(properties):
-    """Turns {'list.0':'foo', 'list.1':'bar'} into ['foo', 'bar']."""
-    prefix = 'list.'
-    indexes = []
-    for k, v in properties.iteritems():
-        if not k.startswith(prefix):
-            continue
-        indexes.append(int(k[len(prefix):]))
-    return [properties[prefix + str(index)] for index in sorted(indexes)]
-
-
-def get_site_from_properties(name, properties):
-    """Turns {'foo.title':'title', ...} into {'title':'title', ...}."""
-    prefix = '{name}.'.format(name=name)
-    try:
-        site = dict((k, properties[prefix + k]) for k in ('title', 'url', 'bgcolor'))
-    except IndexError, e:
-        raise Exception("Could not find required property for '{name}: {error}'"
-                        .format(name=name, error=str(e)))
-    return site
-
-
 def main(args):
     parser = argparse.ArgumentParser()
     parser.add_argument('--verbose', '-v', default=False, action='store_true',
@@ -115,8 +77,8 @@ def main(args):
     opts = parser.parse_args(args)
 
     # Use reversed order so that the first srcdir has higher priority to override keys.
-    all_properties = merge_properties('region.properties', reversed(opts.srcdir))
+    properties = merge_properties('region.properties', reversed(opts.srcdir))
-    names = get_site_list_from_properties(all_properties)
+    names = properties.get_list('browser.suggestedsites.list')
     if opts.verbose:
         print('Reading {len} suggested sites: {names}'.format(len=len(names), names=names))
 
@@ -128,7 +90,7 @@ def main(args):
     # respective image URL.
     sites = []
     for name in names:
-        site = get_site_from_properties(name, all_properties)
+        site = properties.get_dict('browser.suggestedsites.{name}'.format(name=name), required_keys=('title', 'url', 'bgcolor'))
         site['imageurl'] = image_url_template.format(name=name)
         sites.append(site)
 

python/mozbuild/mozbuild/dotproperties.py

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains utility functions for reading .properties files, like
+# region.properties.
+
+
+from __future__ import unicode_literals
+
+import re
+import os
+import sys
+
+if sys.version_info[0] == 3:
+    str_type = str
+else:
+    str_type = basestring
+
+class DotProperties:
+    r'''A thin representation of a key=value .properties file.'''
+
+    def __init__(self, file=None):
+        self._properties = {}
+        if file:
+            self.update(file)
+
+    def update(self, file):
+        '''Updates properties from a file name or file-like object.
+
+        Ignores empty lines and comment lines.'''
+
+        if isinstance(file, str_type):
+            f = open(file, 'rt')
+        else:
+            f = file
+
+        for l in f.readlines():
+            line = l.strip()
+            if not line or line.startswith('#'):
+                continue
+            (k, v) = re.split('\s*=\s*', line, 1)
+            self._properties[k] = v
+
+    def get(self, key, default=None):
+        return self._properties.get(key, default)
+
+    def get_list(self, prefix):
+        '''Turns {'list.0':'foo', 'list.1':'bar'} into ['foo', 'bar'].
+
+        Returns [] to indicate an empty or missing list.'''
+
+        if not prefix.endswith('.'):
+            prefix = prefix + '.'
+        indexes = []
+        for k, v in self._properties.iteritems():
+            if not k.startswith(prefix):
+                continue
+            indexes.append(int(k[len(prefix):]))
+        return [self._properties[prefix + str(index)] for index in sorted(indexes)]
+
+    def get_dict(self, prefix, required_keys=[]):
+        '''Turns {'foo.title':'title', ...} into {'title':'title', ...}.
+
+        If |required_keys| is present, it must be an iterable of required key
+        names.  If a required key is not present, ValueError is thrown.
+
+        Returns {} to indicate an empty or missing dict.'''
+
+        if not prefix.endswith('.'):
+            prefix = prefix + '.'
+
+        D = dict((k[len(prefix):], v) for k, v in self._properties.iteritems() if k.startswith(prefix))
+
+        for required_key in required_keys:
+            if not required_key in D:
+                raise ValueError('Required key %s not present' % required_key)
+
+        return D

python/mozbuild/mozbuild/test/test_dotproperties.py

@@ -0,0 +1,89 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import sys
+import unittest
+
+from StringIO import StringIO
+
+from mozbuild.dotproperties import (
+    DotProperties,
+)
+
+from mozunit import (
+    main,
+)
+
+if sys.version_info[0] == 3:
+    str_type = 'str'
+else:
+    str_type = 'unicode'
+
+
+class TestDotProperties(unittest.TestCase):
+    def test_get(self):
+        contents = StringIO('''
+key=value
+''')
+        p = DotProperties(contents)
+        self.assertEqual(p.get('missing'), None)
+        self.assertEqual(p.get('missing', 'default'), 'default')
+        self.assertEqual(p.get('key'), 'value')
+
+
+    def test_update(self):
+        contents = StringIO('''
+old=old value
+key=value
+''')
+        p = DotProperties(contents)
+        self.assertEqual(p.get('old'), 'old value')
+        self.assertEqual(p.get('key'), 'value')
+
+        new_contents = StringIO('''
+key=new value
+''')
+        p.update(new_contents)
+        self.assertEqual(p.get('old'), 'old value')
+        self.assertEqual(p.get('key'), 'new value')
+
+
+    def test_get_list(self):
+        contents = StringIO('''
+list.0=A
+list.1=B
+list.2=C
+
+order.1=B
+order.0=A
+order.2=C
+''')
+        p = DotProperties(contents)
+        self.assertEqual(p.get_list('missing'), [])
+        self.assertEqual(p.get_list('list'), ['A', 'B', 'C'])
+        self.assertEqual(p.get_list('order'), ['A', 'B', 'C'])
+
+
+    def test_get_dict(self):
+        contents = StringIO('''
+A.title=title A
+
+B.title=title B
+B.url=url B
+''')
+        p = DotProperties(contents)
+        self.assertEqual(p.get_dict('missing'), {})
+        self.assertEqual(p.get_dict('A'), {'title': 'title A'})
+        self.assertEqual(p.get_dict('B'), {'title': 'title B', 'url': 'url B'})
+        with self.assertRaises(ValueError):
+            p.get_dict('A', required_keys=['title', 'url'])
+        with self.assertRaises(ValueError):
+            p.get_dict('missing', required_keys=['key'])
+
+
+if __name__ == '__main__':
+    main()