Bug 811527 - generate_diff.py does not ensure that m-c mirror of mozbase is internally compatible;r=wlach ; DONTBUILD because NPOTB

testing/mozbase/generate_diff.py

@@ -22,8 +22,7 @@ files that show up in `hg st`.
 See: https://bugzilla.mozilla.org/show_bug.cgi?id=702832
 """
 
-from __future__ import with_statement
-
+import imp
 import optparse
 import os
 import re
@@ -32,12 +31,17 @@ import subprocess
 import sys
 import tempfile
 
+from pkg_resources import parse_version
 from subprocess import check_call as call
 
 # globals
 here = os.path.dirname(os.path.abspath(__file__))
 MOZBASE = 'git://github.com/mozilla/mozbase.git'
 version_regex = r"""PACKAGE_VERSION *= *['"]([0-9.]+)["'].*"""
+setup_development = imp.load_source('setup_development',
+                                    os.path.join(here, 'setup_development.py'))
+current_package = None
+current_package_info = {}
 
 def error(msg):
     """err out with a message"""
@@ -123,6 +127,51 @@ def parse_versions(*args):
 def version_tag(directory, version):
     return '%s-%s' % (directory, version)
 
+def setup(**kwargs):
+    """monkey-patch function for setuptools.setup"""
+    assert current_package
+    current_package_info[current_package] = kwargs
+
+def check_consistency(*package_info):
+    """checks consistency between a set of packages"""
+
+    # set versions and dependencies per package
+    versions = {}
+    dependencies = {}
+    for package in package_info:
+        name = package['name']
+        versions[name] = package['version']
+        for dep in package.get('install_requires', []):
+            dependencies.setdefault(name, []).append(dep)
+
+    func_map = {'==': tuple.__eq__,
+                '<=': tuple.__le__,
+                '>=': tuple.__ge__}
+
+    # check dependencies
+    errors = []
+    for package, deps in dependencies.items():
+        for dep in deps:
+            parsed = setup_development.dependency_info(dep)
+            if parsed['Name'] not in versions:
+                # external dependency
+                continue
+            if parsed.get('Version') is None:
+                # no version specified for dependency
+                continue
+
+            # check versions
+            func = func_map[parsed['Type']]
+            comparison = func(parse_version(versions[parsed['Name']]),
+                              parse_version(parsed['Version']))
+
+            if not comparison:
+                # an error
+                errors.append("Dependency for package '%s' failed: %s-%s not %s %s" % (package, parsed['Name'], versions[parsed['Name']], parsed['Type'], parsed['Version']))
+
+    # raise an Exception if errors exist
+    if errors:
+        raise Exception('\n'.join(errors))
 
 ###
 
@@ -151,6 +200,24 @@ def main(args=sys.argv[1:]):
         parser.exit()
     output = options.output
 
+    # gather info from current mozbase packages
+    global current_package
+    setuptools = sys.modules.get('setuptools')
+    sys.modules['setuptools'] = sys.modules[__name__]
+    try:
+        for package in setup_development.mozbase_packages:
+            current_package = package
+            imp.load_source('setup', os.path.join(here, package, 'setup.py'))
+    finally:
+        current_package = None
+        sys.modules.pop('setuptools')
+        if setuptools:
+            sys.modules['setuptools'] = setuptools
+    assert set(current_package_info.keys()) == set(setup_development.mozbase_packages)
+
+    # check consistency of current set of packages
+    check_consistency(*current_package_info.values())
+
     # calculate hg root
     hg_root = os.path.dirname(os.path.dirname(here))  # testing/mozbase
     hg_dir = os.path.join(hg_root, '.hg')
@@ -203,6 +270,30 @@ def main(args=sys.argv[1:]):
             if tag not in _tags:
                 error("Tag for '%s' -- %s -- not in tags")
 
+        # ensure that the versions to mirror are compatible with what is in m-c
+        old_package_info = current_package_info.copy()
+        setuptools = sys.modules.get('setuptools')
+        sys.modules['setuptools'] = sys.modules[__name__]
+        try:
+            for directory, version in versions:
+
+                # checkout appropriate revision of mozbase
+                tag = version_tag(directory, version)
+                checkout(src, tag)
+
+                # update the package information
+                setup_py = os.path.join(src, directory, 'setup.py')
+                current_package = directory
+                imp.load_source('setup', setup_py)
+        finally:
+            current_package = None
+            sys.modules.pop('setuptools')
+            if setuptools:
+                sys.modules['setuptools'] = setuptools
+        checkout(src, 'HEAD')
+        current_package_info['mozprocess']['version'] = '0.9'
+        check_consistency(*current_package_info.values())
+
         # copy mozbase directories to m-c
         for directory, version in versions:
 

testing/mozbase/setup_development.py

@@ -15,9 +15,9 @@ See https://wiki.mozilla.org/Auto-tools/Projects/MozBase
 
 import pkg_resources
 import os
+import subprocess
 import sys
 from optparse import OptionParser
-
 from subprocess import PIPE
 try:
     from subprocess import check_call as call
@@ -29,8 +29,9 @@ except ImportError:
 here = os.path.dirname(os.path.abspath(__file__))
 
 # all python packages
-all_packages = [i for i in os.listdir(here)
-                if os.path.exists(os.path.join(here, i, 'setup.py'))]
+mozbase_packages = [i for i in os.listdir(here)
+                    if os.path.exists(os.path.join(here, i, 'setup.py'))]
+extra_packages = ["sphinx"]
 
 def cycle_check(order, dependencies):
     """ensure no cyclic dependencies"""
@@ -46,7 +47,11 @@ def info(directory):
     assert os.path.exists(os.path.join(directory, 'setup.py'))
 
     # setup the egg info
-    call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
+    try:
+        call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
+    except subprocess.CalledProcessError:
+        print "Error running setup.py in %s" % directory
+        raise
 
     # get the .egg-info directory
     egg_info = [entry for entry in os.listdir(directory)
@@ -148,12 +153,15 @@ def main(args=sys.argv[1:]):
                       help="list what will be installed")
     options, packages = parser.parse_args(args)
 
+    install_extra_packages = False
+
     if not packages:
         # install all packages
-        packages = sorted(all_packages)
+        packages = sorted(mozbase_packages)
+        install_extra_packages = True
 
     # ensure specified packages are in the list
-    assert set(packages).issubset(all_packages), "Packages should be in %s (You gave: %s)" % (all_packages, packages)
+    assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
 
     if options.list_dependencies:
         # list the package dependencies
@@ -182,7 +190,7 @@ def main(args=sys.argv[1:]):
         flag = False
         for value in deps.values():
             for dep in value:
-                if dep in all_packages and dep not in deps:
+                if dep in mozbase_packages and dep not in deps:
                     key, value = get_dependencies(os.path.join(here, dep))
                     deps[key] = [sanitize_dependency(dep) for dep in value]
 
@@ -195,7 +203,7 @@ def main(args=sys.argv[1:]):
                 break
 
     # get the remaining names for the mapping
-    for package in all_packages:
+    for package in mozbase_packages:
         if package in mapping:
             continue
         key, value = get_dependencies(os.path.join(here, package))
@@ -216,8 +224,12 @@ def main(args=sys.argv[1:]):
             print package
         parser.exit()
 
+    # set up the packages for development
+    for package in unrolled:
+        call([sys.executable, 'setup.py', 'develop', '--no-deps'],
+             cwd=os.path.join(here, reverse_mapping[package]))
+
     # install non-mozbase dependencies
-    # (currently none on modern python)
     # these need to be installed separately and the --no-deps flag
     # subsequently used due to a bug in setuptools; see
     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
@@ -227,10 +239,10 @@ def main(args=sys.argv[1:]):
         # easy_install should be available since we rely on setuptools
         call(['easy_install', version])
 
-    # set up the packages for development
-    for package in unrolled:
-        call([sys.executable, 'setup.py', 'develop', '--no-deps'],
-             cwd=os.path.join(here, reverse_mapping[package]))
+    # install extra non-mozbase packages if desired
+    if install_extra_packages:
+        for package in extra_packages:
+            call(['easy_install', package])
 
 if __name__ == '__main__':
     main()