bug 763903: regularly run mozconfig comparisons for firefox. r=ted/bhearsum

--HG--
extra : rebase_source : 84a54e6362d8d6cef6bbd0a88c1f59e3fa7972b6

browser/Makefile.in

@@ -19,3 +19,6 @@ ifdef MOZ_MAINTENANCE_SERVICE
 	$(MAKE) -C installer/windows maintenanceservice_installer
 endif
 endif
+
+check::
+	$(PYTHON) $(topsrcdir)/build/compare-mozconfig/compare-mozconfigs-wrapper.py

browser/config/mozconfigs/whitelist

@@ -0,0 +1,90 @@
+# 'nightly' contains things that are in nightly mozconfigs and allowed to be missing from release builds.
+# Other keys in whitelist contain things are in that branches mozconfigs and allowed to be missing from nightly builds.
+whitelist = {
+    'release': {},
+    'nightly': {},
+    }
+
+all_platforms = ['win32', 'linux32', 'linux64', 'macosx-universal']
+
+for platform in all_platforms:
+    whitelist['nightly'][platform] = [
+        'ac_add_options --enable-update-channel=nightly',
+        'ac_add_options --enable-profiling',
+        'mk_add_options CLIENT_PY_ARGS="--hg-options=\'--verbose --time\' --hgtool=../tools/buildfarm/utils/hgtool.py --skip-chatzilla --skip-comm --skip-inspector --skip-venkman --tinderbox-print"'
+    ]
+
+for platform in ['linux32', 'linux64', 'macosx-universal']:
+    whitelist['nightly'][platform] += [
+        'ac_add_options --enable-codesighs',
+        'mk_add_options MOZ_MAKE_FLAGS="-j4"',
+    ]
+
+for platform in ['linux32', 'linux64', 'macosx-universal', 'win32']:
+    whitelist['nightly'][platform] += ['ac_add_options --enable-signmar']
+    whitelist['nightly'][platform] += ['ac_add_options --enable-js-diagnostics']
+
+whitelist['nightly']['linux32'] += [
+    'CXX=$REAL_CXX',
+    'CXX="ccache $REAL_CXX"',
+    'CC="ccache $REAL_CC"',
+    'mk_add_options PROFILE_GEN_SCRIPT=@TOPSRCDIR@/build/profile_pageloader.pl',
+    'ac_add_options --with-ccache=/usr/bin/ccache',
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+    'STRIP_FLAGS="--strip-debug"',
+    'ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling',
+]
+
+whitelist['nightly']['linux64'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+    'STRIP_FLAGS="--strip-debug"',
+    'ac_add_options --with-ccache=/usr/bin/ccache',
+    'ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling',
+]
+
+whitelist['nightly']['macosx-universal'] += [
+    'ac_add_options --with-macbundlename-prefix=Firefox',
+    'mk_add_options MOZ_MAKE_FLAGS="-j12"',
+    'ac_add_options --with-ccache',
+    'ac_add_options --disable-install-strip',
+    'ac_add_options --enable-instruments',
+    'ac_add_options --enable-dtrace',
+]
+
+whitelist['nightly']['win32'] += [
+    '. $topsrcdir/configs/mozilla2/win32/include/choose-make-flags',
+    'mk_add_options MOZ_MAKE_FLAGS=-j1',
+    'if test "$IS_NIGHTLY" != ""; then',
+    'ac_add_options --disable-auto-deps',
+    'fi',
+    'ac_add_options --enable-metro',
+]
+
+for platform in all_platforms:
+    whitelist['release'][platform] = [
+        'ac_add_options --enable-update-channel=release',
+        'ac_add_options --enable-official-branding',
+        'mk_add_options MOZ_MAKE_FLAGS="-j4"',
+        'export BUILDING_RELEASE=1',
+    ]
+whitelist['release']['win32'] += ['mk_add_options MOZ_PGO=1']
+whitelist['release']['linux32'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    'mk_add_options MOZ_PGO=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+]
+whitelist['release']['linux64'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    'mk_add_options MOZ_PGO=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+]
+
+if __name__ == '__main__':
+    import pprint
+    pprint.pprint(whitelist)

build/compare-mozconfig/compare-mozconfigs-wrapper.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import subprocess
+import sys
+from os import path
+from buildconfig import substs
+
+def determine_platform():
+    platform_mapping = {'WINNT': {'x86_64': 'win64',
+                                  'i386': 'win32'},
+                        'Darwin': {'x86_64': 'macosx-universal',
+                                   'i386':'macosx-universal'},
+                        'Linux': {'x86_64': 'linux64',
+                                  'i386': 'linux32'}}
+
+    os_type = substs['OS_TARGET']
+    cpu_type = substs['TARGET_CPU']
+    return platform_mapping.get(os_type, {}).get(cpu_type, None)
+
+def main():
+    """ A wrapper script that calls compare-mozconfig.py
+    based on the platform that the machine is building for"""
+    platform = determine_platform()
+
+    if platform is not None:
+        python_exe = substs['PYTHON']
+        topsrcdir = substs['top_srcdir']
+
+        # construct paths and args for compare-mozconfig
+        browser_dir = path.join(topsrcdir, 'browser')
+        script_path = path.join(topsrcdir, 'build/compare-mozconfig/compare-mozconfigs.py')
+        whitelist_path = path.join(browser_dir, 'config/mozconfigs/whitelist')
+        beta_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'beta')
+        release_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'release')
+        nightly_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'nightly')
+
+        # compare beta vs nightly
+        ret_code = subprocess.call([python_exe, script_path, '--whitelist',
+                                    whitelist_path, '--no-download',
+                                    platform + ',' + beta_mozconfig_path +
+                                    ',' + nightly_mozconfig_path])
+
+        if ret_code > 0:
+            sys.exit(ret_code)
+
+        # compare release vs nightly
+        ret_code = subprocess.call([python_exe, script_path, '--whitelist',
+                                    whitelist_path, '--no-download',
+                                    platform + ',' + release_mozconfig_path +
+                                    ',' + nightly_mozconfig_path])

build/compare-mozconfig/compare-mozconfigs.py

@@ -0,0 +1,68 @@
+#!/usr/bin/python
+import logging
+import os
+import site
+import sys
+import urllib2
+
+site.addsitedir(os.path.join(os.path.dirname(__file__), "../../lib/python"))
+
+from release.sanity import verify_mozconfigs
+from release.info import readConfig
+from util.hg import make_hg_url
+
+FAILURE_CODE = 1
+SUCCESS_CODE = 0
+
+def get_mozconfig(path, options):
+    """Consumes a path and returns a list of lines from
+    the mozconfig file. If download is required, the path
+    specified should be relative to the root of the hg
+    repository e.g browser/config/mozconfigs/linux32/nightly"""
+    if options.no_download:
+        return open(path, 'r').readlines()
+    else:
+        url = make_hg_url(options.hghost, options.branch, 'http',
+                    options.revision, path)
+        return urllib2.urlopen(url).readlines()
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser()
+
+    parser.add_option('--branch', dest='branch')
+    parser.add_option('--revision', dest='revision')
+    parser.add_option('--hghost', dest='hghost', default='hg.mozilla.org')
+    parser.add_option('--whitelist', dest='whitelist')
+    parser.add_option('--no-download', action='store_true', dest='no_download',
+                      default=False)
+    options, args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+
+    missing_args = options.branch is None or options.revision is None
+    if not options.no_download and missing_args:
+        logging.error('Not enough arguments to download mozconfigs')
+        sys.exit(FAILURE_CODE)
+
+    mozconfig_whitelist = readConfig(options.whitelist, ['whitelist'])
+
+    for arg in args:
+        platform, mozconfig_path, nightly_mozconfig_path = arg.split(',')
+
+        mozconfig_lines = get_mozconfig(mozconfig_path, options)
+        nightly_mozconfig_lines = get_mozconfig(nightly_mozconfig_path, options)
+
+        mozconfig_pair = (mozconfig_path, mozconfig_lines)
+        nightly_mozconfig_pair = (nightly_mozconfig_path,
+                                  nightly_mozconfig_lines)
+
+        passed = verify_mozconfigs(mozconfig_pair, nightly_mozconfig_pair,
+                                   platform, mozconfig_whitelist)
+
+        if passed:
+            logging.info('Mozconfig check passed!')
+        else:
+            logging.error('Mozconfig check failed!')
+            sys.exit(FAILURE_CODE)
+    sys.exit(SUCCESS_CODE)

build/release/info.py

@@ -0,0 +1,218 @@
+from datetime import datetime
+import os
+from os import path
+import re
+import shutil
+import sys
+from urllib2 import urlopen
+
+from release.paths import makeCandidatesDir
+
+import logging
+log = logging.getLogger(__name__)
+
+# If version has two parts with no trailing specifiers like "rc", we
+# consider it a "final" release for which we only create a _RELEASE tag.
+FINAL_RELEASE_REGEX = "^\d+\.\d+$"
+
+
+class ConfigError(Exception):
+    pass
+
+
+def getBuildID(platform, product, version, buildNumber, nightlyDir='nightly',
+               server='stage.mozilla.org'):
+    infoTxt = makeCandidatesDir(product, version, buildNumber, nightlyDir,
+                                protocol='http', server=server) + \
+        '%s_info.txt' % platform
+    try:
+        buildInfo = urlopen(infoTxt).read()
+    except:
+        log.error("Failed to retrieve %s" % infoTxt)
+        raise
+
+    for line in buildInfo.splitlines():
+        key, value = line.rstrip().split('=', 1)
+        if key == 'buildID':
+            return value
+
+
+def findOldBuildIDs(product, version, buildNumber, platforms,
+                    nightlyDir='nightly', server='stage.mozilla.org'):
+    ids = {}
+    if buildNumber <= 1:
+        return ids
+    for n in range(1, buildNumber):
+        for platform in platforms:
+            if platform not in ids:
+                ids[platform] = []
+            try:
+                id = getBuildID(platform, product, version, n, nightlyDir,
+                                server)
+                ids[platform].append(id)
+            except Exception, e:
+                log.error("Hit exception: %s" % e)
+    return ids
+
+
+def getReleaseConfigName(product, branch, version=None, staging=False):
+    # XXX: Horrible hack for bug 842741. Because Thunderbird release
+    # and esr both build out of esr17 repositories we'll bump the wrong
+    # config for release without this.
+    if product == 'thunderbird' and 'esr17' in branch and version and 'esr' not in version:
+        cfg = 'release-thunderbird-comm-release.py'
+    else:
+        cfg = 'release-%s-%s.py' % (product, branch)
+    if staging:
+        cfg = 'staging_%s' % cfg
+    return cfg
+
+
+def readReleaseConfig(configfile, required=[]):
+    return readConfig(configfile, keys=['releaseConfig'], required=required)
+
+
+def readBranchConfig(dir, localconfig, branch, required=[]):
+    shutil.copy(localconfig, path.join(dir, "localconfig.py"))
+    oldcwd = os.getcwd()
+    os.chdir(dir)
+    sys.path.append(".")
+    try:
+        return readConfig("config.py", keys=['BRANCHES', branch],
+                          required=required)
+    finally:
+        os.chdir(oldcwd)
+        sys.path.remove(".")
+
+
+def readConfig(configfile, keys=[], required=[]):
+    c = {}
+    execfile(configfile, c)
+    for k in keys:
+        c = c[k]
+    items = c.keys()
+    err = False
+    for key in required:
+        if key not in items:
+            err = True
+            log.error("Required item `%s' missing from %s" % (key, c))
+    if err:
+        raise ConfigError("Missing at least one item in config, see above")
+    return c
+
+
+def isFinalRelease(version):
+    return bool(re.match(FINAL_RELEASE_REGEX, version))
+
+
+def getBaseTag(product, version):
+    product = product.upper()
+    version = version.replace('.', '_')
+    return '%s_%s' % (product, version)
+
+
+def getTags(baseTag, buildNumber, buildTag=True):
+    t = ['%s_RELEASE' % baseTag]
+    if buildTag:
+        t.append('%s_BUILD%d' % (baseTag, int(buildNumber)))
+    return t
+
+
+def getRuntimeTag(tag):
+    return "%s_RUNTIME" % tag
+
+
+def getReleaseTag(tag):
+    return "%s_RELEASE" % tag
+
+
+def generateRelbranchName(version, prefix='GECKO'):
+    return '%s%s_%s_RELBRANCH' % (
+        prefix, version.replace('.', ''),
+        datetime.now().strftime('%Y%m%d%H'))
+
+
+def getReleaseName(product, version, buildNumber):
+    return '%s-%s-build%s' % (product.title(), version, str(buildNumber))
+
+
+def getRepoMatchingBranch(branch, sourceRepositories):
+    for sr in sourceRepositories.values():
+        if branch in sr['path']:
+            return sr
+    return None
+
+
+def fileInfo(filepath, product):
+    """Extract information about a release file.  Returns a dictionary with the
+    following keys set:
+    'product', 'version', 'locale', 'platform', 'contents', 'format',
+    'pathstyle'
+
+    'contents' is one of 'complete', 'installer'
+    'format' is one of 'mar' or 'exe'
+    'pathstyle' is either 'short' or 'long', and refers to if files are all in
+        one directory, with the locale as part of the filename ('short' paths,
+        firefox 3.0 style filenames), or if the locale names are part of the
+        directory structure, but not the file name itself ('long' paths,
+        firefox 3.5+ style filenames)
+    """
+    try:
+        # Mozilla 1.9.0 style (aka 'short') paths
+        # e.g. firefox-3.0.12.en-US.win32.complete.mar
+        filename = os.path.basename(filepath)
+        m = re.match("^(%s)-([0-9.]+)\.([-a-zA-Z]+)\.(win32)\.(complete|installer)\.(mar|exe)$" % product, filename)
+        if not m:
+            raise ValueError("Could not parse: %s" % filename)
+        return {'product': m.group(1),
+                'version': m.group(2),
+                'locale': m.group(3),
+                'platform': m.group(4),
+                'contents': m.group(5),
+                'format': m.group(6),
+                'pathstyle': 'short',
+                'leading_path': '',
+                }
+    except:
+        # Mozilla 1.9.1 and on style (aka 'long') paths
+        # e.g. update/win32/en-US/firefox-3.5.1.complete.mar
+        #      win32/en-US/Firefox Setup 3.5.1.exe
+        ret = {'pathstyle': 'long'}
+        if filepath.endswith('.mar'):
+            ret['format'] = 'mar'
+            m = re.search("update/(win32|linux-i686|linux-x86_64|mac|mac64)/([-a-zA-Z]+)/(%s)-(\d+\.\d+(?:\.\d+)?(?:\w+(?:\d+)?)?)\.(complete)\.mar" % product, filepath)
+            if not m:
+                raise ValueError("Could not parse: %s" % filepath)
+            ret['platform'] = m.group(1)
+            ret['locale'] = m.group(2)
+            ret['product'] = m.group(3)
+            ret['version'] = m.group(4)
+            ret['contents'] = m.group(5)
+            ret['leading_path'] = ''
+        elif filepath.endswith('.exe'):
+            ret['format'] = 'exe'
+            ret['contents'] = 'installer'
+            # EUballot builds use a different enough style of path than others
+            # that we can't catch them in the same regexp
+            if filepath.find('win32-EUballot') != -1:
+                ret['platform'] = 'win32'
+                m = re.search("(win32-EUballot/)([-a-zA-Z]+)/((?i)%s) Setup (\d+\.\d+(?:\.\d+)?(?:\w+\d+)?(?:\ \w+\ \d+)?)\.exe" % product, filepath)
+                if not m:
+                    raise ValueError("Could not parse: %s" % filepath)
+                ret['leading_path'] = m.group(1)
+                ret['locale'] = m.group(2)
+                ret['product'] = m.group(3).lower()
+                ret['version'] = m.group(4)
+            else:
+                m = re.search("(partner-repacks/[-a-zA-Z0-9_]+/|)(win32|mac|linux-i686)/([-a-zA-Z]+)/((?i)%s) Setup (\d+\.\d+(?:\.\d+)?(?:\w+(?:\d+)?)?(?:\ \w+\ \d+)?)\.exe" % product, filepath)
+                if not m:
+                    raise ValueError("Could not parse: %s" % filepath)
+                ret['leading_path'] = m.group(1)
+                ret['platform'] = m.group(2)
+                ret['locale'] = m.group(3)
+                ret['product'] = m.group(4).lower()
+                ret['version'] = m.group(5)
+        else:
+            raise ValueError("Unknown filetype for %s" % filepath)
+
+        return ret

build/release/sanity.py

@@ -0,0 +1,124 @@
+import difflib
+import logging
+import re
+import urllib2
+from util.commands import run_cmd, get_output
+from util.hg import get_repo_name, make_hg_url
+from subprocess import CalledProcessError
+
+log = logging.getLogger(__name__)
+
+
+def check_buildbot():
+    """check if buildbot command works"""
+    try:
+        run_cmd(['buildbot', '--version'])
+    except CalledProcessError:
+        log.error("FAIL: buildbot command doesn't work", exc_info=True)
+        raise
+
+
+def find_version(contents, versionNumber):
+    """Given an open readable file-handle look for the occurrence
+       of the version # in the file"""
+    ret = re.search(re.compile(re.escape(versionNumber), re.DOTALL), contents)
+    return ret
+
+
+def locale_diff(locales1, locales2):
+    """ accepts two lists and diffs them both ways, returns any differences
+    found """
+    diff_list = [locale for locale in locales1 if not locale in locales2]
+    diff_list.extend(locale for locale in locales2 if not locale in locales1)
+    return diff_list
+
+
+def get_buildbot_username_param():
+    cmd = ['buildbot', 'sendchange', '--help']
+    output = get_output(cmd)
+    if "-W, --who=" in output:
+        return "--who"
+    else:
+        return "--username"
+
+
+def sendchange(branch, revision, username, master, products):
+    """Send the change to buildbot to kick off the release automation"""
+    if isinstance(products, basestring):
+        products = [products]
+    cmd = [
+        'buildbot',
+        'sendchange',
+        get_buildbot_username_param(),
+        username,
+        '--master',
+        master,
+        '--branch',
+        branch,
+        '-p',
+        'products:%s' % ','.join(products),
+        '-p',
+        'script_repo_revision:%s' % revision,
+        'release_build'
+    ]
+    logging.info("Executing: %s" % cmd)
+    run_cmd(cmd)
+
+
+def verify_mozconfigs(mozconfig_pair, nightly_mozconfig_pair, platform,
+                      mozconfigWhitelist={}):
+    """Compares mozconfig to nightly_mozconfig and compare to an optional
+    whitelist of known differences. mozconfig_pair and nightly_mozconfig_pair
+    are pairs containing the mozconfig's identifier and the list of lines in
+    the mozconfig."""
+
+    # unpack the pairs to get the names, the names are just for
+    # identifying the mozconfigs when logging the error messages
+    mozconfig_name, mozconfig_lines = mozconfig_pair
+    nightly_mozconfig_name, nightly_mozconfig_lines = nightly_mozconfig_pair
+
+    missing_args = mozconfig_lines == [] or nightly_mozconfig_lines == []
+    if missing_args:
+        log.info("Missing mozconfigs to compare for %s" % platform)
+        return False
+
+    success = True
+
+    diffInstance = difflib.Differ()
+    diff_result = diffInstance.compare(mozconfig_lines, nightly_mozconfig_lines)
+    diffList = list(diff_result)
+
+    for line in diffList:
+        clean_line = line[1:].strip()
+        if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
+            # skip comment lines
+            if clean_line.startswith('#'):
+                continue
+            # compare to whitelist
+            message = ""
+            if line[0] == '-':
+                if platform in mozconfigWhitelist.get('release', {}):
+                    if clean_line in \
+                            mozconfigWhitelist['release'][platform]:
+                        continue
+            elif line[0] == '+':
+                if platform in mozconfigWhitelist.get('nightly', {}):
+                    if clean_line in \
+                            mozconfigWhitelist['nightly'][platform]:
+                        continue
+                    else:
+                        log.warning("%s not in %s %s!" % (
+                            clean_line, platform,
+                            mozconfigWhitelist['nightly'][platform]))
+            else:
+                log.error("Skipping line %s!" % line)
+                continue
+            message = "found in %s but not in %s: %s"
+            if line[0] == '-':
+                log.error(message % (mozconfig_name,
+                                     nightly_mozconfig_name, clean_line))
+            else:
+                log.error(message % (nightly_mozconfig_name,
+                                     mozconfig_name, clean_line))
+            success = False
+    return success

build/util/hg.py

@@ -0,0 +1,611 @@
+"""Functions for interacting with hg"""
+import os
+import re
+import subprocess
+from urlparse import urlsplit
+from ConfigParser import RawConfigParser
+
+from util.commands import run_cmd, get_output, remove_path
+from util.retry import retry
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class DefaultShareBase:
+    pass
+DefaultShareBase = DefaultShareBase()
+
+
+class HgUtilError(Exception):
+    pass
+
+
+def _make_absolute(repo):
+    if repo.startswith("file://"):
+        path = repo[len("file://"):]
+        repo = "file://%s" % os.path.abspath(path)
+    elif "://" not in repo:
+        repo = os.path.abspath(repo)
+    return repo
+
+
+def make_hg_url(hgHost, repoPath, protocol='https', revision=None,
+                filename=None):
+    """construct a valid hg url from a base hg url (hg.mozilla.org),
+    repoPath, revision and possible filename"""
+    base = '%s://%s' % (protocol, hgHost)
+    repo = '/'.join(p.strip('/') for p in [base, repoPath])
+    if not filename:
+        if not revision:
+            return repo
+        else:
+            return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])
+    else:
+        assert revision
+        return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])
+
+
+def get_repo_name(repo):
+    return repo.rstrip('/').split('/')[-1]
+
+
+def get_repo_path(repo):
+    repo = _make_absolute(repo)
+    if repo.startswith("/"):
+        return repo.lstrip("/")
+    else:
+        return urlsplit(repo).path.lstrip("/")
+
+
+def get_revision(path):
+    """Returns which revision directory `path` currently has checked out."""
+    return get_output(['hg', 'parent', '--template', '{node|short}'], cwd=path)
+
+
+def get_branch(path):
+    return get_output(['hg', 'branch'], cwd=path).strip()
+
+
+def get_branches(path):
+    branches = []
+    for line in get_output(['hg', 'branches', '-c'], cwd=path).splitlines():
+        branches.append(line.split()[0])
+    return branches
+
+
+def is_hg_cset(rev):
+    """Retruns True if passed revision represents a valid HG revision
+    (long or short(er) 40 bit hex)"""
+    try:
+        int(rev, 16)
+        return True
+    except (TypeError, ValueError):
+        return False
+
+
+def hg_ver():
+    """Returns the current version of hg, as a tuple of
+    (major, minor, build)"""
+    ver_string = get_output(['hg', '-q', 'version'])
+    match = re.search("\(version ([0-9.]+)\)", ver_string)
+    if match:
+        bits = match.group(1).split(".")
+        if len(bits) < 3:
+            bits += (0,)
+        ver = tuple(int(b) for b in bits)
+    else:
+        ver = (0, 0, 0)
+    log.debug("Running hg version %s", ver)
+    return ver
+
+
+def purge(dest):
+    """Purge the repository of all untracked and ignored files."""
+    try:
+        run_cmd(['hg', '--config', 'extensions.purge=', 'purge',
+                 '-a', '--all', dest], cwd=dest)
+    except subprocess.CalledProcessError, e:
+        log.debug('purge failed: %s' % e)
+        raise
+
+
+def update(dest, branch=None, revision=None):
+    """Updates working copy `dest` to `branch` or `revision`.  If neither is
+    set then the working copy will be updated to the latest revision on the
+    current branch.  Local changes will be discarded."""
+    # If we have a revision, switch to that
+    if revision is not None:
+        cmd = ['hg', 'update', '-C', '-r', revision]
+        run_cmd(cmd, cwd=dest)
+    else:
+        # Check & switch branch
+        local_branch = get_output(['hg', 'branch'], cwd=dest).strip()
+
+        cmd = ['hg', 'update', '-C']
+
+        # If this is different, checkout the other branch
+        if branch and branch != local_branch:
+            cmd.append(branch)
+
+        run_cmd(cmd, cwd=dest)
+    return get_revision(dest)
+
+
+def clone(repo, dest, branch=None, revision=None, update_dest=True,
+          clone_by_rev=False, mirrors=None, bundles=None):
+    """Clones hg repo and places it at `dest`, replacing whatever else is
+    there.  The working copy will be empty.
+
+    If `revision` is set, only the specified revision and its ancestors will
+    be cloned.
+
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and clone from the mirrors before
+    cloning from `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it. If successful, will pull in new revisions from mirrors or
+    the master repo. If unbundling fails, will fall back to doing a regular
+    clone from mirrors or the master repo.
+
+    Regardless of how the repository ends up being cloned, the 'default' path
+    will point to `repo`.
+    """
+    if os.path.exists(dest):
+        remove_path(dest)
+
+    if bundles:
+        log.info("Attempting to initialize clone with bundles")
+        for bundle in bundles:
+            if os.path.exists(dest):
+                remove_path(dest)
+            init(dest)
+            log.info("Trying to use bundle %s", bundle)
+            try:
+                if not unbundle(bundle, dest):
+                    remove_path(dest)
+                    continue
+                adjust_paths(dest, default=repo)
+                # Now pull / update
+                return pull(repo, dest, update_dest=update_dest,
+                            mirrors=mirrors, revision=revision, branch=branch)
+            except Exception:
+                remove_path(dest)
+                log.exception("Problem unbundling/pulling from %s", bundle)
+                continue
+        else:
+            log.info("Using bundles failed; falling back to clone")
+
+    if mirrors:
+        log.info("Attempting to clone from mirrors")
+        for mirror in mirrors:
+            log.info("Cloning from %s", mirror)
+            try:
+                retval = clone(mirror, dest, branch, revision,
+                               update_dest=update_dest, clone_by_rev=clone_by_rev)
+                adjust_paths(dest, default=repo)
+                return retval
+            except:
+                log.exception("Problem cloning from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+            # We may have a partial repo here; mercurial() copes with that
+            # We need to make sure our paths are correct though
+            if os.path.exists(os.path.join(dest, '.hg')):
+                adjust_paths(dest, default=repo)
+            return mercurial(repo, dest, branch, revision, autoPurge=True,
+                             update_dest=update_dest, clone_by_rev=clone_by_rev)
+
+    cmd = ['hg', 'clone']
+    if not update_dest:
+        cmd.append('-U')
+
+    if clone_by_rev:
+        if revision:
+            cmd.extend(['-r', revision])
+        elif branch:
+            # hg >= 1.6 supports -b branch for cloning
+            ver = hg_ver()
+            if ver >= (1, 6, 0):
+                cmd.extend(['-b', branch])
+
+    cmd.extend([repo, dest])
+    run_cmd(cmd)
+
+    if update_dest:
+        return update(dest, branch, revision)
+
+
+def common_args(revision=None, branch=None, ssh_username=None, ssh_key=None):
+    """Fill in common hg arguments, encapsulating logic checks that depend on
+       mercurial versions and provided arguments"""
+    args = []
+    if ssh_username or ssh_key:
+        opt = ['-e', 'ssh']
+        if ssh_username:
+            opt[1] += ' -l %s' % ssh_username
+        if ssh_key:
+            opt[1] += ' -i %s' % ssh_key
+        args.extend(opt)
+    if revision:
+        args.extend(['-r', revision])
+    elif branch:
+        if hg_ver() >= (1, 6, 0):
+            args.extend(['-b', branch])
+    return args
+
+
+def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
+    """Pulls changes from hg repo and places it in `dest`.
+
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and pull from the mirrors first before
+    `repo`."""
+
+    if mirrors:
+        for mirror in mirrors:
+            try:
+                return pull(mirror, dest, update_dest=update_dest, **kwargs)
+            except:
+                log.exception("Problem pulling from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+
+    # Convert repo to an absolute path if it's a local repository
+    repo = _make_absolute(repo)
+    cmd = ['hg', 'pull']
+    # Don't pass -r to "hg pull", except when it's a valid HG revision.
+    # Pulling using tag names is dangerous: it uses the local .hgtags, so if
+    # the tag has moved on the remote side you won't pull the new revision the
+    # remote tag refers to.
+    pull_kwargs = kwargs.copy()
+    if 'revision' in pull_kwargs and \
+       not is_hg_cset(pull_kwargs['revision']):
+        del pull_kwargs['revision']
+
+    cmd.extend(common_args(**pull_kwargs))
+
+    cmd.append(repo)
+    run_cmd(cmd, cwd=dest)
+
+    if update_dest:
+        branch = None
+        if 'branch' in kwargs and kwargs['branch']:
+            branch = kwargs['branch']
+        revision = None
+        if 'revision' in kwargs and kwargs['revision']:
+            revision = kwargs['revision']
+        return update(dest, branch=branch, revision=revision)
+
+# Defines the places of attributes in the tuples returned by `out'
+REVISION, BRANCH = 0, 1
+
+
+def out(src, remote, **kwargs):
+    """Check for outgoing changesets present in a repo"""
+    cmd = ['hg', '-q', 'out', '--template', '{node} {branches}\n']
+    cmd.extend(common_args(**kwargs))
+    cmd.append(remote)
+    if os.path.exists(src):
+        try:
+            revs = []
+            for line in get_output(cmd, cwd=src).rstrip().split("\n"):
+                try:
+                    rev, branch = line.split()
+                # Mercurial displays no branch at all if the revision is on
+                # "default"
+                except ValueError:
+                    rev = line.rstrip()
+                    branch = "default"
+                revs.append((rev, branch))
+            return revs
+        except subprocess.CalledProcessError, inst:
+            # In some situations, some versions of Mercurial return "1"
+            # if no changes are found, so we need to ignore this return code
+            if inst.returncode == 1:
+                return []
+            raise
+
+
+def push(src, remote, push_new_branches=True, force=False, **kwargs):
+    cmd = ['hg', 'push']
+    cmd.extend(common_args(**kwargs))
+    if force:
+        cmd.append('-f')
+    if push_new_branches:
+        cmd.append('--new-branch')
+    cmd.append(remote)
+    run_cmd(cmd, cwd=src)
+
+
+def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
+              shareBase=DefaultShareBase, allowUnsharedLocalClones=False,
+              clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False):
+    """Makes sure that `dest` is has `revision` or `branch` checked out from
+    `repo`.
+
+    Do what it takes to make that happen, including possibly clobbering
+    dest.
+
+    If allowUnsharedLocalClones is True and we're trying to use the share
+    extension but fail, then we will be able to clone from the shared repo to
+    our destination.  If this is False, the default, then if we don't have the
+    share extension we will just clone from the remote repository.
+
+    If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
+    This is slower, but useful when cloning repos with lots of heads.
+
+    If `mirrors` is set, will try and use the mirrors before `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it instead of doing a full clone. If successful, will pull in
+    new revisions from mirrors or the master repo. If unbundling fails, will
+    fall back to doing a regular clone from mirrors or the master repo.
+    """
+    dest = os.path.abspath(dest)
+    if shareBase is DefaultShareBase:
+        shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
+
+    log.info("Reporting hg version in use")
+    cmd = ['hg', '-q', 'version']
+    run_cmd(cmd, cwd='.')
+
+    if shareBase:
+        # Check that 'hg share' works
+        try:
+            log.info("Checking if share extension works")
+            output = get_output(['hg', 'help', 'share'], dont_log=True)
+            if 'no commands defined' in output:
+                # Share extension is enabled, but not functional
+                log.info("Disabling sharing since share extension doesn't seem to work (1)")
+                shareBase = None
+            elif 'unknown command' in output:
+                # Share extension is disabled
+                log.info("Disabling sharing since share extension doesn't seem to work (2)")
+                shareBase = None
+        except subprocess.CalledProcessError:
+            # The command failed, so disable sharing
+            log.info("Disabling sharing since share extension doesn't seem to work (3)")
+            shareBase = None
+
+    # Check that our default path is correct
+    if os.path.exists(os.path.join(dest, '.hg')):
+        hgpath = path(dest, "default")
+
+        # Make sure that our default path is correct
+        if hgpath != _make_absolute(repo):
+            log.info("hg path isn't correct (%s should be %s); clobbering",
+                     hgpath, _make_absolute(repo))
+            remove_path(dest)
+
+    # If the working directory already exists and isn't using share we update
+    # the working directory directly from the repo, ignoring the sharing
+    # settings
+    if os.path.exists(dest):
+        if not os.path.exists(os.path.join(dest, ".hg")):
+            log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
+            remove_path(dest)
+        elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
+            try:
+                if autoPurge:
+                    purge(dest)
+                return pull(repo, dest, update_dest=update_dest, branch=branch,
+                            revision=revision,
+                            mirrors=mirrors)
+            except subprocess.CalledProcessError:
+                log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
+                log.debug("Exception:", exc_info=True)
+                remove_path(dest)
+
+    # If that fails for any reason, and sharing is requested, we'll try to
+    # update the shared repository, and then update the working directory from
+    # that.
+    if shareBase:
+        sharedRepo = os.path.join(shareBase, get_repo_path(repo))
+        dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')
+
+        if os.path.exists(sharedRepo):
+            hgpath = path(sharedRepo, "default")
+
+            # Make sure that our default path is correct
+            if hgpath != _make_absolute(repo):
+                log.info("hg path isn't correct (%s should be %s); clobbering",
+                         hgpath, _make_absolute(repo))
+                # we need to clobber both the shared checkout and the dest,
+                # since hgrc needs to be in both places
+                remove_path(sharedRepo)
+                remove_path(dest)
+
+        if os.path.exists(dest_sharedPath):
+            # Make sure that the sharedpath points to sharedRepo
+            dest_sharedPath_data = os.path.normpath(
+                open(dest_sharedPath).read())
+            norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))
+            if dest_sharedPath_data != norm_sharedRepo:
+                # Clobber!
+                log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",
+                         dest_sharedPath_data, repo, norm_sharedRepo)
+                remove_path(dest)
+
+        try:
+            log.info("Updating shared repo")
+            mercurial(repo, sharedRepo, branch=branch, revision=revision,
+                      update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
+                      mirrors=mirrors, bundles=bundles, autoPurge=False)
+            if os.path.exists(dest):
+                if autoPurge:
+                    purge(dest)
+                return update(dest, branch=branch, revision=revision)
+
+            try:
+                log.info("Trying to share %s to %s", sharedRepo, dest)
+                return share(sharedRepo, dest, branch=branch, revision=revision)
+            except subprocess.CalledProcessError:
+                if not allowUnsharedLocalClones:
+                    # Re-raise the exception so it gets caught below.
+                    # We'll then clobber dest, and clone from original repo
+                    raise
+
+                log.warning("Error calling hg share from %s to %s;"
+                            "falling back to normal clone from shared repo",
+                            sharedRepo, dest)
+                # Do a full local clone first, and then update to the
+                # revision we want
+                # This lets us use hardlinks for the local clone if the OS
+                # supports it
+                clone(sharedRepo, dest, update_dest=False,
+                      mirrors=mirrors, bundles=bundles)
+                return update(dest, branch=branch, revision=revision)
+        except subprocess.CalledProcessError:
+            log.warning(
+                "Error updating %s from sharedRepo (%s): ", dest, sharedRepo)
+            log.debug("Exception:", exc_info=True)
+            remove_path(dest)
+    # end if shareBase
+
+    if not os.path.exists(os.path.dirname(dest)):
+        os.makedirs(os.path.dirname(dest))
+
+    # Share isn't available or has failed, clone directly from the source
+    return clone(repo, dest, branch, revision,
+                 update_dest=update_dest, mirrors=mirrors,
+                 bundles=bundles, clone_by_rev=clone_by_rev)
+
+
+def apply_and_push(localrepo, remote, changer, max_attempts=10,
+                   ssh_username=None, ssh_key=None, force=False):
+    """This function calls `changer' to make changes to the repo, and tries
+       its hardest to get them to the origin repo. `changer' must be a
+       callable object that receives two arguments: the directory of the local
+       repository, and the attempt number. This function will push ALL
+       changesets missing from remote."""
+    assert callable(changer)
+    branch = get_branch(localrepo)
+    changer(localrepo, 1)
+    for n in range(1, max_attempts + 1):
+        new_revs = []
+        try:
+            new_revs = out(src=localrepo, remote=remote,
+                           ssh_username=ssh_username,
+                           ssh_key=ssh_key)
+            if len(new_revs) < 1:
+                raise HgUtilError("No revs to push")
+            push(src=localrepo, remote=remote, ssh_username=ssh_username,
+                 ssh_key=ssh_key, force=force)
+            return
+        except subprocess.CalledProcessError, e:
+            log.debug("Hit error when trying to push: %s" % str(e))
+            if n == max_attempts:
+                log.debug("Tried %d times, giving up" % max_attempts)
+                for r in reversed(new_revs):
+                    run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                             r[REVISION]], cwd=localrepo)
+                raise HgUtilError("Failed to push")
+            pull(remote, localrepo, update_dest=False,
+                 ssh_username=ssh_username, ssh_key=ssh_key)
+            # After we successfully rebase or strip away heads the push is
+            # is attempted again at the start of the loop
+            try:
+                run_cmd(['hg', '--config', 'ui.merge=internal:merge',
+                         'rebase'], cwd=localrepo)
+            except subprocess.CalledProcessError, e:
+                log.debug("Failed to rebase: %s" % str(e))
+                update(localrepo, branch=branch)
+                for r in reversed(new_revs):
+                    run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                             r[REVISION]], cwd=localrepo)
+                changer(localrepo, n + 1)
+
+
+def share(source, dest, branch=None, revision=None):
+    """Creates a new working directory in "dest" that shares history with
+       "source" using Mercurial's share extension"""
+    run_cmd(['hg', 'share', '-U', source, dest])
+    return update(dest, branch=branch, revision=revision)
+
+
+def cleanOutgoingRevs(reponame, remote, username, sshKey):
+    outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
+                                          ssh_username=username,
+                                          ssh_key=sshKey))
+    for r in reversed(outgoingRevs):
+        run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                 r[REVISION]], cwd=reponame)
+
+
+def path(src, name='default'):
+    """Returns the remote path associated with "name" """
+    try:
+        return get_output(['hg', 'path', name], cwd=src).strip()
+    except subprocess.CalledProcessError:
+        return None
+
+
+def init(dest):
+    """Initializes an empty repo in `dest`"""
+    run_cmd(['hg', 'init', dest])
+
+
+def unbundle(bundle, dest):
+    """Unbundles the bundle located at `bundle` into `dest`.
+
+    `bundle` can be a local file or remote url."""
+    try:
+        get_output(['hg', 'unbundle', bundle], cwd=dest, include_stderr=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def adjust_paths(dest, **paths):
+    """Adjusts paths in `dest`/.hg/hgrc so that names in `paths` are set to
+    paths[name].
+
+    Note that any comments in the hgrc will be lost if changes are made to the
+    file."""
+    hgrc = os.path.join(dest, '.hg', 'hgrc')
+    config = RawConfigParser()
+    config.read(hgrc)
+
+    if not config.has_section('paths'):
+        config.add_section('paths')
+
+    changed = False
+    for path_name, path_value in paths.items():
+        if (not config.has_option('paths', path_name) or
+                config.get('paths', path_name) != path_value):
+            changed = True
+            config.set('paths', path_name, path_value)
+
+    if changed:
+        config.write(open(hgrc, 'w'))
+
+
+def commit(dest, msg, user=None):
+    cmd = ['hg', 'commit', '-m', msg]
+    if user:
+        cmd.extend(['-u', user])
+    run_cmd(cmd, cwd=dest)
+    return get_revision(dest)
+
+
+def tag(dest, tags, user=None, msg=None, rev=None, force=None):
+    cmd = ['hg', 'tag']
+    if user:
+        cmd.extend(['-u', user])
+    if msg:
+        cmd.extend(['-m', msg])
+    if rev:
+        cmd.extend(['-r', rev])
+    if force:
+        cmd.append('-f')
+    cmd.extend(tags)
+    run_cmd(cmd, cwd=dest)
+    return get_revision(dest)