diff --git a/testing/mozharness/mozharness/base/vcs/hgtool.py b/testing/mozharness/mozharness/base/vcs/hgtool.py index 05eee931bfd..a8cae874938 100644 --- a/testing/mozharness/mozharness/base/vcs/hgtool.py +++ b/testing/mozharness/mozharness/base/vcs/hgtool.py @@ -1,10 +1,12 @@ import os import re import urlparse +from collections import namedtuple from mozharness.base.script import ScriptMixin from mozharness.base.log import LogMixin, OutputParser, WARNING from mozharness.base.errors import HgErrorList, VCSException +from mozharness.base.transfer import TransferMixin HgtoolErrorList = [{ 'substr': 'abort: HTTP Error 404: Not Found', @@ -28,7 +30,7 @@ class HgtoolParser(OutputParser): super(HgtoolParser, self).parse_single_line(line) -class HgtoolVCS(ScriptMixin, LogMixin): +class HgtoolVCS(ScriptMixin, LogMixin, TransferMixin): def __init__(self, log_obj=None, config=None, vcs_config=None, script_obj=None): super(HgtoolVCS, self).__init__() @@ -113,3 +115,38 @@ class HgtoolVCS(ScriptMixin, LogMixin): raise VCSException("Unable to checkout") return parser.got_revision + + def query_pushinfo(self, repository, revision): + """Query the pushdate and pushid of a repository/revision. + This is intended to be used on hg.mozilla.org/mozilla-central and + similar. It may or may not work for other hg repositories. + """ + PushInfo = namedtuple('PushInfo', ['pushid', 'pushdate']) + + try: + url = '%s/json-pushes?changeset=%s' % (repository, revision) + self.info('Pushdate URL is: %s' % url) + contents = self.retry(self.load_json_from_url, args=(url,)) + + # The contents should be something like: + # { + # "28537": { + # "changesets": [ + # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5", + # ], + # "date": 1428072488, + # "user": "user@mozilla.com" + # } + # } + # + # So we grab the first element ("28537" in this case) and then pull + # out the 'date' field. + pushid = contents.iterkeys().next() + self.info('Pushid is: %s' % pushid) + pushdate = contents[pushid]['date'] + self.info('Pushdate is: %s' % pushdate) + return PushInfo(pushid, pushdate) + + except Exception: + self.exception("Failed to get push info from hg.mozilla.org") + raise diff --git a/testing/mozharness/mozharness/base/vcs/vcsbase.py b/testing/mozharness/mozharness/base/vcs/vcsbase.py index de0ddacfd72..2382f17fea5 100755 --- a/testing/mozharness/mozharness/base/vcs/vcsbase.py +++ b/testing/mozharness/mozharness/base/vcs/vcsbase.py @@ -53,19 +53,16 @@ class VCSMixin(object): self.rmtree(dest) raise + def _get_vcs_class(self, vcs): + vcs = vcs or self.config.get('default_vcs', getattr(self, 'default_vcs', None)) + vcs_class = VCS_DICT.get(vcs) + return vcs_class + def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs): """ Check out a single repo. """ c = self.config - if not vcs: - if c.get('default_vcs'): - vcs = c['default_vcs'] - else: - try: - vcs = self.default_vcs - except AttributeError: - pass - vcs_class = VCS_DICT.get(vcs) + vcs_class = self._get_vcs_class(vcs) if not vcs_class: self.error("Running vcs_checkout with kwargs %s" % str(kwargs)) raise VCSException("No VCS set!") @@ -110,6 +107,20 @@ class VCSMixin(object): self.chdir(orig_dir) return revision_dict + def vcs_query_pushinfo(self, repository, revision, vcs=None): + """Query the pushid/pushdate of a repository/revision + Returns a namedtuple with "pushid" and "pushdate" elements + """ + vcs_class = self._get_vcs_class(vcs) + if not vcs_class: + raise VCSException("No VCS set in vcs_query_pushinfo!") + vcs_obj = vcs_class( + log_obj=self.log_obj, + config=self.config, + script_obj=self, + ) + return vcs_obj.query_pushinfo(repository, revision) + class VCSScript(VCSMixin, BaseScript): def __init__(self, **kwargs): diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 1d27da1342d..288e2e08793 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -29,7 +29,6 @@ import re from mozharness.base.config import BaseConfig, parse_config_file from mozharness.base.log import ERROR, OutputParser, FATAL from mozharness.base.script import PostScriptRun -from mozharness.base.transfer import TransferMixin from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_STATUS_DICT, \ TBPL_EXCEPTION, TBPL_RETRY, EXIT_STATUS_DICT, TBPL_WARNING, TBPL_SUCCESS, \ @@ -534,7 +533,7 @@ def generate_build_UID(): class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin, SigningMixin, VirtualenvMixin, MercurialScript, - TransferMixin, InfluxRecordingMixin): + InfluxRecordingMixin): def __init__(self, **kwargs): # objdir is referenced in _query_abs_dirs() so let's make sure we # have that attribute before calling BaseScript.__init__ @@ -559,7 +558,6 @@ class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin, self.repo_path = None self.buildid = None self.builduid = None - self.pushdate = None self.query_buildid() # sets self.buildid self.query_builduid() # sets self.builduid self.generated_build_props = False @@ -735,39 +733,6 @@ or run without that action (ie: --no-{action})" self.buildid = buildid return self.buildid - def query_pushdate(self): - if self.pushdate: - return self.pushdate - - try: - url = '%s/json-pushes?changeset=%s' % ( - self._query_repo(), - self.query_revision(), - ) - self.info('Pushdate URL is: %s' % url) - contents = self.retry(self.load_json_from_url, args=(url,)) - - # The contents should be something like: - # { - # "28537": { - # "changesets": [ - # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5", - # ], - # "date": 1428072488, - # "user": "user@mozilla.com" - # } - # } - # - # So we grab the first element ("28537" in this case) and then pull - # out the 'date' field. - self.pushdate = contents.itervalues().next()['date'] - self.info('Pushdate is: %s' % self.pushdate) - except Exception: - self.exception("Failed to get pushdate from hg.mozilla.org") - raise - - return self.pushdate - def _query_objdir(self): if self.objdir: return self.objdir @@ -1380,6 +1345,10 @@ or run without that action (ie: --no-{action})" self.warning('Skipping S3 file upload: No taskcluster credentials.') return + repo = self._query_repo() + revision = self.query_revision() + pushinfo = self.vcs_query_pushinfo(repo, revision) + # We need to create & activate the virtualenv so that we can import # taskcluster (and its dependent modules, like requests and hawk). # Normally we could create the virtualenv as an action, but due to some @@ -1411,7 +1380,7 @@ or run without that action (ie: --no-{action})" fmt = { 'index': index, 'project': self.buildbot_config['properties']['branch'], - 'head_rev': self.query_revision(), + 'head_rev': revision, 'build_product': self.config['stage_product'], 'build_name': self.query_build_name(), 'build_type': self.query_build_type(), @@ -1422,7 +1391,7 @@ or run without that action (ie: --no-{action})" self.info("Using routes: %s" % routes) tc = Taskcluster(self.branch, - self.query_pushdate(), # Use pushdate as the rank + pushinfo.pushdate, # Use pushdate as the rank client_id, access_token, self.log_obj, @@ -1431,7 +1400,7 @@ or run without that action (ie: --no-{action})" # TODO: Bug 1165980 - these should be in tree routes.extend([ "%s.buildbot.branches.%s.%s" % (index, self.branch, self.stage_platform), - "%s.buildbot.revisions.%s.%s.%s" % (index, self.query_revision(), self.branch, self.stage_platform), + "%s.buildbot.revisions.%s.%s.%s" % (index, revision, self.branch, self.stage_platform), ]) task = tc.create_task(routes) tc.claim_task(task) diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 1ce12a3abf0..216bf180f69 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -989,6 +989,16 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, self.set_buildbot_property('funsize_info', json.dumps(funsize_info), write_to_file=True) + def query_repo(self): + # Find the name of our repository + mozilla_dir = self.config['mozilla_dir'] + repo = None + for repository in self.config['repos']: + if repository.get('dest') == mozilla_dir: + repo = repository['repo'] + break + return repo + def taskcluster_upload(self): auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) credentials = {} @@ -1016,6 +1026,10 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, branch = self.config['branch'] platform = self.config['platform'] revision = self._query_revision() + repo = self.query_repo() + if not repo: + self.fatal("Unable to determine repository for querying the push info.") + pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool') routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], 'testing/taskcluster/routes.json') @@ -1038,10 +1052,10 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, } fmt.update(self.buildid_to_dict(self._query_buildid())) routes.append(template.format(**fmt)) - self.info('Using routes: %s' % routes) + self.info('Using routes: %s' % routes) tc = Taskcluster(branch, - self.query_pushdate(), + pushinfo.pushdate, # Use pushdate as the rank client_id, access_token, self.log_obj, @@ -1057,48 +1071,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, tc.create_artifact(task, upload_file) tc.report_completed(task) - def query_pushdate(self): - if self.pushdate: - return self.pushdate - - mozilla_dir = self.config['mozilla_dir'] - repo = None - for repository in self.config['repos']: - if repository.get('dest') == mozilla_dir: - repo = repository['repo'] - break - - if not repo: - self.fatal("Unable to determine repository for querying the pushdate.") - try: - url = '%s/json-pushes?changeset=%s' % ( - repo, - self._query_revision(), - ) - self.info('Pushdate URL is: %s' % url) - contents = self.retry(self.load_json_from_url, args=(url,)) - - # The contents should be something like: - # { - # "28537": { - # "changesets": [ - # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5", - # ], - # "date": 1428072488, - # "user": "user@mozilla.com" - # } - # } - # - # So we grab the first element ("28537" in this case) and then pull - # out the 'date' field. - self.pushdate = contents.itervalues().next()['date'] - self.info('Pushdate is: %s' % self.pushdate) - except Exception: - self.exception("Failed to get pushdate from hg.mozilla.org") - raise - - return self.pushdate - # main {{{ if __name__ == '__main__': single_locale = DesktopSingleLocale()