From eb0a882729e79d0d69c35b34812b53c7e7498d1c Mon Sep 17 00:00:00 2001 From: Mike Shal Date: Wed, 19 Aug 2015 15:28:43 -0400 Subject: [PATCH] Bug 1195865 - Upload android l10n repacks to Taskcluster; r=jlund --- .../mozharness/mozilla/l10n/locales.py | 9 ++ testing/mozharness/scripts/desktop_l10n.py | 9 +- testing/mozharness/scripts/mobile_l10n.py | 95 +++++++++++++++++-- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py index 39051600ca5..8411faaa8e0 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -189,6 +189,15 @@ class LocalesMixin(ChunkingMixin): tag_override=c.get('tag_override')) self.gecko_locale_revisions = revs + def query_repo(self): + # Find the name of our repository + mozilla_dir = self.config['mozilla_dir'] + repo = None + for repository in self.config['repos']: + if repository.get('dest') == mozilla_dir: + repo = repository['repo'] + break + return repo # GaiaLocalesMixin {{{1 class GaiaLocalesMixin(object): diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 022700af71f..39f10b0cc50 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -161,6 +161,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, "list-locales", "setup", "repack", + "create-virtualenv", "taskcluster-upload", "funsize-props", "submit-to-balrog", @@ -1011,14 +1012,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, self.warning('Skipping S3 file upload: No taskcluster credentials.') return - # We need to activate the virtualenv so that we can import taskcluster - # (and its dependent modules, like requests and hawk). Normally we - # could create the virtualenv as an action, but due to some odd - # dependencies with query_build_env() being called from build(), which - # is necessary before the virtualenv can be created. - self.disable_mock() - self.create_virtualenv() - self.enable_mock() self.activate_virtualenv() # Enable Taskcluster debug logging, so at least we get some debug diff --git a/testing/mozharness/scripts/mobile_l10n.py b/testing/mozharness/scripts/mobile_l10n.py index ae21b121f7e..8f568b62f4a 100755 --- a/testing/mozharness/scripts/mobile_l10n.py +++ b/testing/mozharness/scripts/mobile_l10n.py @@ -15,6 +15,7 @@ import os import re import subprocess import sys +import shlex try: import simplejson as json @@ -37,12 +38,15 @@ from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.l10n.locales import LocalesMixin from mozharness.mozilla.mock import MockMixin from mozharness.mozilla.updates.balrog import BalrogMixin +from mozharness.base.python import VirtualenvMixin +from mozharness.mozilla.taskcluster_helper import Taskcluster # MobileSingleLocale {{{1 class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, MobileSigningMixin, TransferMixin, TooltoolMixin, - BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin): + BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin, + VirtualenvMixin): config_options = [[ ['--locale', ], {"action": "extend", @@ -104,21 +108,35 @@ class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, ]] def __init__(self, require_config_file=True): - LocalesMixin.__init__(self) - MercurialScript.__init__( - self, - config_options=self.config_options, - all_actions=[ + buildscript_kwargs = { + 'all_actions': [ "clobber", "pull", "list-locales", "setup", "repack", "upload-repacks", + "create-virtualenv", + "taskcluster-upload", "submit-to-balrog", "summary", ], - require_config_file=require_config_file + 'config': { + 'taskcluster_credentials_file': 'oauth.txt', + 'virtualenv_modules': [ + 'requests==2.2.1', + 'PyHawk-with-a-single-extra-commit==0.1.5', + 'taskcluster==0.0.15', + ], + 'virtualenv_path': 'venv', + }, + } + LocalesMixin.__init__(self) + MercurialScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + **buildscript_kwargs ) self.base_package_name = None self.buildid = None @@ -447,6 +465,69 @@ class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, self.summarize_success_count(success_count, total_count, message="Repacked %d of %d binaries successfully.") + def taskcluster_upload(self): + auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) + credentials = {} + execfile(auth, credentials) + client_id = credentials.get('taskcluster_clientId') + access_token = credentials.get('taskcluster_accessToken') + if not client_id or not access_token: + self.warning('Skipping S3 file upload: No taskcluster credentials.') + return + + self.activate_virtualenv() + + dirs = self.query_abs_dirs() + locales = self.query_locales() + make = self.query_exe("make") + upload_env = self.query_upload_env() + cwd = dirs['abs_locales_dir'] + branch = self.config['branch'] + revision = self.query_revision() + repo = self.query_repo() + pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool') + routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], + 'testing/taskcluster/routes.json') + with open(routes_json) as f: + contents = json.load(f) + templates = contents['l10n'] + + for locale in locales: + output = self.get_output_from_command_m( + "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale), + cwd=cwd, + env=upload_env, + ) + files = shlex.split(output) + abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files] + + routes = [] + fmt = { + 'index': self.config.get('taskcluster_index', 'index.garbage.staging'), + 'project': branch, + 'head_rev': revision, + 'build_product': self.config['stage_product'], + 'build_name': self.query_build_name(), + 'build_type': self.query_build_type(), + 'locale': locale, + } + for template in templates: + routes.append(template.format(**fmt)) + + self.info('Using routes: %s' % routes) + tc = Taskcluster(branch, + pushinfo.pushdate, # Use pushdate as the rank + client_id, + access_token, + self.log_obj, + ) + task = tc.create_task(routes) + tc.claim_task(task) + + for upload_file in abs_files: + tc.create_artifact(task, upload_file) + tc.report_completed(task) + def upload_repacks(self): c = self.config dirs = self.query_abs_dirs()