Bug 1216152 - l10n jobs should attach artifacts to their BBB tasks. r=jlund

testing/mozharness/mozharness/mozilla/taskcluster_helper.py

@@ -68,13 +68,17 @@ class Taskcluster(LogMixin):
     def claim_task(self, task):
         self.taskcluster_queue.claimTask(
             task['status']['taskId'],
-            task['status']['runs'][0]['runId'],
+            task['status']['runs'][-1]['runId'],
             {
                 "workerGroup": self.buildbot,
                 "workerId": self.buildbot,
             })
 
-    def create_artifact(self, task, filename):
+    def get_task(self, task_id):
+        return self.taskcluster_queue.status(task_id)
+
+    @staticmethod
+    def get_mime_type(ext, default='application/octet-stream'):
         mime_types = {
             ".asc": "text/plain",
             ".checksums": "text/plain",
@@ -85,30 +89,50 @@ class Taskcluster(LogMixin):
             ".xpi": "application/x-xpinstall",
             ".zip": "application/zip",
         }
-        mime_type = mime_types.get(os.path.splitext(filename)[1], 'application/octet-stream')
+        return mime_types.get(ext, default)
+
+    @property
+    def expiration(self):
+        return datetime.utcnow() + timedelta(weeks=52)
+
+    def create_artifact(self, task, filename):
+        mime_type = self.get_mime_type(os.path.splitext(filename)[1])
         content_length = os.path.getsize(filename)
-
+        self.info("Uploading to S3: filename=%s mimetype=%s length=%s" % (
-        self.info("Uploading to S3: filename=%s mimetype=%s length=%s" % (filename, mime_type, content_length))
+            filename, mime_type, content_length))
-
-        expiration = datetime.utcnow() + timedelta(weeks=52)
         artifact = self.taskcluster_queue.createArtifact(
             task['status']['taskId'],
-            task['status']['runs'][0]['runId'],
+            task['status']['runs'][-1]['runId'],
             'public/build/%s' % os.path.basename(filename),
             {
                 "storageType": "s3",
-                "expires": expiration,
+                "expires": self.expiration,
                 "contentType": mime_type,
             })
         self.put_file(filename, artifact['putUrl'], mime_type)
+        return self.get_taskcluster_url(filename)
+
+    def create_reference_artifact(self, task, filename, url):
+        mime_type = self.get_mime_type(os.path.splitext(filename)[1])
+        self.info("Create reference artifact: filename=%s mimetype=%s url=%s" %
+                  (filename, mime_type, url))
+        self.taskcluster_queue.createArtifact(
+            task['status']['taskId'],
+            task['status']['runs'][-1]['runId'],
+            'public/build/%s' % os.path.basename(filename),
+            {
+                "storageType": "reference",
+                "expires": self.expiration,
+                "contentType": mime_type,
+                "url": url,
+            })
 
     def report_completed(self, task):
-        self.taskcluster_queue.reportCompleted(
+        task_id = task['status']['taskId']
-            task['status']['taskId'],
+        run_id = task['status']['runs'][-1]['runId']
-            task['status']['runs'][0]['runId'],
+        self.info("Resolving %s, run %s. Full task:" % (task_id, run_id))
-            {
+        self.info(str(task))
-                "success": True,
+        self.taskcluster_queue.reportCompleted(task_id, run_id)
-            })
 
     def get_taskcluster_url(self, filename):
         return 'https://queue.taskcluster.net/v1/task/%s/artifacts/public/build/%s' % (

testing/mozharness/scripts/desktop_l10n.py

@@ -13,8 +13,6 @@ import re
 import sys
 import time
 import shlex
-import logging
-
 import subprocess
 
 # load modules from parent dir
@@ -36,7 +34,6 @@ from mozharness.mozilla.updates.balrog import BalrogMixin
 from mozharness.mozilla.taskcluster_helper import Taskcluster
 from mozharness.base.python import VirtualenvMixin
 from mozharness.mozilla.mock import ERROR_MSGS
-from mozharness.base.log import FATAL
 
 try:
     import simplejson as json
@@ -184,7 +181,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
                 'virtualenv_modules': [
                     'requests==2.8.1',
                     'PyHawk-with-a-single-extra-commit==0.1.5',
-                    'taskcluster==0.0.15',
+                    'taskcluster==0.0.26',
                 ],
                 'virtualenv_path': 'venv',
             },
@@ -960,7 +957,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
         self.activate_virtualenv()
 
         branch = self.config['branch']
-        platform = self.config['platform']
         revision = self._query_revision()
         repo = self.query_l10n_repo()
         if not repo:
@@ -974,6 +970,19 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
             contents = json.load(f)
             templates = contents['l10n']
 
+        # Release promotion creates a special task to accumulate all artifacts
+        # under the same task
+        artifacts_task = None
+        self.read_buildbot_config()
+        if "artifactsTaskId" in self.buildbot_config.get("properties", {}):
+            artifacts_task_id = self.buildbot_config["properties"]["artifactsTaskId"]
+            artifacts_tc = Taskcluster(
+                    branch=branch, rank=pushinfo.pushdate, client_id=client_id,
+                    access_token=access_token, log_obj=self.log_obj,
+                    task_id=artifacts_task_id)
+            artifacts_task = artifacts_tc.get_task(artifacts_task_id)
+            artifacts_tc.claim_task(artifacts_task)
+
         for locale, files in self.upload_files.iteritems():
             self.info("Uploading files to S3 for locale '%s': %s" % (locale, files))
             routes = []
@@ -1009,9 +1018,16 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
                 # check the uploaded file against the property conditions so that we
                 # can set the buildbot config with the correct URLs for package
                 # locations.
-                tc.create_artifact(task, upload_file)
+                artifact_url = tc.create_artifact(task, upload_file)
+                if artifacts_task:
+                    artifacts_tc.create_reference_artifact(
+                            artifacts_task, upload_file, artifact_url)
+
             tc.report_completed(task)
 
+        if artifacts_task:
+            artifacts_tc.report_completed(artifacts_task)
+
 # main {{{
 if __name__ == '__main__':
     single_locale = DesktopSingleLocale()

testing/mozharness/scripts/fx_desktop_build.py

@@ -92,7 +92,7 @@ class FxDesktopBuild(BuildScript, object):
                 'virtualenv_modules': [
                     'requests==2.8.1',
                     'PyHawk-with-a-single-extra-commit==0.1.5',
-                    'taskcluster==0.0.15',
+                    'taskcluster==0.0.26',
                 ],
                 'virtualenv_path': 'venv',
                 #

testing/mozharness/scripts/mobile_l10n.py

@@ -127,7 +127,7 @@ class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
                 'virtualenv_modules': [
                     'requests==2.8.1',
                     'PyHawk-with-a-single-extra-commit==0.1.5',
-                    'taskcluster==0.0.15',
+                    'taskcluster==0.0.26',
                 ],
                 'virtualenv_path': 'venv',
             },