Bug 1216152 - l10n jobs should attach artifacts to their BBB tasks. r=jlund

This commit is contained in:
Rail Aliiev 2015-12-29 08:17:15 -05:00
parent 39fa802c76
commit c4cc1530b6
4 changed files with 63 additions and 23 deletions

View File

@ -68,13 +68,17 @@ class Taskcluster(LogMixin):
def claim_task(self, task):
self.taskcluster_queue.claimTask(
task['status']['taskId'],
task['status']['runs'][0]['runId'],
task['status']['runs'][-1]['runId'],
{
"workerGroup": self.buildbot,
"workerId": self.buildbot,
})
def create_artifact(self, task, filename):
def get_task(self, task_id):
return self.taskcluster_queue.status(task_id)
@staticmethod
def get_mime_type(ext, default='application/octet-stream'):
mime_types = {
".asc": "text/plain",
".checksums": "text/plain",
@ -85,30 +89,50 @@ class Taskcluster(LogMixin):
".xpi": "application/x-xpinstall",
".zip": "application/zip",
}
mime_type = mime_types.get(os.path.splitext(filename)[1], 'application/octet-stream')
return mime_types.get(ext, default)
@property
def expiration(self):
return datetime.utcnow() + timedelta(weeks=52)
def create_artifact(self, task, filename):
mime_type = self.get_mime_type(os.path.splitext(filename)[1])
content_length = os.path.getsize(filename)
self.info("Uploading to S3: filename=%s mimetype=%s length=%s" % (filename, mime_type, content_length))
expiration = datetime.utcnow() + timedelta(weeks=52)
self.info("Uploading to S3: filename=%s mimetype=%s length=%s" % (
filename, mime_type, content_length))
artifact = self.taskcluster_queue.createArtifact(
task['status']['taskId'],
task['status']['runs'][0]['runId'],
task['status']['runs'][-1]['runId'],
'public/build/%s' % os.path.basename(filename),
{
"storageType": "s3",
"expires": expiration,
"expires": self.expiration,
"contentType": mime_type,
})
self.put_file(filename, artifact['putUrl'], mime_type)
return self.get_taskcluster_url(filename)
def create_reference_artifact(self, task, filename, url):
mime_type = self.get_mime_type(os.path.splitext(filename)[1])
self.info("Create reference artifact: filename=%s mimetype=%s url=%s" %
(filename, mime_type, url))
self.taskcluster_queue.createArtifact(
task['status']['taskId'],
task['status']['runs'][-1]['runId'],
'public/build/%s' % os.path.basename(filename),
{
"storageType": "reference",
"expires": self.expiration,
"contentType": mime_type,
"url": url,
})
def report_completed(self, task):
self.taskcluster_queue.reportCompleted(
task['status']['taskId'],
task['status']['runs'][0]['runId'],
{
"success": True,
})
task_id = task['status']['taskId']
run_id = task['status']['runs'][-1]['runId']
self.info("Resolving %s, run %s. Full task:" % (task_id, run_id))
self.info(str(task))
self.taskcluster_queue.reportCompleted(task_id, run_id)
def get_taskcluster_url(self, filename):
return 'https://queue.taskcluster.net/v1/task/%s/artifacts/public/build/%s' % (

View File

@ -13,8 +13,6 @@ import re
import sys
import time
import shlex
import logging
import subprocess
# load modules from parent dir
@ -36,7 +34,6 @@ from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.mozilla.taskcluster_helper import Taskcluster
from mozharness.base.python import VirtualenvMixin
from mozharness.mozilla.mock import ERROR_MSGS
from mozharness.base.log import FATAL
try:
import simplejson as json
@ -184,7 +181,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
'virtualenv_modules': [
'requests==2.8.1',
'PyHawk-with-a-single-extra-commit==0.1.5',
'taskcluster==0.0.15',
'taskcluster==0.0.26',
],
'virtualenv_path': 'venv',
},
@ -960,7 +957,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
self.activate_virtualenv()
branch = self.config['branch']
platform = self.config['platform']
revision = self._query_revision()
repo = self.query_l10n_repo()
if not repo:
@ -974,6 +970,19 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
contents = json.load(f)
templates = contents['l10n']
# Release promotion creates a special task to accumulate all artifacts
# under the same task
artifacts_task = None
self.read_buildbot_config()
if "artifactsTaskId" in self.buildbot_config.get("properties", {}):
artifacts_task_id = self.buildbot_config["properties"]["artifactsTaskId"]
artifacts_tc = Taskcluster(
branch=branch, rank=pushinfo.pushdate, client_id=client_id,
access_token=access_token, log_obj=self.log_obj,
task_id=artifacts_task_id)
artifacts_task = artifacts_tc.get_task(artifacts_task_id)
artifacts_tc.claim_task(artifacts_task)
for locale, files in self.upload_files.iteritems():
self.info("Uploading files to S3 for locale '%s': %s" % (locale, files))
routes = []
@ -1009,9 +1018,16 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
# check the uploaded file against the property conditions so that we
# can set the buildbot config with the correct URLs for package
# locations.
tc.create_artifact(task, upload_file)
artifact_url = tc.create_artifact(task, upload_file)
if artifacts_task:
artifacts_tc.create_reference_artifact(
artifacts_task, upload_file, artifact_url)
tc.report_completed(task)
if artifacts_task:
artifacts_tc.report_completed(artifacts_task)
# main {{{
if __name__ == '__main__':
single_locale = DesktopSingleLocale()

View File

@ -92,7 +92,7 @@ class FxDesktopBuild(BuildScript, object):
'virtualenv_modules': [
'requests==2.8.1',
'PyHawk-with-a-single-extra-commit==0.1.5',
'taskcluster==0.0.15',
'taskcluster==0.0.26',
],
'virtualenv_path': 'venv',
#

View File

@ -127,7 +127,7 @@ class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
'virtualenv_modules': [
'requests==2.8.1',
'PyHawk-with-a-single-extra-commit==0.1.5',
'taskcluster==0.0.15',
'taskcluster==0.0.26',
],
'virtualenv_path': 'venv',
},