From f1e7454cabdf0831873e8770d2ec80cff2cac182 Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Tue, 28 Jul 2015 17:06:19 +0200 Subject: [PATCH] Backed out changeset e3c15b9f89b3 (bug 1185643) for possibly breaking mulet tests on a CLOSED TREE --- testing/docker/desktop-build/Dockerfile | 3 +++ .../desktop-build/bin/checkout-sources.sh | 18 +++++++++++++++++- .../desktop-build}/configs/disable_signing.py | 0 .../platform_supports_post_upload_to_latest.py | 0 .../desktop32-build/bin/checkout-sources.sh | 18 +++++++++++++++++- testing/taskcluster/mach_commands.py | 9 +++++++++ .../taskcluster/scripts/builder/build-linux.sh | 2 +- testing/taskcluster/tasks/build.yml | 3 +++ 8 files changed, 50 insertions(+), 3 deletions(-) rename testing/{mozharness => docker/desktop-build}/configs/disable_signing.py (100%) rename testing/{mozharness => docker/desktop-build}/configs/platform_supports_post_upload_to_latest.py (100%) diff --git a/testing/docker/desktop-build/Dockerfile b/testing/docker/desktop-build/Dockerfile index dd31d2ad16f..c702bb9d3fe 100644 --- a/testing/docker/desktop-build/Dockerfile +++ b/testing/docker/desktop-build/Dockerfile @@ -6,6 +6,9 @@ MAINTAINER Morgan Reece Phillips ADD bin /home/worker/bin RUN chmod +x /home/worker/bin/* +# Add custom mozharness configs +ADD configs /home/worker/configs + ## # COMPILER HACKS ## diff --git a/testing/docker/desktop-build/bin/checkout-sources.sh b/testing/docker/desktop-build/bin/checkout-sources.sh index 93ebd2ce39c..caedd192789 100644 --- a/testing/docker/desktop-build/bin/checkout-sources.sh +++ b/testing/docker/desktop-build/bin/checkout-sources.sh @@ -10,7 +10,7 @@ set -x -e # the canonical repo to clone and *_HEAD_REPO as the repo containing the # desired revision. For Mercurial clones, only *_HEAD_REV is required; for Git # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1 -# as *_HEAD_REV. +# as *_HEAD_REV. For compatibility, we also accept MOZHARNESS_{REV,REF} : GECKO_REPOSITORY ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central} : GECKO_BASE_REPOSITORY ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}} @@ -18,6 +18,15 @@ set -x -e : GECKO_HEAD_REV ${GECKO_HEAD_REV:=default} : GECKO_HEAD_REF ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}} +: MOZHARNESS_REPOSITORY ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness} +: MOZHARNESS_BASE_REPOSITORY ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}} +: MOZHARNESS_HEAD_REPOSITORY ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}} +: MOZHARNESS_REV ${MOZHARNESS_REV:=production} +: MOZHARNESS_REF ${MOZHARNESS_REF:=${MOZHARNESS_REV}} +: MOZHARNESS_HEAD_REV ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}} +: MOZHARNESS_HEAD_REF ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}} +: MOZHARNESS_DISABLE ${MOZHARNESS_DISABLE:=false} + : TOOLS_REPOSITORY ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools} : TOOLS_BASE_REPOSITORY ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}} : TOOLS_HEAD_REPOSITORY ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}} @@ -33,6 +42,12 @@ set -x -e set -v +# check out mozharness +if [ ! "$MOZHARNESS_DISABLE" = "true" ] +then + tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF +fi + # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools) if [ ! "$TOOLS_DISABLE" = true ] then @@ -44,4 +59,5 @@ then fi fi +# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared) tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF diff --git a/testing/mozharness/configs/disable_signing.py b/testing/docker/desktop-build/configs/disable_signing.py similarity index 100% rename from testing/mozharness/configs/disable_signing.py rename to testing/docker/desktop-build/configs/disable_signing.py diff --git a/testing/mozharness/configs/platform_supports_post_upload_to_latest.py b/testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py similarity index 100% rename from testing/mozharness/configs/platform_supports_post_upload_to_latest.py rename to testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py diff --git a/testing/docker/desktop32-build/bin/checkout-sources.sh b/testing/docker/desktop32-build/bin/checkout-sources.sh index 082729b3709..6cc2acfb759 100644 --- a/testing/docker/desktop32-build/bin/checkout-sources.sh +++ b/testing/docker/desktop32-build/bin/checkout-sources.sh @@ -10,7 +10,7 @@ set -x -e # the canonical repo to clone and *_HEAD_REPO as the repo containing the # desired revision. For Mercurial clones, only *_HEAD_REV is required; for Git # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1 -# as *_HEAD_REV. +# as *_HEAD_REV. For compatibility, we also accept MOZHARNESS_{REV,REF} : GECKO_REPOSITORY ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central} : GECKO_BASE_REPOSITORY ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}} @@ -18,16 +18,31 @@ set -x -e : GECKO_HEAD_REV ${GECKO_HEAD_REV:=default} : GECKO_HEAD_REF ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}} +: MOZHARNESS_REPOSITORY ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness} +: MOZHARNESS_BASE_REPOSITORY ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}} +: MOZHARNESS_HEAD_REPOSITORY ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}} +: MOZHARNESS_REV ${MOZHARNESS_REV:=production} +: MOZHARNESS_REF ${MOZHARNESS_REF:=${MOZHARNESS_REV}} +: MOZHARNESS_HEAD_REV ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}} +: MOZHARNESS_HEAD_REF ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}} + : TOOLS_REPOSITORY ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools} : TOOLS_BASE_REPOSITORY ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}} : TOOLS_HEAD_REPOSITORY ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}} : TOOLS_HEAD_REV ${TOOLS_HEAD_REV:=default} : TOOLS_HEAD_REF ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}} +: MH_CUSTOM_BUILD_VARIANT_CFG ${MH_CUSTOM_BUILD_VARIANT_CFG} +: MH_BRANCH ${MH_BRANCH:=mozilla-central} +: MH_BUILD_POOL ${MH_BUILD_POOL:=staging} + : WORKSPACE ${WORKSPACE:=/home/worker/workspace} set -v +# check out mozharness +tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF + # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools) tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF if [ ! -d build ]; then @@ -35,4 +50,5 @@ if [ ! -d build ]; then ln -s $WORKSPACE/build/tools build/tools fi +# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared) tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF diff --git a/testing/taskcluster/mach_commands.py b/testing/taskcluster/mach_commands.py index db2e84236c9..c4b4c3ba340 100644 --- a/testing/taskcluster/mach_commands.py +++ b/testing/taskcluster/mach_commands.py @@ -21,6 +21,7 @@ from mach.decorators import ( ROOT = os.path.dirname(os.path.realpath(__file__)) GECKO = os.path.realpath(os.path.join(ROOT, '..', '..')) DOCKER_ROOT = os.path.join(ROOT, '..', 'docker') +MOZHARNESS_CONFIG = os.path.join(GECKO, 'testing', 'mozharness', 'mozharness.json') # XXX: If/when we have the taskcluster queue use construct url instead ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' @@ -39,6 +40,10 @@ DEFAULT_JOB_PATH = os.path.join( ROOT, 'tasks', 'branches', 'base_jobs.yml' ) +def load_mozharness_info(): + with open(MOZHARNESS_CONFIG) as content: + return json.load(content) + def docker_image(name): ''' Determine the docker tag/revision from an in tree docker file ''' repository_path = os.path.join(DOCKER_ROOT, name, 'REGISTRY') @@ -258,6 +263,7 @@ class Graph(object): jobs = templates.load(job_path, {}) job_graph = parse_commit(message, jobs) + mozharness = load_mozharness_info() # Template parameters used when expanding the graph parameters = dict(gaia_info().items() + { @@ -272,6 +278,9 @@ class Graph(object): 'owner': params['owner'], 'from_now': json_time_from_now, 'now': current_json_time(), + 'mozharness_repository': mozharness['repo'], + 'mozharness_rev': mozharness['revision'], + 'mozharness_ref':mozharness.get('reference', mozharness['revision']), 'revision_hash': params['revision_hash'] }.items()) diff --git a/testing/taskcluster/scripts/builder/build-linux.sh b/testing/taskcluster/scripts/builder/build-linux.sh index 6ffe4bb8594..28596a2ef6e 100644 --- a/testing/taskcluster/scripts/builder/build-linux.sh +++ b/testing/taskcluster/scripts/builder/build-linux.sh @@ -118,7 +118,7 @@ done # Mozharness would ordinarily do the checkouts itself, but they are disabled # here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above. -$WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \ +./${MOZHARNESS_SCRIPT} ${config_cmds} \ $debug_flag \ $custom_build_variant_cfg_flag \ --disable-mock \ diff --git a/testing/taskcluster/tasks/build.yml b/testing/taskcluster/tasks/build.yml index 7251cb1f24b..79a9764249d 100644 --- a/testing/taskcluster/tasks/build.yml +++ b/testing/taskcluster/tasks/build.yml @@ -52,6 +52,9 @@ task: GECKO_HEAD_REPOSITORY: '{{head_repository}}' GECKO_HEAD_REV: '{{head_rev}}' GECKO_HEAD_REF: '{{head_ref}}' + MOZHARNESS_REPOSITORY: '{{mozharness_repository}}' + MOZHARNESS_REV: '{{mozharness_rev}}' + MOZHARNESS_REF: '{{mozharness_ref}}' TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool' TOOLTOOL_REV: 'master'