Backout "Bug 1144808 part 2: Harden phone-builder image."

Things just got too hard for B2G people to change build scripts, so we
are backing out this.

testing/docker/phone-builder/Dockerfile

@@ -13,5 +13,5 @@ RUN           pip install awscli
 RUN           npm install -g bower gulp apm grunt-cli
 
 # Set a default command useful for debugging
-ENTRYPOINT ["bootstrap.py"]
+ENTRYPOINT ["validate_task.py"]
 

testing/docker/phone-builder/VERSION

@@ -1 +1 @@
-0.1.5
+0.0.22

testing/docker/phone-builder/bin/bootstrap.py

@@ -1,131 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-import os
-import json
-import urllib2
-import sys
-import re
-import subprocess
-import shutil
-
-HOME = os.getenv('HOME')
-
-WORKSPACE = os.path.join(HOME, 'workspace')
-ARTIFACTS_PUBLIC = os.path.normpath(os.path.join(HOME, 'artifacts-public'))
-SCRIPTS_PATH = os.path.join(HOME, 'bin')
-COMMANDS = {
-    'phone': 'build-phone.sh',
-    'phone-ota': 'build-phone-ota.sh',
-    'dolphin': 'build-dolphin.sh'
-}
-
-# Be careful when adding username containing '.', as it expands to any character
-# in regular expressions. It must be escaped properly:
-#  right: r'foo\.bar',
-#  wrong: 'foo.bar' # matches fooabar, foo1bar, foo_bar, etc
-
-github_allowed_accounts = [
-    'walac',
-    'nhirata',
-    'selenamarie',
-    'ShakoHo',
-]
-
-bitbucket_allowed_accounts = [
-    'walac',
-    'selenamarie',
-]
-
-def build_repo_matcher():
-    github_expr = r'(github\.com/(' + '|'.join(github_allowed_accounts) + ')/)'
-    bitbucket_expr = r'(bitbucket\.org/(' + '|'.join(bitbucket_allowed_accounts) + ')/)'
-    mozilla_expr = r'((hg|git)\.mozilla\.org)'
-    expr = r'^https?://(' + '|'.join((github_expr, bitbucket_expr, mozilla_expr)) + ')'
-    return re.compile(expr)
-
-repo_matcher = build_repo_matcher()
-
-def get_task(taskid):
-    return json.load(urllib2.urlopen('https://queue.taskcluster.net/v1/task/' + taskid))
-
-def check_repo(repo):
-    if not repo_matcher.match(repo):
-        print('Invalid repository "{}"'.format(repo), file=sys.stderr)
-        return -1
-    return 0
-
-# Cleanup artifacts and known credentials. This is to avoid a malicious
-# task to map a directory containing sensible files expose secret files.
-def cleanup(task):
-    payload = task['payload']
-    for key, value in payload.get('artifacts', {}).items():
-        shutil.rmtree(value['path'], ignore_errors=True)
-
-    shutil.rmtree(os.path.join(HOME, '.aws'), ignore_errors=True)
-
-    try:
-        os.remove(os.path.join(HOME, 'socorro.token'))
-    except (IOError, OSError):
-        pass
-
-def check_task(task):
-    repositories_to_check = [
-        'GECKO_HEAD_REPOSITORY',
-        'GECKO_BASE_REPOSITORY',
-    ]
-
-    payload = task['payload']
-
-    for repo in repositories_to_check:
-        if repo not in payload['env']:
-            print('Repository {} is not in payload.env.'.format(repo), file=sys.stderr)
-            return -1
-        ret = check_repo(payload['env'][repo])
-        if ret != 0:
-            return ret
-
-    for key, value in payload.get('artifacts', {}).items():
-        if key.startswith('public') and \
-                os.path.normpath(value['path']) != ARTIFACTS_PUBLIC:
-            print('{} cannot be a public artifact.'.format(value['path']),
-                  file=sys.stderr)
-            return -1
-
-    if sys.argv[1] not in COMMANDS:
-        print("Invalid build command '{}', valid commands are '{}'".format(sys.argv[1], ", ".join(COMMANDS.keys())))
-        return -1
-
-    return 0
-
-def run():
-    command = COMMANDS[sys.argv[1]]
-
-    checkout_gecko = ['checkout-gecko', WORKSPACE]
-    cd_scripts = ['cd', SCRIPTS_PATH]
-    build = ['buildbot_step', '"Build"', os.path.join(SCRIPTS_PATH, command), WORKSPACE]
-    and_ = ['&&']
-    command = ' '.join(checkout_gecko + and_ + cd_scripts + and_ + build)
-
-    try:
-        return subprocess.call(command, shell=True)
-    except subprocess.CalledProcessError as e:
-        return e.returncode
-
-def main():
-    taskid = os.getenv('TASK_ID')
-
-    # If the task id is None, we assume we are running docker locally
-    if taskid is not None:
-        task = get_task(taskid)
-        ret = check_task(task)
-        if ret != 0:
-            cleanup(task)
-            return ret
-
-    if len(sys.argv) > 1:
-        return run()
-
-
-if __name__ == '__main__':
-    sys.exit(main())

testing/docker/phone-builder/bin/validate_task.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import os
+import os.path
+import json
+import urllib2
+import sys
+import re
+import subprocess
+
+repo_matcher = re.compile(r'[a-z]+://(hg|git)\.mozilla\.org')
+
+def get_task(taskid):
+    return json.load(urllib2.urlopen('https://queue.taskcluster.net/v1/task/' + taskid))
+
+def check_task(task):
+    payload = task['payload']
+
+    if 'GECKO_HEAD_REPOSITORY' not in payload['env']:
+        print('Task has no head gecko repository', file=sys.stderr)
+        return -1
+
+    repo = payload['env']['GECKO_HEAD_REPOSITORY']
+    # if it is not a mozilla repository, fail
+    if not repo_matcher.match(repo):
+        print('Invalid head repository', repo, file=sys.stderr)
+        return -1
+
+    if 'GECKO_BASE_REPOSITORY' not in payload['env']:
+        print('Task has no base gecko repository', file=sys.stderr)
+        return -1
+
+    repo = payload['env']['GECKO_BASE_REPOSITORY']
+    if not repo_matcher.match(repo):
+        print('Invalid base repository', repo, file=sys.stderr)
+        return -1
+
+    locations = task["extra"]["locations"]
+    if "img" in locations:
+        img = locations["img"]
+        if img.startswith("public"):
+            print('Cannot upload images to public', file=sys.stderr)
+            return -1
+
+    return 0
+
+def main():
+    taskid = os.getenv('TASK_ID')
+
+    # If the task id is None, we assume we are running docker locally
+    if taskid is not None:
+        task = get_task(taskid)
+        ret = check_task(task)
+        if ret != 0:
+            return ret
+
+    if len(sys.argv) > 1:
+        try:
+            return subprocess.call(sys.argv[1:], shell=True)
+        except subprocess.CalledProcessError as e:
+            return e.returncode
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

testing/docker/phone-builder/tests/invalid_base_repo.yml

@@ -23,13 +23,13 @@ task:
       # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js
       REVISION: 'tip'
       GECKO_HEAD_REPOSITORY: 'http://hg.mozilla.org/mozilla-central'
-      GECKO_BASE_REPOSITORY: 'http://github.com/mozilla/gecko-dev'
+      GECKO_BASE_REPOSITORY: 'git@github.com:mozilla/gecko-dev.git'
 
     image: 'quay.io/mozilla/phone-builder:0.0.1'
     maxRunTime: 14400
 
     command:
-      - phone
+      - build-phone.sh
 
     artifacts:
       'private/build':

testing/docker/phone-builder/tests/invalid_command.yml

@@ -1,52 +0,0 @@
-taskId: 1
-task:
-  metadata:
-    source: http://todo.com/soon
-    owner: user@domain.com
-    name: B2G flame-kk opt
-    description: B2G flame-kk opt
-
-  workerType: b2gbuild
-  provisionerId: aws-provisioner
-
-  scopes:
-    - 'docker-worker:cache:build-phone-objects'
-    - 'docker-worker:image:{{#docker_image}}phone-builder{{/docker_image}}'
-
-  payload:
-    cache:
-      build-phone-objects: '/home/worker/object-folder-flame-kk-1'
-
-    env:
-      TARGET: 'flame-kk'
-      B2G_DEBUG: '1'
-      # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js
-      REVISION: 'tip'
-      GECKO_HEAD_REPOSITORY: 'http://hg.mozilla.org/mozilla-central'
-      GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
-
-    image: '{{#docker_image}}phone-builder{{/docker_image}}'
-    maxRunTime: 14400
-
-    command:
-      - build-phone.sh
-
-    artifacts:
-      'private/build':
-        type: directory
-        path: '/home/worker/artifacts/'
-        expires: '{{#from_now}}1 year{{/from_now}}'
-
-  extra:
-    # Rather then enforcing particular conventions we require that all build
-    # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
-    locations:
-      build: 'private/build/b2g-android-arm.tar.gz'
-      img: 'private/build/flame-kk.zip'
-      tests: 'private/build/gaia.zip'
-      symbols: 'private/build/b2g-crashreporter-symbols.zip'
-      sources: 'private/build/sources.xml'
-
-    treeherder:
-      symbol: B

testing/docker/phone-builder/tests/invalid_head_repo.yml

@@ -22,14 +22,14 @@ task:
       B2G_DEBUG: '1'
       # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js
       REVISION: 'tip'
-      GECKO_HEAD_REPOSITORY: 'http://bitbucket.org/mozilla/gecko-dev'
+      GECKO_HEAD_REPOSITORY: 'git@github.com:mozilla/gecko-dev.git'
       GECKO_BASE_REPOSITORY: 'http://hg.mozilla.org/mozilla-central'
 
     image: 'quay.io/mozilla/phone-builder:0.0.1'
     maxRunTime: 14400
 
     command:
-      - phone
+      - build-phone.sh
 
     artifacts:
       'private/build':

testing/docker/phone-builder/tests/public.yml

@@ -29,7 +29,7 @@ task:
     maxRunTime: 14400
 
     command:
-      - phone
+      - build-phone.sh
 
     artifacts:
       'public/build':
@@ -41,11 +41,10 @@ task:
     # tasks provide the "build" extra field to specify where the build and tests
     # files are located.
     locations:
-      build: 'private/build/emulator.tar.gz'
-      tests: 'private/build/b2g-tests.zip'
-      symbols: 'private/build/b2g-crashreporter-symbols.zip'
-      sources: 'private/build/sources.xml'
-      img: 'private/build/image.zip'
+      build: 'public/build/emulator.tar.gz'
+      tests: 'public/build/b2g-tests.zip'
+      symbols: 'public/build/b2g-crashreporter-symbols.zip'
+      sources: 'public/build/sources.xml'
 
     treeherder:
       symbol: B

testing/docker/phone-builder/tests/test_validation.py

@@ -4,31 +4,28 @@ import unittest
 import sys
 import yaml
 sys.path.append('../bin')
-from bootstrap import check_task
-import glob
+from validate_task import check_task
 
 def load_task(task_file):
     content = open(task_file, 'r')
-    task = yaml.load(content)['task']
-    sys.argv[1:] = task['payload']['command']
-    return task
+    return yaml.load(content)['task']
 
 class TaskValidationTest(unittest.TestCase):
-    def __init__(self, methodName='runTest'):
-        super(TaskValidationTest, self).__init__(methodName)
-        sys.argv.append('')
+    def test_valid_task(self):
+        task = load_task('valid.yml')
+        self.assertEquals(check_task(task), 0)
 
-    def test_valid_tasks(self):
-        valid_tasks = glob.glob('valid*.yml')
-        for t in valid_tasks:
-            task = load_task(t)
-            self.assertEqual(check_task(task), 0)
+    def test_invalid_base_repo(self):
+        task = load_task('invalid_base_repo.yml')
+        self.assertEquals(check_task(task), -1)
 
-    def test_invalid_tasks(self):
-        invalid_tasks = glob.glob('invalid*.yml')
-        for t in invalid_tasks:
-            task = load_task(t)
-            self.assertNotEquals(check_task(task), 0)
+    def test_invalid_head_repo(self):
+        task = load_task('invalid_head_repo.yml')
+        self.assertEquals(check_task(task), -1)
+
+    def test_public_artifact(self):
+        task = load_task('public.yml')
+        self.assertEquals(check_task(task), -1)
 
 if __name__ == '__main__':
     unittest.main()

testing/docker/phone-builder/tests/valid.yml

@@ -23,13 +23,13 @@ task:
       # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js
       REVISION: 'tip'
       GECKO_HEAD_REPOSITORY: 'http://hg.mozilla.org/mozilla-central'
-      GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
+      GECKO_BASE_REPOSITORY: 'http://hg.mozilla.org/mozilla-central'
 
     image: '{{#docker_image}}phone-builder{{/docker_image}}'
     maxRunTime: 14400
 
     command:
-      - phone
+      - build-phone.sh
 
     artifacts:
       'private/build':

testing/docker/phone-builder/tests/valid_external_repo.yml

@@ -1,52 +0,0 @@
-taskId: 1
-task:
-  metadata:
-    source: http://todo.com/soon
-    owner: user@domain.com
-    name: B2G flame-kk opt
-    description: B2G flame-kk opt
-
-  workerType: b2gbuild
-  provisionerId: aws-provisioner
-
-  scopes:
-    - 'docker-worker:cache:build-phone-objects'
-    - 'docker-worker:image:{{#docker_image}}phone-builder{{/docker_image}}'
-
-  payload:
-    cache:
-      build-phone-objects: '/home/worker/object-folder-flame-kk-1'
-
-    env:
-      TARGET: 'flame-kk'
-      B2G_DEBUG: '1'
-      # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js
-      REVISION: 'tip'
-      GECKO_HEAD_REPOSITORY: 'http://github.com/walac/gecko-dev'
-      GECKO_BASE_REPOSITORY: 'https://bitbucket.org/walac/gecko-dev'
-
-    image: '{{#docker_image}}phone-builder{{/docker_image}}'
-    maxRunTime: 14400
-
-    command:
-      - phone
-
-    artifacts:
-      'private/build':
-        type: directory
-        path: '/home/worker/artifacts/'
-        expires: '{{#from_now}}1 year{{/from_now}}'
-
-  extra:
-    # Rather then enforcing particular conventions we require that all build
-    # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
-    locations:
-      build: 'private/build/b2g-android-arm.tar.gz'
-      img: 'private/build/flame-kk.zip'
-      tests: 'private/build/gaia.zip'
-      symbols: 'private/build/b2g-crashreporter-symbols.zip'
-      sources: 'private/build/sources.xml'
-
-    treeherder:
-      symbol: B

testing/taskcluster/scripts/phone-builder/pre-build.sh

@@ -4,7 +4,7 @@
 DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 PATH=$DIRNAME:$PATH
 
-export WORKSPACE=$HOME/workspace
+WORKSPACE=$1
 
 gecko_objdir=/home/worker/objdir-gecko/objdir
 
@@ -16,10 +16,7 @@ test $GECKO_HEAD_REV # Should be an hg revision to pull down
 test $TARGET
 test $VARIANT
 
-export CCACHE_DIR=$WORKSPACE/ccache
-
-ccache -M 12G
-ccache -s
+. ../builder/setup-ccache.sh
 
 # Figure out where the remote manifest is so we can use caches for it.
 MANIFEST=$(repository-url.py $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV b2g/config/$TARGET/sources.xml)

testing/taskcluster/tasks/builds/b2g_aries_spark_debug.yml

@@ -23,7 +23,10 @@ task:
       B2G_SYSTEM_APPS: '1'
       MOZHARNESS_CONFIG: b2g/taskcluster-spark.py
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_aries_spark_opt.yml

@@ -23,7 +23,10 @@ task:
       B2G_SYSTEM_APPS: '1'
       MOZHARNESS_CONFIG: b2g/taskcluster-spark.py
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_aries_spark_ota_base.yml

@@ -13,7 +13,10 @@ task:
       DEBUG: 0
       MOZHARNESS_CONFIG: b2g/taskcluster-spark-ota.py
     command:
-      - phone-ota
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone-ota.sh $HOME/workspace
 
   extra:
     treeherder:

testing/taskcluster/tasks/builds/b2g_dolphin_base.yml

@@ -15,7 +15,10 @@ task:
     maxRunTime: 7200
 
     command:
-      - dolphin
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-dolphin.sh $HOME/workspace
 
   extra:
     # Rather then enforcing particular conventions we require that all build

testing/taskcluster/tasks/builds/b2g_flame_kk_debug.yml

@@ -20,7 +20,10 @@ task:
       DEBUG: 0
       VARIANT: userdebug
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_flame_kk_opt.yml

@@ -19,7 +19,10 @@ task:
       TARGET: 'flame-kk'
       DEBUG: 0
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_flame_kk_ota_base.yml

@@ -12,7 +12,10 @@ task:
       TARGET: 'flame-kk'
       DEBUG: 0
     command:
-      - phone-ota
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone-ota.sh $HOME/workspace
 
   extra:
     treeherder:

testing/taskcluster/tasks/builds/b2g_flame_kk_spark_eng.yml

@@ -16,6 +16,7 @@ task:
       build-flame-kk-spark-eng-objdir-gecko-{{project}}: /home/worker/objdir-gecko
     env:
       TARGET: 'flame-kk'
+      MOZHARNESS_CONFIG: b2g/taskcluster-spark.py
   extra:
     treeherderEnv:
       - staging

testing/taskcluster/tasks/builds/b2g_nexus_4_eng.yml

@@ -15,6 +15,11 @@ task:
     env:
       TARGET: 'nexus-4'
       DEBUG: 0
+    command:
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_nexus_4_kk_eng.yml

@@ -17,6 +17,11 @@ task:
     env:
       TARGET: 'nexus-4-kk'
       DEBUG: 0
+    command:
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_nexus_4_kk_user.yml

@@ -18,7 +18,10 @@ task:
       TARGET: 'nexus-4-kk'
       DEBUG: 0
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_nexus_4_user.yml

@@ -17,7 +17,10 @@ task:
       TARGET: 'nexus-4'
       DEBUG: 0
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_nexus_5l_eng.yml

@@ -15,6 +15,11 @@ task:
     env:
       TARGET: 'nexus-5-l'
       DEBUG: 0
+    command:
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_nexus_5l_user.yml

@@ -17,7 +17,10 @@ task:
       TARGET: 'nexus-5-l'
       DEBUG: 0
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
   extra:
     treeherderEnv:
       - production

testing/taskcluster/tasks/builds/b2g_phone_eng_base.yml

@@ -14,4 +14,7 @@ task:
       GAIA_KEYBOARD_LAYOUTS: 'en,pt-BR,es,de,fr,pl,zh-Hans-Pinyin,zh-Hant-Zhuyin,en-Dvorak'
       B2G_UPDATE_CHANNEL: 'default'
     command:
-      - phone
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/phone-builder &&
+        buildbot_step 'Build' ./build-phone.sh $HOME/workspace