Bug 1125973: Docker images for building Firefox Desktop and for Android; r=garndt

This creates two images:

 * ubuntu-build is a basic Ubuntu Trusty image with the build prerequisites
   installed via MozBootstrap (plus some additional requirements).  It also
   contains the worker user and basic directory structure expected by
   Mozharness.

 * desktop-build is a refinement of ubuntu-build with specifics for building
   Firefox Desktop (and, as it turns out, Firefox for Android).  It sports a
   `bin/build.sh` which acts as a fairly generic mozharness-runner that first
   checks out the desired source code revisions, then invokes Mozharness.  It
   supports:

    * caches -- tooltool, workspace, tc-vcs
    * starting and stopping Xvfb if necessary (desktop tests require this)
    * specifying mozharness build variant, branch, and build pool
    * supplying a RelengAPI token
    * copying uploads to the artifacts directory

testing/docker/desktop-build/Dockerfile

@@ -0,0 +1,32 @@
+FROM          quay.io/mozilla/ubuntu-build:0.0.1
+MAINTAINER    Morgan Reece Phillips <winter2718@gmail.com>
+
+# Add build scripts; these are the entry points from the taskcluster worker, and
+# operate on environment variables
+ADD             bin /home/worker/bin
+RUN             chmod +x /home/worker/bin/*
+
+
+## COMPILER HACKS
+
+# So that the compiler can find installed libs
+ENV           LIBRARY_PATH  /usr/lib/x86_64-linux-gnu:$LIBRARY_PATH
+
+# a.out.h needs to exist one directory lower, or the compiler will not find it
+RUN ln -s /usr/include/linux/a.out.h /usr/include/a.out.h
+
+# Without this, zlib.h can't find zconf.h, so hey, symlinks to the rescue, right?
+RUN ln -s /usr/include/x86_64-linux-gnu/zconf.h /usr/include/zconf.h
+
+# Stubbed out credentials, which will force the upload step to be skipped.
+# Note that this needs to be in the parent of the workspace directory and in
+# the directory where mozharness is run (not its --work-dir)
+ADD           oauth.txt /home/worker/
+
+# stubbed out buildprops, which keeps mozharness from choking
+# Note that this needs to be in the parent of the workspace directory and in
+# the directory where mozharness is run (not its --work-dir)
+ADD           buildprops.json /home/worker/
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

testing/docker/desktop-build/VERSION

@@ -0,0 +1 @@
+0.0.7

testing/docker/desktop-build/bin/build.sh

@@ -0,0 +1,134 @@
+#! /bin/bash -vex
+
+set -x
+
+# Inputs, with defaults
+
+: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+
+: GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
+: GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
+: GECKO_REV                     ${GECKO_REV:=default}
+
+: MOZHARNESS_BASE_REPOSITORY    ${MOZHARNESS_BASE_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
+: MOZHARNESS_HEAD_REPOSITORY    ${MOZHARNESS_HEAD_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
+: MOZHARNESS_REV                ${MOZHARNESS_REV:=production}
+
+: TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=https://hg.mozilla.org/build/tools}
+: TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=https://hg.mozilla.org/build/tools}
+: TOOLS_REV                     ${TOOLS_REV:=default}
+
+: TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
+
+: RELENGAPI_TOKEN               ${RELENGAPI_TOKEN+HIDDEN}
+
+: NEED_XVFB                     ${NEED_XVFB:=false}
+
+: MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
+: MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
+: MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
+
+: MOZ_SIGNING_SERVERS           ${MOZ_SIGNING_SERVERS}
+: MOZ_SIGN_CMD                  ${MOZ_SIGN_CMD}
+
+: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
+
+# buildbot
+export CCACHE_COMPRESS=1
+export CCACHE_DIR=/builds/ccache
+export CCACHE_HASHDIR=
+export CCACHE_UMASK=002
+
+export MOZ_AUTOMATION=1
+export MOZ_CRASHREPORTER_NO_REPORT=1
+export MOZ_OBJDIR=obj-firefox
+export MOZ_SYMBOLS_EXTRA_BUILDID=linux64
+export POST_SYMBOL_UPLOAD_CMD=/usr/local/bin/post-symbol-upload.py
+export TINDERBOX_OUTPUT=1
+
+# Ensure that in tree libraries can be found
+export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
+
+# test required parameters are supplied
+test ${MOZHARNESS_SCRIPT}
+test ${MOZHARNESS_CONFIG}
+
+cleanup() {
+    [ -n "$xvfb_pid" ] && kill $xvfb_pid
+}
+trap cleanup EXIT INT
+
+# check out mozharness
+tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_REV
+
+# check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
+tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_REV
+if [ ! -d build ]; then
+    mkdir -p build
+    ln -s $WORKSPACE/build/tools build/tools
+fi
+
+# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared)
+tc-vcs checkout /builds/hg-shared/mozilla-central $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_REV
+
+# run mozharness in XVfb, if necessary; this is an array to maintain the quoting in the -s argument
+if $NEED_XVFB; then
+    # Some mozharness scripts set DISPLAY=:2
+    Xvfb :2 -screen 0 1024x768x24 &
+    export DISPLAY=:2
+    xvfb_pid=$!
+    # Only error code 255 matters, because it signifies that no
+    # display could be opened. As long as we can open the display
+    # tests should work.
+    sleep 2 # we need to sleep so that Xvfb has time to startup
+    xvinfo || if [ $? == 255 ]; then exit 255; fi
+fi
+
+# set up mozharness configuration, via command line, env, etc.
+
+debug_flag=""
+if [ 0$DEBUG -ne 0 ]; then
+  debug_flag='--debug'
+fi
+
+custom_build_variant_cfg_flag=""
+if [ -n "${MH_CUSTOM_BUILD_VARIANT_CFG}" ]; then
+    custom_build_variant_cfg_flag="--custom-build-variant-cfg=${MH_CUSTOM_BUILD_VARIANT_CFG}"
+fi
+
+set +x
+# mozharness scripts look for the relengapi token at this location, so put it there,
+# if specified
+if [ -n "${RELENGAPI_TOKEN}" ]; then
+    echo 'Storing $RELENGAPI_TOKEN in /builds/relengapi.tok'
+    echo ${RELENGAPI_TOKEN} > /builds/relengapi.tok
+    # unset it so that mozharness doesn't "helpfully" log it
+    unset RELENGAPI_TOKEN
+fi
+set -x
+
+# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
+# cache.  However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
+# entirely effective.
+export TOOLTOOL_CACHE
+
+./${MOZHARNESS_SCRIPT} \
+  --config ${MOZHARNESS_CONFIG} \
+  $debug_flag \
+  $custom_build_variant_cfg_flag \
+  --disable-mock \
+  --no-setup-mock \
+  --no-clone-tools \
+  --no-clobber \
+  --no-update \
+  --log-level=debug \
+  --work-dir=$WORKSPACE/build \
+  --no-action=generate-build-stats \
+  --branch=${MH_BRANCH} \
+  --build-pool=${MH_BUILD_POOL}
+
+# if mozharness has created an "upload" directory, copy all of that into artifacts
+if [ -d $WORKSPACE/build/upload ]; then
+    cp -r $WORKSPACE/build/upload/* $HOME/artifacts/
+fi

testing/docker/desktop-build/buildprops.json

@@ -0,0 +1,8 @@
+{
+  "properties": {
+    "buildername": ""
+  },
+  "sourcestamp": {
+    "changes": []
+  }
+}

testing/docker/desktop-build/oauth.txt

@@ -0,0 +1,2 @@
+taskcluster_clientId = None
+taskcluster_accessToken = None

testing/docker/ubuntu-build/Dockerfile

@@ -0,0 +1,36 @@
+FROM          ubuntu:14.04
+MAINTAINER    Morgan Reece Phillips <winter2718@gmail.com>
+
+### add worker user and setup its workspace
+RUN useradd -d /home/worker -s /bin/bash -m worker
+
+# install non-build specific dependencies in a single layer
+ADD           system-setup.sh   /tmp/system-setup.sh
+RUN           bash /tmp/system-setup.sh
+
+# configure git and install tc-vcs
+RUN git config --global user.email "nobody@mozilla.com" && \
+    git config --global user.name "mozilla"
+RUN npm install -g taskcluster-vcs@2.3.5 || true
+
+# Ensure that build specific dependencies live in a single layer
+ADD           build-setup.sh   /tmp/build-setup.sh
+RUN           bash /tmp/build-setup.sh
+
+# Builds need the share module enabled
+ADD           hgrc /home/worker/.hgrc
+
+# Set variable normally configured at login, by the shells parent process, these
+# are taken from GNU su manual
+ENV           HOME          /home/worker
+ENV           SHELL         /bin/bash
+ENV           USER          worker
+ENV           LOGNAME       worker
+ENV           HOSTNAME      taskcluster-worker
+
+# Declare default working folder
+RUN chown -R worker:worker /home/worker/* /home/worker/.*
+WORKDIR       /home/worker
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

testing/docker/ubuntu-build/VERSION

@@ -0,0 +1 @@
+0.0.1

testing/docker/ubuntu-build/build-setup.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ve
+
+test `whoami` == 'root';
+
+# run mozbootstrap to install build specific dependencies
+wget -q https://hg.mozilla.org/mozilla-central/raw-file/default/python/mozboot/bin/bootstrap.py
+python bootstrap.py --application-choice=desktop --no-interactive
+
+# note that TC will replace workspace with a cache mount; there's no sense
+# creating anything inside there
+mkdir -p /home/worker/workspace
+chown worker:worker /home/worker/workspace
+
+# /builds is *not* replaced with a mount in the docker container. The worker
+# user writes to lots of subdirectories, though, so it's owned by that user
+mkdir -p /builds
+chown worker:worker /builds
+
+# install tooltool directly from github where tooltool_wrapper.sh et al. expect
+# to find it
+wget -O /builds/tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py
+chmod +x /builds/tooltool.py
+
+# check out the tools repo; this will be updated as necessary in each container
+# but it changes infrequently so it makes sense to cache in place
+tc-vcs checkout /builds/tools https://hg.mozilla.org/build/tools
+chown -R worker:worker /builds/tools
+
+rm /tmp/build-setup.sh

testing/docker/ubuntu-build/hgrc

@@ -0,0 +1,2 @@
+[extensions]
+share =

testing/docker/ubuntu-build/system-setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ve
+
+test `whoami` == 'root';
+
+apt-get update -y
+apt-get install -y \
+    wget \
+    python g++-multilib \
+    git\
+    nodejs-legacy \
+    npm \
+    curl \
+    x11-utils \
+    python-virtualenv
+
+# see https://bugzilla.mozilla.org/show_bug.cgi?id=1161075
+apt-get install -y openjdk-7-jdk
+
+# the Android SDK contains some 32-bit binaries (aapt among them) that require this
+apt-get install -y lib32z1
+
+rm /tmp/system-setup.sh