Add tools for generating images.

This formalizes the adhoc scripts previously in kokoro. The image targets can
be used by e.g. benchmarks in order to automated image prepation.

PiperOrigin-RevId: 290982744
This commit is contained in:
Adin Scannell
2020-01-22 10:39:45 -08:00
committed by gVisor bot
parent 6a59e7f510
commit cb3906ae00
22 changed files with 590 additions and 46 deletions
-34
View File
@@ -1,34 +0,0 @@
## Image Update
After making changes to files in the directory, you must run the following
commands to update the image Kokoro uses:
```shell
gcloud config set project gvisor-kokoro-testing
third_party/gvisor/kokoro/ubuntu1604/build.sh
third_party/gvisor/kokoro/ubuntu1804/build.sh
```
Note: the command above will change your default project for `gcloud`. Run
`gcloud config set project` again to revert back to your default project.
Note: Files in `third_party/gvisor/kokoro/ubuntu1804/` as symlinks to
`ubuntu1604`, therefore both images must be updated.
After the script finishes, the last few lines of the output will container the
image name. If the output was lost, you can run `build.sh` again to print the
image name.
```
NAME PROJECT FAMILY DEPRECATED STATUS
image-6777fa4666a968c8 gvisor-kokoro-testing READY
+ cleanup
+ gcloud compute instances delete --quiet build-tlfrdv
Deleted [https://www.googleapis.com/compute/v1/projects/gvisor-kokoro-testing/zones/us-central1-f/instances/build-tlfrdv].
```
To setup Kokoro to use the new image, copy the image names to their
corresponding file below:
* //devtools/kokoro/config/gcp/gvisor/ubuntu1604.gcl
* //devtools/kokoro/config/gcp/gvisor/ubuntu1804.gcl
-1
View File
@@ -1 +0,0 @@
../ubuntu1604/10_core.sh
-1
View File
@@ -1 +0,0 @@
../ubuntu1604/20_bazel.sh
-1
View File
@@ -1 +0,0 @@
../ubuntu1604/25_docker.sh
-1
View File
@@ -1 +0,0 @@
../ubuntu1604/30_containerd.sh
-1
View File
@@ -1 +0,0 @@
../ubuntu1604/40_kokoro.sh
+68
View File
@@ -0,0 +1,68 @@
load("@rules_cc//cc:defs.bzl", "cc_binary")
load("//tools/images:defs.bzl", "vm_image", "vm_test")
package(
default_visibility = ["//:sandbox"],
licenses = ["notice"],
)
genrule(
name = "zone",
outs = ["zone.txt"],
cmd = "gcloud config get-value compute/zone > $@",
tags = [
"local",
"manual",
],
)
sh_binary(
name = "builder",
srcs = ["build.sh"],
)
sh_binary(
name = "executer",
srcs = ["execute.sh"],
)
cc_binary(
name = "test",
testonly = 1,
srcs = ["test.cc"],
linkstatic = 1,
deps = [
"//test/util:test_main",
"@com_google_googletest//:gtest",
],
)
vm_image(
name = "ubuntu1604",
family = "ubuntu-1604-lts",
project = "ubuntu-os-cloud",
scripts = [
"//tools/images/ubuntu1604",
],
)
vm_test(
name = "ubuntu1604_test",
image = ":ubuntu1604",
targets = [":test"],
)
vm_image(
name = "ubuntu1804",
family = "ubuntu-1804-lts",
project = "ubuntu-os-cloud",
scripts = [
"//tools/images/ubuntu1804",
],
)
vm_test(
name = "ubuntu1804_test",
image = ":ubuntu1804",
targets = [":test"],
)
+101
View File
@@ -0,0 +1,101 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is responsible for building a new GCP image that: 1) has nested
# virtualization enabled, and 2) has been completely set up with the
# image_setup.sh script. This script should be idempotent, as we memoize the
# setup script with a hash and check for that name.
set -xeou pipefail
# Parameters.
declare -r USERNAME=${USERNAME:-test}
declare -r IMAGE_PROJECT=${IMAGE_PROJECT:-ubuntu-os-cloud}
declare -r IMAGE_FAMILY=${IMAGE_FAMILY:-ubuntu-1604-lts}
declare -r ZONE=${ZONE:-us-central1-f}
# Random names.
declare -r DISK_NAME=$(mktemp -u disk-XXXXXX | tr A-Z a-z)
declare -r SNAPSHOT_NAME=$(mktemp -u snapshot-XXXXXX | tr A-Z a-z)
declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z)
# Hash inputs in order to memoize the produced image.
declare -r SETUP_HASH=$( (echo ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && cat "$@") | sha256sum - | cut -d' ' -f1 | cut -c 1-16)
declare -r IMAGE_NAME=${IMAGE_FAMILY:-image-}${SETUP_HASH}
# Does the image already exist? Skip the build.
declare -r existing=$(gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)")
if ! [[ -z "${existing}" ]]; then
echo "${existing}"
exit 0
fi
# gcloud has path errors; is this a result of being a genrule?
export PATH=${PATH:-/bin:/usr/bin:/usr/local/bin}
# Start a unique instance. Note that this instance will have a unique persistent
# disk as it's boot disk with the same name as the instance.
gcloud compute instances create \
--quiet \
--image-project "${IMAGE_PROJECT}" \
--image-family "${IMAGE_FAMILY}" \
--boot-disk-size "200GB" \
--zone "${ZONE}" \
"${INSTANCE_NAME}" >/dev/null
function cleanup {
gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}"
}
trap cleanup EXIT
# Wait for the instance to become available (up to 5 minutes).
declare timeout=300
declare success=0
declare -r start=$(date +%s)
declare -r end=$((${start}+${timeout}))
while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then
success=$((${success}+1))
fi
done
if [[ "${success}" -eq "0" ]]; then
echo "connect timed out after ${timeout} seconds."
exit 1
fi
# Run the install scripts provided.
for arg; do
gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- sudo bash - <"${arg}" >/dev/null
done
# Stop the instance; required before creating an image.
gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null
# Create a snapshot of the instance disk.
gcloud compute disks snapshot \
--quiet \
--zone "${ZONE}" \
--snapshot-names="${SNAPSHOT_NAME}" \
"${INSTANCE_NAME}" >/dev/null
# Create the disk image.
gcloud compute images create \
--quiet \
--source-snapshot="${SNAPSHOT_NAME}" \
--licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \
"${IMAGE_NAME}" >/dev/null
# Finish up.
echo "${IMAGE_NAME}"
+178
View File
@@ -0,0 +1,178 @@
"""Image configuration.
Images can be generated by using the vm_image rule. For example,
vm_image(
name = "ubuntu",
project = "...",
family = "...",
scripts = [
"script.sh",
"other.sh",
],
)
This will always create an vm_image in the current default gcloud project. The
rule has a text file as its output containing the image name. This will enforce
serialization for all dependent rules.
Images are always named per the hash of all the hermetic input scripts. This
allows images to be memoized quickly and easily.
The vm_test rule can be used to execute a command remotely. For example,
vm_test(
name = "mycommand",
image = ":myimage",
targets = [":test"],
)
"""
def _vm_image_impl(ctx):
script_paths = []
for script in ctx.files.scripts:
script_paths.append(script.short_path)
resolved_inputs, argv, runfiles_manifests = ctx.resolve_command(
command = "USERNAME=%s ZONE=$(cat %s) IMAGE_PROJECT=%s IMAGE_FAMILY=%s %s %s > %s" %
(
ctx.attr.username,
ctx.files.zone[0].path,
ctx.attr.project,
ctx.attr.family,
ctx.executable.builder.path,
" ".join(script_paths),
ctx.outputs.out.path,
),
tools = [ctx.attr.builder] + ctx.attr.scripts,
)
ctx.actions.run_shell(
tools = resolved_inputs,
outputs = [ctx.outputs.out],
progress_message = "Building image...",
execution_requirements = {"local": "true"},
command = argv,
input_manifests = runfiles_manifests,
)
return [DefaultInfo(files = depset([ctx.outputs.out]))]
_vm_image = rule(
attrs = {
"builder": attr.label(
executable = True,
default = "//tools/images:builder",
cfg = "host",
),
"username": attr.string(default = "$(whoami)"),
"zone": attr.label(
default = "//tools/images:zone",
cfg = "host",
),
"family": attr.string(mandatory = True),
"project": attr.string(mandatory = True),
"scripts": attr.label_list(allow_files = True),
},
outputs = {
"out": "%{name}.txt",
},
implementation = _vm_image_impl,
)
def vm_image(**kwargs):
_vm_image(
tags = [
"local",
"manual",
],
**kwargs
)
def _vm_test_impl(ctx):
runner = ctx.actions.declare_file("%s-executer" % ctx.label.name)
# Note that the remote execution case must actually generate an
# intermediate target in order to collect all the relevant runfiles so that
# they can be copied over for remote execution.
runner_content = "\n".join([
"#!/bin/bash",
"export ZONE=$(cat %s)" % ctx.files.zone[0].short_path,
"export USERNAME=%s" % ctx.attr.username,
"export IMAGE=$(cat %s)" % ctx.files.image[0].short_path,
"export SUDO=%s" % "true" if ctx.attr.sudo else "false",
"%s %s" % (
ctx.executable.executer.short_path,
" ".join([
target.files_to_run.executable.short_path
for target in ctx.attr.targets
]),
),
"",
])
ctx.actions.write(runner, runner_content, is_executable = True)
# Return with all transitive files.
runfiles = ctx.runfiles(
transitive_files = depset(transitive = [
depset(target.data_runfiles.files)
for target in ctx.attr.targets
if hasattr(target, "data_runfiles")
]),
files = ctx.files.executer + ctx.files.zone + ctx.files.image +
ctx.files.targets,
collect_default = True,
collect_data = True,
)
return [DefaultInfo(executable = runner, runfiles = runfiles)]
_vm_test = rule(
attrs = {
"image": attr.label(
mandatory = True,
cfg = "host",
),
"executer": attr.label(
executable = True,
default = "//tools/images:executer",
cfg = "host",
),
"username": attr.string(default = "$(whoami)"),
"zone": attr.label(
default = "//tools/images:zone",
cfg = "host",
),
"sudo": attr.bool(default = True),
"machine": attr.string(default = "n1-standard-1"),
"targets": attr.label_list(
mandatory = True,
allow_empty = False,
cfg = "target",
),
},
test = True,
implementation = _vm_test_impl,
)
def vm_test(
installer = "//tools/installers:head",
**kwargs):
"""Runs the given targets as a remote test.
Args:
installer: Script to run before all targets.
**kwargs: All test arguments. Should include targets and image.
"""
targets = kwargs.pop("targets", [])
if installer:
targets = [installer] + targets
targets = [
] + targets
_vm_test(
tags = [
"local",
"manual",
],
targets = targets,
local = 1,
**kwargs
)
+152
View File
@@ -0,0 +1,152 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Required input.
if ! [[ -v IMAGE ]]; then
echo "no image provided: set IMAGE."
exit 1
fi
# Parameters.
declare -r USERNAME=${USERNAME:-test}
declare -r KEYNAME=$(mktemp --tmpdir -u key-XXXXXX)
declare -r SSHKEYS=$(mktemp --tmpdir -u sshkeys-XXXXXX)
declare -r INSTANCE_NAME=$(mktemp -u test-XXXXXX | tr A-Z a-z)
declare -r MACHINE=${MACHINE:-n1-standard-1}
declare -r ZONE=${ZONE:-us-central1-f}
declare -r SUDO=${SUDO:-false}
# This script is executed as a test rule, which will reset the value of HOME.
# Unfortunately, it is needed to load the gconfig credentials. We will reset
# HOME when we actually execute in the remote environment, defined below.
export HOME=$(eval echo ~$(whoami))
# Generate unique keys for this test.
[[ -f "${KEYNAME}" ]] || ssh-keygen -t rsa -N "" -f "${KEYNAME}" -C "${USERNAME}"
cat > "${SSHKEYS}" <<EOF
${USERNAME}:$(cat ${KEYNAME}.pub)
EOF
# Start a unique instance. This means that we first generate a unique set of ssh
# keys to ensure that only we have access to this instance. Note that we must
# constrain ourselves to Haswell or greater in order to have nested
# virtualization available.
gcloud compute instances create \
--min-cpu-platform "Intel Haswell" \
--preemptible \
--no-scopes \
--metadata block-project-ssh-keys=TRUE \
--metadata-from-file ssh-keys="${SSHKEYS}" \
--machine-type "${MACHINE}" \
--image "${IMAGE}" \
--zone "${ZONE}" \
"${INSTANCE_NAME}"
function cleanup {
gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}"
}
trap cleanup EXIT
# Wait for the instance to become available (up to 5 minutes).
declare timeout=300
declare success=0
declare -r start=$(date +%s)
declare -r end=$((${start}+${timeout}))
while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
if gcloud compute ssh --ssh-key-file="${KEYNAME}" --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
success=$((${success}+1))
fi
done
if [[ "${success}" -eq "0" ]]; then
echo "connect timed out after ${timeout} seconds."
exit 1
fi
# Copy the local directory over.
tar czf - --dereference --exclude=.git . |
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- tar xzf -
# Execute the command remotely.
for cmd; do
# Setup relevant environment.
#
# N.B. This is not a complete test environment, but is complete enough to
# provide rudimentary sharding and test output support.
declare -a PREFIX=( "env" )
if [[ -v TEST_SHARD_INDEX ]]; then
PREFIX+=( "TEST_SHARD_INDEX=${TEST_SHARD_INDEX}" )
fi
if [[ -v TEST_SHARD_STATUS_FILE ]]; then
SHARD_STATUS_FILE=$(mktemp -u test-shard-status-XXXXXX)
PREFIX+=( "TEST_SHARD_STATUS_FILE=/tmp/${SHARD_STATUS_FILE}" )
fi
if [[ -v TEST_TOTAL_SHARDS ]]; then
PREFIX+=( "TEST_TOTAL_SHARDS=${TEST_TOTAL_SHARDS}" )
fi
if [[ -v TEST_TMPDIR ]]; then
REMOTE_TMPDIR=$(mktemp -u test-XXXXXX)
PREFIX+=( "TEST_TMPDIR=/tmp/${REMOTE_TMPDIR}" )
# Create remotely.
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
mkdir -p "/tmp/${REMOTE_TMPDIR}"
fi
if [[ -v XML_OUTPUT_FILE ]]; then
TEST_XML_OUTPUT=$(mktemp -u xml-output-XXXXXX)
PREFIX+=( "XML_OUTPUT_FILE=/tmp/${TEST_XML_OUTPUT}" )
fi
if [[ "${SUDO}" == "true" ]]; then
PREFIX+=( "sudo" "-E" )
fi
# Execute the command.
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${PREFIX[@]}" "${cmd}"
# Collect relevant results.
if [[ -v TEST_SHARD_STATUS_FILE ]]; then
gcloud compute scp \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${SHARD_STATUS_FILE}" \
"${TEST_SHARD_STATUS_FILE}" 2>/dev/null || true # Allowed to fail.
fi
if [[ -v XML_OUTPUT_FILE ]]; then
gcloud compute scp \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${TEST_XML_OUTPUT}" \
"${XML_OUTPUT_FILE}" 2>/dev/null || true # Allowed to fail.
fi
# Clean up the temporary directory.
if [[ -v TEST_TMPDIR ]]; then
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
rm -rf "/tmp/${REMOTE_TMPDIR}"
fi
done
+23
View File
@@ -0,0 +1,23 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "gtest/gtest.h"
namespace {
TEST(Image, Sanity) {
// Do nothing.
}
} // namespace
+7
View File
@@ -0,0 +1,7 @@
package(licenses = ["notice"])
filegroup(
name = "ubuntu1604",
srcs = glob(["*.sh"]),
visibility = ["//:sandbox"],
)
+7
View File
@@ -0,0 +1,7 @@
package(licenses = ["notice"])
alias(
name = "ubuntu1804",
actual = "//tools/images/ubuntu1604",
visibility = ["//:sandbox"],
)
+22
View File
@@ -0,0 +1,22 @@
# Installers for use by the tools/vm_test rules.
package(
default_visibility = ["//:sandbox"],
licenses = ["notice"],
)
sh_binary(
name = "head",
srcs = ["head.sh"],
data = ["//runsc"],
)
sh_binary(
name = "master",
srcs = ["master.sh"],
)
sh_binary(
name = "shim",
srcs = ["shim.sh"],
)
@@ -14,7 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Install our runtime.
third_party/gvisor/runsc/runsc install
# Run the image_build.sh script with appropriate parameters.
IMAGE_PROJECT=ubuntu-os-cloud IMAGE_FAMILY=ubuntu-1604-lts $(dirname $0)/../../tools/image_build.sh $(dirname $0)/??_*.sh
# Restart docker.
service docker restart || true

Some files were not shown because too many files have changed in this diff Show More