Initial golang Benchmarks

PiperOrigin-RevId: 321021071
This commit is contained in:
Zach Koopmans
2020-07-13 13:29:30 -07:00
committed by gVisor bot
parent 505b4f5e5f
commit b8d3d09bd1
19 changed files with 684 additions and 27 deletions
+21
View File
@@ -0,0 +1,21 @@
FROM ubuntu:18.04
RUN set -x \
&& apt-get update \
&& apt-get install -y \
wget \
git \
pkg-config \
zip \
g++ \
zlib1g-dev \
unzip \
python3 \
&& rm -rf /var/lib/apt/lists/*
RUN wget https://github.com/bazelbuild/bazel/releases/download/0.27.0/bazel-0.27.0-installer-linux-x86_64.sh
RUN chmod +x bazel-0.27.0-installer-linux-x86_64.sh
RUN ./bazel-0.27.0-installer-linux-x86_64.sh
RUN mkdir abseil-cpp && cd abseil-cpp \
&& git init && git remote add origin https://github.com/abseil/abseil-cpp.git \
&& git fetch --depth 1 origin 43ef2148c0936ebf7cb4be6b19927a9d9d145b8f && git checkout FETCH_HEAD
+8
View File
@@ -0,0 +1,8 @@
FROM ubuntu:18.04
RUN set -x \
&& apt-get update \
&& apt-get install -y \
iperf \
&& rm -rf /var/lib/apt/lists/*
+5
View File
@@ -119,3 +119,8 @@ func Save(logger testutil.Logger, image string, w io.Writer) error {
cmd.Stdout = w // Send directly to the writer.
return cmd.Run()
}
// Runtime returns the value of the flag runtime.
func Runtime() string {
return *runtime
}
+9 -24
View File
@@ -16,30 +16,15 @@
source $(dirname $0)/common.sh
# gcloud may be installed as a "snap". If it is, include it in PATH.
declare -r snap="/snap/bin"
if [[ -d "${snap}" ]]; then
export PATH="${PATH}:${snap}"
make load-all-images
if [[ -z "${1:-}" ]]; then
target=$(query "attr(tags, manual, tests(//test/benchmarks/...))")
else
target="$1"
fi
# Make sure we can find gcloud and exit if not.
which gcloud
install_runsc_for_benchmarks benchmark
# Exporting for subprocesses as GCP APIs and tools check this environmental
# variable for authentication.
export GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_KEYSTORE_DIR}/${GCLOUD_CREDENTIALS}"
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"
gcloud config set project ${PROJECT}
gcloud config set compute/zone ${ZONE}
bazel run //benchmarks:benchmarks -- \
--verbose \
run-gcp \
"(startup|absl)" \
--internal \
--runtime=runc \
--runtime=runsc \
--installers=head
echo $target
benchmark_runsc $target "${@:2}"
+27
View File
@@ -42,6 +42,15 @@ function test_runsc() {
test --test_arg=--runtime=${RUNTIME} "$@"
}
function benchmark_runsc() {
test_runsc -c opt \
--nocache_test_results \
--test_arg=-test.bench=. \
--test_arg=-test.benchmem \
--jobs=1 \
"$@"
}
function install_runsc_for_test() {
local -r test_name=$1
shift
@@ -63,6 +72,24 @@ function install_runsc_for_test() {
"$@"
}
function install_runsc_for_benchmarks() {
local -r test_name=$1
shift
if [[ -z "${test_name}" ]]; then
echo "Missing mandatory test name"
exit 1
fi
# Add test to the name, so it doesn't conflict with other runtimes.
set_runtime $(find_branch_name)_"${test_name}"
# ${RUNSC_TEST_NAME} is set by tests (see dockerutil) to pass the test name
# down to the runtime.
install_runsc "${RUNTIME}" \
--TESTONLY-test-name-env=RUNSC_TEST_NAME \
"$@"
}
# Installs the runsc with given runtime name. set_runtime must have been called
# to set runtime and logs location.
function install_runsc() {
+1 -1
View File
@@ -64,7 +64,7 @@ function run_as_root() {
}
function query() {
QUERY_RESULT=$(bazel query "$@")
bazel query "$@"
}
function collect_logs() {
+1 -1
View File
@@ -19,5 +19,5 @@ source $(dirname $0)/common.sh
make load-packetdrill
install_runsc_for_test runsc-d
query "attr(tags, manual, tests(//test/packetdrill/...))"
QUERY_RESULT=$(query "attr(tags, manual, tests(//test/packetdrill/...))")
test_runsc $QUERY_RESULT
+1 -1
View File
@@ -19,5 +19,5 @@ source $(dirname $0)/common.sh
make load-packetimpact
install_runsc_for_test runsc-d
query "attr(tags, packetimpact, tests(//test/packetimpact/...))"
QUERY_RESULT=$(query "attr(tags, packetimpact, tests(//test/packetimpact/...))")
test_runsc $QUERY_RESULT
+118
View File
@@ -0,0 +1,118 @@
# Benchmark tools
This package and subpackages are for running macro benchmarks on `runsc`. They
are meant to replace the previous //benchmarks benchmark-tools written in
python.
Benchmarks are meant to look like regular golang benchmarks using the testing.B
library.
## Setup
To run benchmarks you will need:
* Docker installed (17.09.0 or greater).
The easiest way to run benchmarks is to use the script at
//scripts/benchmark.sh.
If not using the script, you will need:
* `runsc` configured with docker
Note: benchmarks call the runtime by name. If docker can run it with
`--runtime=` flag, these tools should work.
## Running benchmarks
The easiest way to run is with the script at //scripts/benchmarks.sh. The script
will run all benchmarks under //test/benchmarks if a target is not provided.
```bash
./script/benchmarks.sh //path/to/target
```
If you want to run benchmarks manually:
* Run `make load-all-images` from `//`
* Run with:
```bash
bazel test --test_arg=--runtime=RUNTIME -c opt --test_output=streamed --test_timeout=600 --test_arg=-test.bench=. --nocache_test_results //path/to/target
```
## Writing benchmarks
Benchmarks consist of docker images as Dockerfiles and golang testing.B
benchmarks.
### Dockerfiles:
* Are stored at //images.
* New Dockerfiles go in an appropriately named directory at
`//images/benchmarks/my-cool-dockerfile`.
* Dockerfiles for benchmarks should:
* Use explicitly versioned packages.
* Not use ENV and CMD statements...it is easy to add these in the API.
* Note: A common pattern for getting access to a tmpfs mount is to copy files
there after container start. See: //test/benchmarks/build/bazel_test.go. You
can also make your own with `RunOpts.Mounts`.
### testing.B packages
In general, benchmarks should look like this:
```golang
var h harness.Harness
func BenchmarkMyCoolOne(b *testing.B) {
machine, err := h.GetMachine()
// check err
ctx := context.Background()
container := machine.GetContainer(ctx, b)
defer container.CleanUp(ctx)
b.ResetTimer()
//Respect b.N.
for i := 0; i < b.N; i++ {
out, err := container.Run(ctx, dockerutil.RunOpts{
Image: "benchmarks/my-cool-image",
Env: []string{"MY_VAR=awesome"},
other options...see dockerutil
}, "sh", "-c", "echo MY_VAR" ...)
//check err
b.StopTimer()
// Do parsing and reporting outside of the timer.
number := parseMyMetric(out)
b.ReportMetric(number, "my-cool-custom-metric")
b.StartTimer()
}
}
func TestMain(m *testing.M) {
h.Init()
os.Exit(m.Run())
}
```
Some notes on the above:
* The harness is initiated in the TestMain method and made global to test
module. The harness will handle any presetup that needs to happen with
flags, remote virtual machines (eventually), and other services.
* Respect `b.N` in that users of the benchmark may want to "run for an hour"
or something of the sort.
* Use the `b.ReportMetric` method to report custom metrics.
* Set the timer if time is useful for reporting. There isn't a way to turn off
default metrics in testing.B (B/op, allocs/op, ns/op).
* Take a look at dockerutil at //pkg/test/dockerutil to see all methods
available from containers. The API is based on the "official"
[docker API for golang](https://pkg.go.dev/mod/github.com/docker/docker).
* `harness.GetMachine` marks how many machines this tests needs. If you have a
client and server and to mark them as multiple machines, call it
`GetMachine` twice.
+24
View File
@@ -0,0 +1,24 @@
load("//tools:defs.bzl", "go_library", "go_test")
package(licenses = ["notice"])
go_library(
name = "fs",
srcs = ["fs.go"],
)
go_test(
name = "fs_test",
size = "large",
srcs = ["bazel_test.go"],
library = ":fs",
tags = [
# Requires docker and runsc to be configured before test runs.
"local",
"manual",
],
deps = [
"//pkg/test/dockerutil",
"//test/benchmarks/harness",
],
)
+103
View File
@@ -0,0 +1,103 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fs
import (
"context"
"os"
"strings"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
)
var h harness.Harness
// Note: CleanCache versions of this test require running with root permissions.
func BenchmarkABSL(b *testing.B) {
// Get a machine from the Harness on which to run.
machine, err := h.GetMachine()
if err != nil {
b.Fatalf("failed to get machine: %v", err)
}
// Dimensions here are clean/dirty cache (do or don't drop caches)
// and if the mount on which we are compiling is a tmpfs/bind mount.
benchmarks := []struct {
name string
clearCache bool // clearCache drops caches before running.
tmpfs bool // tmpfs will run compilation on a tmpfs.
}{
{name: "CleanCache", clearCache: true, tmpfs: false},
{name: "DirtyCache", clearCache: false, tmpfs: false},
{name: "CleanCacheTmpfs", clearCache: true, tmpfs: true},
{name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true},
}
for _, bm := range benchmarks {
b.Run(bm.name, func(b *testing.B) {
// Grab a container.
ctx := context.Background()
container := machine.GetContainer(ctx, b)
defer container.CleanUp(ctx)
workdir := "/abseil-cpp"
// Start a container.
if err := container.Spawn(ctx, dockerutil.RunOpts{
Image: "benchmarks/absl",
}, "sleep", "1000"); err != nil {
b.Fatalf("run failed with: %v", err)
}
// If we are running on a tmpfs, copy to /tmp which is a tmpfs.
if bm.tmpfs {
if _, err := container.Exec(ctx, dockerutil.ExecOpts{},
"cp", "-r", "/abseil-cpp", "/tmp/."); err != nil {
b.Fatal("failed to copy directory: %v", err)
}
workdir = "/tmp" + workdir
}
// Drop Caches.
if bm.clearCache {
if out, err := machine.RunCommand("/bin/sh -c sync; echo 3 > /proc/sys/vm/drop_caches"); err != nil {
b.Fatalf("failed to drop caches: %v %s", err, out)
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
got, err := container.Exec(ctx, dockerutil.ExecOpts{
WorkDir: workdir,
}, "bazel", "build", "-c", "opt", "absl/base/...")
if err != nil {
b.Fatalf("build failed with: %v", err)
}
b.StopTimer()
want := "Build completed successfully"
if !strings.Contains(got, want) {
b.Fatalf("string %s not in: %s", want, got)
}
b.StartTimer()
}
})
}
}
func TestMain(m *testing.M) {
h.Init()
os.Exit(m.Run())
}
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package fs holds benchmarks around filesystem performance.
package fs
+18
View File
@@ -0,0 +1,18 @@
load("//tools:defs.bzl", "go_library")
package(licenses = ["notice"])
go_library(
name = "harness",
testonly = 1,
srcs = [
"harness.go",
"machine.go",
"util.go",
],
visibility = ["//:sandbox"],
deps = [
"//pkg/test/dockerutil",
"//pkg/test/testutil",
],
)
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package harness holds utility code for running benchmarks on Docker.
package harness
import (
"flag"
"gvisor.dev/gvisor/pkg/test/dockerutil"
)
// Harness is a handle for managing state in benchmark runs.
type Harness struct {
}
// Init performs any harness initilialization before runs.
func (h *Harness) Init() error {
flag.Parse()
dockerutil.EnsureSupportedDockerVersion()
return nil
}
// GetMachine returns this run's implementation of machine.
func (h *Harness) GetMachine() (Machine, error) {
return &localMachine{}, nil
}
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package harness
import (
"context"
"net"
"os/exec"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/pkg/test/testutil"
)
// Machine describes a real machine for use in benchmarks.
type Machine interface {
// GetContainer gets a container from the machine,
GetContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container
// RunCommand runs cmd on this machine.
RunCommand(cmd string, args ...string) (string, error)
// Returns IP Address for the machine.
IPAddress() (net.IP, error)
}
// localMachine describes this machine.
type localMachine struct {
}
// GetContainer implements Machine.GetContainer for localMachine.
func (l *localMachine) GetContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container {
return dockerutil.MakeContainer(ctx, logger)
}
// RunCommand implements Machine.RunCommand for localMachine.
func (l *localMachine) RunCommand(cmd string, args ...string) (string, error) {
c := exec.Command(cmd, args...)
out, err := c.CombinedOutput()
return string(out), err
}
// IPAddress implements Machine.IPAddress.
func (l *localMachine) IPAddress() (net.IP, error) {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
return nil, err
}
defer conn.Close()
addr := conn.LocalAddr().(*net.UDPAddr)
return addr.IP, nil
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package harness
import (
"context"
"fmt"
"net"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/pkg/test/testutil"
)
// WaitUntilServing grabs a container from `machine` and waits for a server at
// IP:port.
func WaitUntilServing(ctx context.Context, machine Machine, server net.IP, port int) error {
var logger testutil.DefaultLogger = "netcat"
netcat := machine.GetContainer(ctx, logger)
defer netcat.CleanUp(ctx)
cmd := fmt.Sprintf("while ! nc -zv %s %d; do true; done", server.String(), port)
_, err := netcat.Run(ctx, dockerutil.RunOpts{
Image: "packetdrill",
}, "sh", "-c", cmd)
return err
}
+25
View File
@@ -0,0 +1,25 @@
load("//tools:defs.bzl", "go_library", "go_test")
package(licenses = ["notice"])
go_library(
name = "network",
testonly = 1,
srcs = ["network.go"],
)
go_test(
name = "network_test",
size = "large",
srcs = ["iperf_test.go"],
library = ":network",
tags = [
# Requires docker and runsc to be configured before test runs.
"manual",
"local",
],
deps = [
"//pkg/test/dockerutil",
"//test/benchmarks/harness",
],
)
+151
View File
@@ -0,0 +1,151 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package network
import (
"context"
"fmt"
"os"
"regexp"
"strconv"
"strings"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
)
var h harness.Harness
func BenchmarkIperf(b *testing.B) {
// Get two machines
clientMachine, err := h.GetMachine()
if err != nil {
b.Fatalf("failed to get machine: %v", err)
}
serverMachine, err := h.GetMachine()
if err != nil {
b.Fatalf("failed to get machine: %v", err)
}
for _, bm := range []struct {
name string
clientRuntime string
serverRuntime string
}{
// We are either measuring the server or the client. The other should be
// runc. e.g. Upload sees how fast the runtime under test uploads to a native
// server.
{name: "Upload", clientRuntime: dockerutil.Runtime(), serverRuntime: "runc"},
{name: "Download", clientRuntime: "runc", serverRuntime: dockerutil.Runtime()},
} {
b.Run(bm.name, func(b *testing.B) {
// Get a container from the server and set its runtime.
ctx := context.Background()
server := serverMachine.GetContainer(ctx, b)
defer server.CleanUp(ctx)
server.Runtime = bm.serverRuntime
// Get a container from the client and set its runtime.
client := clientMachine.GetContainer(ctx, b)
defer client.CleanUp(ctx)
client.Runtime = bm.clientRuntime
// iperf serves on port 5001 by default.
port := 5001
// Start the server.
if err := server.Spawn(ctx, dockerutil.RunOpts{
Image: "benchmarks/iperf",
Ports: []int{port},
}, "iperf", "-s"); err != nil {
b.Fatalf("failed to start server with: %v", err)
}
ip, err := serverMachine.IPAddress()
if err != nil {
b.Fatalf("failed to find server ip: %v", err)
}
servingPort, err := server.FindPort(ctx, port)
if err != nil {
b.Fatalf("failed to find port %d: %v", port, err)
}
// Make sure the server is up and serving before we run.
if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil {
b.Fatalf("failed to wait for server: %v", err)
}
// iperf report in Kb realtime
cmd := fmt.Sprintf("iperf -f K --realtime -c %s -p %d", ip.String(), servingPort)
// Run the client.
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, err := client.Run(ctx, dockerutil.RunOpts{
Image: "benchmarks/iperf",
}, strings.Split(cmd, " ")...)
if err != nil {
b.Fatalf("failed to run client: %v", err)
}
b.StopTimer()
// Parse bandwidth and report it.
bW, err := bandwidth(out)
if err != nil {
b.Fatalf("failed to parse bandwitdth from %s: %v", out, err)
}
b.ReportMetric(bW, "KBytes/sec")
b.StartTimer()
}
})
}
}
// bandwidth parses the Bandwidth number from an iperf report. A sample is below.
func bandwidth(data string) (float64, error) {
re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`)
match := re.FindStringSubmatch(data)
if len(match) < 1 {
return 0, fmt.Errorf("failed get bandwidth: %s", data)
}
return strconv.ParseFloat(match[1], 64)
}
func TestParser(t *testing.T) {
sampleData := `
------------------------------------------------------------
Client connecting to 10.138.15.215, TCP port 32779
TCP window size: 45.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 459520 KBytes 45900 KBytes/sec
`
bandwidth, err := bandwidth(sampleData)
if err != nil || bandwidth != 45900 {
t.Fatalf("failed with: %v and %f", err, bandwidth)
}
}
func TestMain(m *testing.M) {
h.Init()
os.Exit(m.Run())
}
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package network holds benchmarks around raw network performance.
package network