diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 0eb19843e..cc492fbd4 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -402,6 +402,9 @@ steps: - <<: *benchmarks label: ":cd: FIO benchmarks (randread/randwrite)" command: make -i benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:fio_test BENCHMARKS_FILTER=Fio/operation\.rand BENCHMARKS_OPTIONS=--test.benchtime=15s + - <<: *benchmarks + label: ":cd: Ruby CI/CD benchmarks" + command: make -i benchmark-platforms BENCHMARKS_SUITE=fio BENCHMARKS_TARGETS=test/benchmarks/fs:rubydev_test BENCHMARKS_OPTIONS=-test.benchtime=1ns - <<: *benchmarks label: ":globe_with_meridians: HTTPD benchmarks" command: make -i benchmark-platforms BENCHMARKS_FILTER="Continuous" BENCHMARKS_SUITE=httpd BENCHMARKS_TARGETS=test/benchmarks/network:httpd_test diff --git a/images/benchmarks/rubydev/Dockerfile b/images/benchmarks/rubydev/Dockerfile new file mode 100644 index 000000000..78f190b9e --- /dev/null +++ b/images/benchmarks/rubydev/Dockerfile @@ -0,0 +1,14 @@ +FROM ruby:3.0-alpine + +# Some of these dependencies are called as subprocesses by the fastlane tests. +RUN apk add --no-cache ruby ruby-dev ruby-bundler ruby-json build-base bash \ + wget git unzip +RUN wget -q \ + 'https://github.com/fastlane/fastlane/archive/refs/tags/2.207.0.tar.gz' \ + -O /tmp/fastlane.tar.gz \ + && mkdir /fastlane \ + && cd /fastlane \ + && tar xfz /tmp/fastlane.tar.gz --strip-components=1 \ + && rm /tmp/fastlane.tar.gz \ + && for i in 1 2 3; do if bundle install; then break; fi; done +COPY . /files/ diff --git a/images/benchmarks/rubydev/run_fastlane_tests.sh b/images/benchmarks/rubydev/run_fastlane_tests.sh new file mode 100644 index 000000000..6ce404f91 --- /dev/null +++ b/images/benchmarks/rubydev/run_fastlane_tests.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# Copyright 2022 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Run only a subset of tests, otherwise this is simply too long of a benchmark. +# This list was compiled using: +# $ find -name '*_spec.rb' | shuf | head -100 | cut -d/ -f2- | sort +specs=( + cert/spec/commands_generator_spec.rb + credentials_manager/spec/account_manager_spec.rb + fastlane/spec/action_metadata_spec.rb + fastlane/spec/actions_helper_spec.rb + fastlane/spec/actions_specs/appetize_view_url_generator_spec.rb + fastlane/spec/actions_specs/apteligent_spec.rb + fastlane/spec/actions_specs/backup_file_spec.rb + fastlane/spec/actions_specs/clean_cocoapods_cache_spec.rb + fastlane/spec/actions_specs/delete_keychain_spec.rb + fastlane/spec/actions_specs/deploygate_spec.rb + fastlane/spec/actions_specs/ensure_git_status_clean_spec.rb + fastlane/spec/actions_specs/ensure_xcode_version_spec.rb + fastlane/spec/actions_specs/get_github_release_spec.rb + fastlane/spec/actions_specs/git_branch_spec.rb + fastlane/spec/actions_specs/git_pull_spec.rb + fastlane/spec/actions_specs/hg_add_tag_spec.rb + fastlane/spec/actions_specs/hg_commit_version_bump_spec.rb + fastlane/spec/actions_specs/hockey_spec.rb + fastlane/spec/actions_specs/import_certificate_spec.rb + fastlane/spec/actions_specs/import_spec.rb + fastlane/spec/actions_specs/jazzy_spec.rb + fastlane/spec/actions_specs/push_git_tags_spec.rb + fastlane/spec/actions_specs/puts_spec.rb + fastlane/spec/actions_specs/reset_git_repo_spec.rb + fastlane/spec/actions_specs/rsync_spec.rb + fastlane/spec/actions_specs/say_spec.rb + fastlane/spec/actions_specs/set_changelog_spec.rb + fastlane/spec/actions_specs/set_info_plist_value_spec.rb + fastlane/spec/actions_specs/set_pod_key_spec.rb + fastlane/spec/actions_specs/setup_ci_spec.rb + fastlane/spec/actions_specs/setup_circle_ci_spec.rb + fastlane/spec/actions_specs/spm_spec.rb + fastlane/spec/actions_specs/swiftlint_spec.rb + fastlane/spec/actions_specs/testfairy_spec.rb + fastlane/spec/actions_specs/update_app_identifier_spec.rb + fastlane/spec/actions_specs/update_keychain_access_groups_spec.rb + fastlane/spec/actions_specs/update_project_provisioning_spec.rb + fastlane/spec/actions_specs/version_bump_podspec_spec.rb + fastlane/spec/actions_specs/xcode_server_get_assets_spec.rb + fastlane/spec/actions_specs/xcodebuild_spec.rb + fastlane/spec/actions_specs/xctool_action_spec.rb + fastlane/spec/actions_specs/zip_spec.rb + fastlane/spec/command_line_handler_spec.rb + fastlane/spec/env_spec.rb + fastlane/spec/gradle_helper_spec.rb + fastlane/spec/helper/adb_helper_spec.rb + fastlane/spec/helper/s3_client_helper_spec.rb + fastlane/spec/helper/xcodeproj_helper_spec.rb + fastlane/spec/lane_list_spec.rb + fastlane/spec/runner_spec.rb + fastlane_core/spec/app_identifier_guesser_spec.rb + fastlane_core/spec/configuration_file_spec.rb + fastlane_core/spec/configuration_spec.rb + fastlane_core/spec/core_ext/cfpropertylist_ext_spec.rb + fastlane_core/spec/core_ext/shellwords_ext_spec.rb + fastlane_core/spec/fastlane_user_dir_spec.rb + fastlane_core/spec/ios_app_identifier_guesser_spec.rb + fastlane_core/spec/languages_spec.rb + gym/spec/code_signing_mapping_spec.rb + gym/spec/gymfile_spec.rb + gym/spec/options_spec.rb + gym/spec/xcodebuild_fixes/generic_archive_fix_spec.rb + match/spec/encryption/openssl_spec.rb + match/spec/storage/gitlab/client_spec.rb + precheck/spec/rules/curse_words_rule_spec.rb + precheck/spec/rules/rule_spec.rb + precheck/spec/rules/unreachable_urls_rule_spec.rb + scan/spec/commands_generator_spec.rb + scan/spec/error_handler_spec.rb + scan/spec/test_result_parser_spec.rb + screengrab/spec/commands_generator_spec.rb + sigh/spec/manager_spec.rb + sigh/spec/runner_spec.rb + snapshot/spec/test_command_generator_xcode_8_spec.rb + spaceship/spec/connect_api/client_spec.rb + spaceship/spec/connect_api/models/app_spec.rb + spaceship/spec/connect_api/models/app_store_version_spec.rb + spaceship/spec/connect_api/models/beta_feedback_spec.rb + spaceship/spec/connect_api/models/build_beta_detail_spec.rb + spaceship/spec/connect_api/models/build_delivery_spec.rb + spaceship/spec/connect_api/models/bundle_id_spec.rb + spaceship/spec/connect_api/testflight/testflight_client_spec.rb + spaceship/spec/connect_api/token_spec.rb + spaceship/spec/du/du_client_spec.rb + spaceship/spec/portal/app_group_spec.rb + spaceship/spec/portal/app_spec.rb + spaceship/spec/portal/enterprise_spec.rb + spaceship/spec/portal/merchant_spec.rb + spaceship/spec/portal/passbook_spec.rb + spaceship/spec/spaceship_base_spec.rb + spaceship/spec/spaceship_spec.rb + spaceship/spec/test_flight/app_test_info_spec.rb + spaceship/spec/tunes/app_analytics_spec.rb + spaceship/spec/tunes/app_submission_spec.rb + spaceship/spec/tunes/app_version_spec.rb + spaceship/spec/tunes/application_spec.rb + spaceship/spec/tunes/b2b_organization_spec.rb + spaceship/spec/tunes/members_spec.rb + spaceship/spec/two_step_or_factor_client_spec.rb + supply/spec/commands_generator_spec.rb +) +pattern="{" +for spec in "${specs[@]}"; do + pattern="${pattern}${spec}," +done +pattern="$(echo "$pattern" | sed -r 's/,$//')}" +exec bundle exec rspec --pattern "$pattern" diff --git a/images/benchmarks/rubydev/tc_no_op.rb b/images/benchmarks/rubydev/tc_no_op.rb new file mode 100644 index 000000000..7c72dacbd --- /dev/null +++ b/images/benchmarks/rubydev/tc_no_op.rb @@ -0,0 +1,10 @@ +# A unit test that tests nothing. +# Based on the unit test that Stripe used to benchmark gVisor: +# https://stripe.com/blog/fast-secure-builds-choose-two + +require "test/unit" + +class TestNoOp < Test::Unit::TestCase + def test_noop + end +end diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD index c2b981a07..30fb083c4 100644 --- a/test/benchmarks/fs/BUILD +++ b/test/benchmarks/fs/BUILD @@ -9,6 +9,7 @@ benchmark_test( deps = [ "//pkg/cleanup", "//pkg/test/dockerutil", + "//test/benchmarks/fs/fsbench", "//test/benchmarks/harness", "//test/benchmarks/tools", "@com_github_docker_docker//api/types/mount:go_default_library", @@ -27,3 +28,14 @@ benchmark_test( "@com_github_docker_docker//api/types/mount:go_default_library", ], ) + +benchmark_test( + name = "rubydev_test", + srcs = ["rubydev_test.go"], + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/fs/fsbench", + "//test/benchmarks/harness", + ], +) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index 5391ad4cc..eca5caf70 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -11,28 +11,19 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +// Package bazel_test benchmarks builds using bazel. package bazel_test import ( "context" - "fmt" "os" - "strings" "testing" - "gvisor.dev/gvisor/pkg/cleanup" - "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/fs/fsbench" "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" ) -// Dimensions here are clean/dirty cache (do or don't drop caches) -// and if the mount on which we are compiling is a tmpfs/bind mount. -type benchmark struct { - clearCache bool // clearCache drops caches before running. - fstype harness.FileSystemType // type of filesystem to use. -} - // Note: CleanCache versions of this test require running with root permissions. func BenchmarkBuildABSL(b *testing.B) { runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...") @@ -50,112 +41,22 @@ func BenchmarkBuildGRPC(b *testing.B) { runBuildBenchmark(b, "benchmarks/build-grpc", "/grpc", ":grpc") } -func runBuildBenchmark(b *testing.B, image, workdir, target string) { +func runBuildBenchmark(b *testing.B, image, workDir, target string) { b.Helper() + ctx := context.Background() // Get a machine from the Harness on which to run. machine, err := harness.GetMachine() if err != nil { b.Fatalf("Failed to get machine: %v", err) } defer machine.CleanUp() - - benchmarks := make([]benchmark, 0, 6) - for _, filesys := range []harness.FileSystemType{harness.BindFS, harness.TmpFS, harness.RootFS} { - benchmarks = append(benchmarks, benchmark{ - clearCache: true, - fstype: filesys, - }) - benchmarks = append(benchmarks, benchmark{ - clearCache: false, - fstype: filesys, - }) - } - - for _, bm := range benchmarks { - pageCache := tools.Parameter{ - Name: "page_cache", - Value: "dirty", - } - if bm.clearCache { - pageCache.Value = "clean" - } - - filesystem := tools.Parameter{ - Name: "filesystem", - Value: string(bm.fstype), - } - name, err := tools.ParametersToName(pageCache, filesystem) - if err != nil { - b.Fatalf("Failed to parse parameters: %v", err) - } - - b.Run(name, func(b *testing.B) { - // Grab a container. - ctx := context.Background() - container := machine.GetContainer(ctx, b) - cu := cleanup.Make(func() { - container.CleanUp(ctx) - }) - defer cu.Clean() - mts, prefix, err := harness.MakeMount(machine, bm.fstype, &cu) - if err != nil { - b.Fatalf("Failed to make mount: %v", err) - } - - runOpts := dockerutil.RunOpts{ - Image: image, - Mounts: mts, - } - - // Start a container and sleep. - if err := container.Spawn(ctx, runOpts, "sleep", fmt.Sprintf("%d", 1000000)); err != nil { - b.Fatalf("run failed with: %v", err) - } - - cpCmd := fmt.Sprintf("mkdir -p %s && cp -r %s %s/.", prefix, workdir, prefix) - if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, - "/bin/sh", "-c", cpCmd); err != nil { - b.Fatalf("failed to copy directory: %v (%s)", err, out) - } - - b.ResetTimer() - b.StopTimer() - - // Drop Caches and bazel clean should happen inside the loop as we may use - // time options with b.N. (e.g. Run for an hour.) - for i := 0; i < b.N; i++ { - // Drop Caches for clear cache runs. - if bm.clearCache { - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches: %v. You probably need root.", err) - } - } - - b.StartTimer() - got, err := container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: prefix + workdir, - }, "bazel", "build", "-c", "opt", target) - if err != nil { - b.Fatalf("build failed with: %v logs: %s", err, got) - } - b.StopTimer() - - want := "Build completed successfully" - if !strings.Contains(got, want) { - b.Fatalf("string %s not in: %s", want, got) - } - - // Clean bazel in the case we are doing another run. - if i < b.N-1 { - if _, err = container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: prefix + workdir, - }, "bazel", "clean"); err != nil { - b.Fatalf("build failed with: %v", err) - } - } - } - }) - } + fsbench.RunWithDifferentFilesystems(ctx, b, machine, fsbench.FSBenchmark{ + Image: image, + WorkDir: workDir, + RunCmd: []string{"bazel", "build", "-c", "opt", target}, + WantOutput: "Build completed successfully", + CleanCmd: []string{"blaze", "clean"}, + }) } // TestMain is the main method for package fs. diff --git a/test/benchmarks/fs/fsbench/BUILD b/test/benchmarks/fs/fsbench/BUILD new file mode 100644 index 000000000..eeab4aa50 --- /dev/null +++ b/test/benchmarks/fs/fsbench/BUILD @@ -0,0 +1,20 @@ +# Package fsbench provides utility functions for filesystem-related benchmarks. + +load("//tools:defs.bzl", "go_library") + +package( + default_visibility = ["//test/benchmarks/fs:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "fsbench", + testonly = 1, + srcs = ["fsbench.go"], + deps = [ + "//pkg/cleanup", + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + ], +) diff --git a/test/benchmarks/fs/fsbench/fsbench.go b/test/benchmarks/fs/fsbench/fsbench.go new file mode 100644 index 000000000..f468eba76 --- /dev/null +++ b/test/benchmarks/fs/fsbench/fsbench.go @@ -0,0 +1,167 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fsbench provides utility functions for filesystem benchmarks. +package fsbench + +import ( + "context" + "fmt" + "strings" + "testing" + + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" +) + +// FSBenchmark represents a set of work to perform within a container that is instrumented with +// different filesystem configurations. +type FSBenchmark struct { + // Image is the Docker image to load. + Image string + // WorkDir is where the action takes place. + // The commands below are run from a directory that has the same file as what the container image + // has at this directory. + WorkDir string + // RunCmd is the command to run to execute the benchmark. + RunCmd []string + // WantOutput, if set, is verified to be a substring of the output of RunCmd. + WantOutput string + // CleanCmd, if set, is run to clean up between benchmarks. + CleanCmd []string + // Variants is a list of benchmarka variants to run. + // If unset, the typical set is used. + Variants []Variant +} + +// Variant is a specific configuration for a benchmark. +// Dimensions here are clean/dirty cache (do or don't drop caches) +// and if the mount on which we are compiling is a tmpfs/bind mount. +type Variant struct { + // clearCache drops caches before running. + clearCache bool + // fsType is the type of filesystem to use. + fsType harness.FileSystemType +} + +// TypicalVariants returns the typical full set of benchmark variants. +func TypicalVariants() []Variant { + variants := make([]Variant, 0, 6) + for _, filesys := range []harness.FileSystemType{harness.BindFS, harness.TmpFS, harness.RootFS} { + variants = append(variants, Variant{ + clearCache: true, + fsType: filesys, + }) + variants = append(variants, Variant{ + clearCache: false, + fsType: filesys, + }) + } + return variants +} + +// RunWithDifferentFilesystems runs a +func RunWithDifferentFilesystems(ctx context.Context, b *testing.B, machine harness.Machine, bm FSBenchmark) { + b.Helper() + + benchmarkVariants := bm.Variants + if len(benchmarkVariants) == 0 { + benchmarkVariants = TypicalVariants() + } + for _, variant := range benchmarkVariants { + pageCache := tools.Parameter{ + Name: "page_cache", + Value: "dirty", + } + if variant.clearCache { + pageCache.Value = "clean" + } + + filesystem := tools.Parameter{ + Name: "filesystem", + Value: string(variant.fsType), + } + name, err := tools.ParametersToName(pageCache, filesystem) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + + b.Run(name, func(b *testing.B) { + // Grab a container. + container := machine.GetContainer(ctx, b) + cu := cleanup.Make(func() { + container.CleanUp(ctx) + }) + defer cu.Clean() + mts, prefix, err := harness.MakeMount(machine, variant.fsType, &cu) + if err != nil { + b.Fatalf("Failed to make mount: %v", err) + } + + runOpts := dockerutil.RunOpts{ + Image: bm.Image, + Mounts: mts, + } + + // Start a container and sleep. + if err := container.Spawn(ctx, runOpts, "sleep", "24h"); err != nil { + b.Fatalf("run failed with: %v", err) + } + + cpCmd := fmt.Sprintf("mkdir -p %s && cp -r %s %s/.", prefix, bm.WorkDir, prefix) + if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, + "/bin/sh", "-c", cpCmd); err != nil { + b.Fatalf("failed to copy directory: %v (%s)", err, out) + } + + b.ResetTimer() + b.StopTimer() + + // Drop Caches and bazel clean should happen inside the loop as we may use + // time options with b.N. (e.g. Run for an hour.) + for i := 0; i < b.N; i++ { + // Drop Caches for clear cache runs. + if variant.clearCache { + if err := harness.DropCaches(machine); err != nil { + b.Skipf("failed to drop caches: %v. You probably need root.", err) + } + } + + b.StartTimer() + got, err := container.Exec(ctx, dockerutil.ExecOpts{ + WorkDir: prefix + bm.WorkDir, + }, bm.RunCmd...) + if err != nil { + b.Fatalf("Command %v failed with: %v logs: %s", bm.RunCmd, err, got) + } + b.StopTimer() + + if bm.WantOutput != "" && !strings.Contains(got, bm.WantOutput) { + b.Fatalf("string %s not in: %s", bm.WantOutput, got) + } + + // Clean the container in case we are doing another run. + if i < b.N-1 && len(bm.CleanCmd) != 0 { + if _, err = container.Exec(ctx, dockerutil.ExecOpts{ + WorkDir: prefix + bm.WorkDir, + }, bm.CleanCmd...); err != nil { + b.Fatalf("Cleanup command %v failed with: %v", bm.CleanCmd, err) + } + } + } + }) + } +} diff --git a/test/benchmarks/fs/rubydev_test.go b/test/benchmarks/fs/rubydev_test.go new file mode 100644 index 000000000..340540dff --- /dev/null +++ b/test/benchmarks/fs/rubydev_test.go @@ -0,0 +1,81 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rubydev_test benchmarks Ruby CI/CD-type workloads. +package rubydev_test + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "gvisor.dev/gvisor/test/benchmarks/fs/fsbench" + "gvisor.dev/gvisor/test/benchmarks/harness" +) + +func runRubyBenchmark(b *testing.B, bm fsbench.FSBenchmark, cleanupDirPatterns []string) { + b.Helper() + ctx := context.Background() + machine, err := harness.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer machine.CleanUp() + bm.Image = "benchmarks/rubydev" + cleanupDirPatterns = append(cleanupDirPatterns, "$HOME/.bundle/cache", "/tmp/rspec_failed_tests.txt") + bm.CleanCmd = []string{"bash", "-c", fmt.Sprintf("rm -rf %s", strings.Join(cleanupDirPatterns, " "))} + fsbench.RunWithDifferentFilesystems(ctx, b, machine, bm) +} + +// BenchmarkRubyNoOpTest runs a no-op Ruby test. +// This is the test case that Stripe used to benchmark gVisor: +// https://stripe.com/blog/fast-secure-builds-choose-two +func BenchmarkRubyNoOpTest(b *testing.B) { + runRubyBenchmark(b, fsbench.FSBenchmark{ + Image: "benchmarks/rubydev", + WorkDir: "/files", + RunCmd: []string{"ruby", "tc_no_op.rb"}, + WantOutput: "100% passed", + }, nil) +} + +// BenchmarkRubySpecTest runs a complex test suite from the Fastlane project: +// https://github.com/fastlane/fastlane +func BenchmarkRubySpecTest(b *testing.B) { + runRubyBenchmark(b, fsbench.FSBenchmark{ + Image: "benchmarks/rubydev", + WorkDir: "/fastlane", + RunCmd: []string{"bash", "/files/run_fastlane_tests.sh"}, + WantOutput: "3613 examples, 0 failures", + }, []string{ + // Fastlane tests pollute the filesystem a lot. + // To find out, run `find / -exec stat -c "%n %y" {} \; | sort` before and after running tests + // for the first time, and diff them. + "$HOME/Library", // Yes, even on Linux. + "$HOME/.fastlane", + "/tmp/fastlane*", + "/tmp/spaceship*", + "/tmp/profile_download*", + "/tmp/d*-*-*/*.mobileprovision", + }) +} + +// TestMain is the main method for this package. +func TestMain(m *testing.M) { + harness.Init() + harness.SetFixedBenchmarks() + os.Exit(m.Run()) +}