Add loadtime metric to BenchmarkRubySpecTest.

As of right now, BenchmarkRubySpecTest is not too useful because majority of
the reported time is consumed by the various tests running. We care about this
workload's start up time. It's output has the load time in the format
"files took X seconds to load". This metric is important for us. So capture it
and report it.

PiperOrigin-RevId: 521819612
This commit is contained in:
Ayush Ranjan
2023-04-04 11:33:11 -07:00
committed by gVisor bot
parent 33f7c6326e
commit 2c2bb55aad
2 changed files with 26 additions and 0 deletions
+8
View File
@@ -45,6 +45,10 @@ type FSBenchmark struct {
// Variants is a list of benchmarka variants to run.
// If unset, the typical set is used.
Variants []Variant
// Callback is an optional function that is called after each execution of
// the workload being benchmarked. It can be used to perform workload
// specific metric reporting.
Callback func(b *testing.B, output string)
}
// Variant is a specific configuration for a benchmark.
@@ -153,6 +157,10 @@ func RunWithDifferentFilesystems(ctx context.Context, b *testing.B, machine harn
b.Fatalf("string %s not in: %s", bm.WantOutput, got)
}
if bm.Callback != nil {
bm.Callback(b, got)
}
// Clean the container in case we are doing another run.
if i < b.N-1 && len(bm.CleanCmd) != 0 {
if _, err = container.Exec(ctx, dockerutil.ExecOpts{
+18
View File
@@ -19,6 +19,8 @@ import (
"context"
"fmt"
"os"
"regexp"
"strconv"
"strings"
"testing"
@@ -52,6 +54,14 @@ func BenchmarkRubyNoOpTest(b *testing.B) {
}, nil)
}
func extractLoadTime(output string) (float64, error) {
submatches := regexp.MustCompile(`files took (\d[\d,]*[\.]?[\d]*) seconds to load`).FindStringSubmatch(output)
if len(submatches) != 2 {
return 0, fmt.Errorf("count not find load time in output = %q", output)
}
return strconv.ParseFloat(submatches[1], 64)
}
// BenchmarkRubySpecTest runs a complex test suite from the Fastlane project:
// https://github.com/fastlane/fastlane
func BenchmarkRubySpecTest(b *testing.B) {
@@ -60,6 +70,14 @@ func BenchmarkRubySpecTest(b *testing.B) {
WorkDir: "/fastlane",
RunCmd: []string{"bash", "/files/run_fastlane_tests.sh"},
WantOutput: "3613 examples, 0 failures",
Callback: func(b *testing.B, output string) {
loadTime, err := extractLoadTime(output)
if err != nil {
b.Errorf("failed to extract load time from fastlane test suite output: %v", err)
return
}
b.ReportMetric(loadTime, "load-sec")
},
}, []string{
// Fastlane tests pollute the filesystem a lot.
// To find out, run `find / -exec stat -c "%n %y" {} \; | sort` before and after running tests