From 2c2bb55aad2c8fa7178c3839b4a7fab158b8424c Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Tue, 4 Apr 2023 11:30:24 -0700
Subject: [PATCH] Add loadtime metric to BenchmarkRubySpecTest.

As of right now, BenchmarkRubySpecTest is not too useful because majority of
the reported time is consumed by the various tests running. We care about this
workload's start up time. It's output has the load time in the format
"files took X seconds to load". This metric is important for us. So capture it
and report it.

PiperOrigin-RevId: 521819612
---
 test/benchmarks/fs/fsbench/fsbench.go |  8 ++++++++
 test/benchmarks/fs/rubydev_test.go    | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/test/benchmarks/fs/fsbench/fsbench.go b/test/benchmarks/fs/fsbench/fsbench.go
index f468eba76..55f7ee88a 100644
--- a/test/benchmarks/fs/fsbench/fsbench.go
+++ b/test/benchmarks/fs/fsbench/fsbench.go
@@ -45,6 +45,10 @@ type FSBenchmark struct {
 	// Variants is a list of benchmarka variants to run.
 	// If unset, the typical set is used.
 	Variants []Variant
+	// Callback is an optional function that is called after each execution of
+	// the workload being benchmarked. It can be used to perform workload
+	// specific metric reporting.
+	Callback func(b *testing.B, output string)
 }
 
 // Variant is a specific configuration for a benchmark.
@@ -153,6 +157,10 @@ func RunWithDifferentFilesystems(ctx context.Context, b *testing.B, machine harn
 					b.Fatalf("string %s not in: %s", bm.WantOutput, got)
 				}
 
+				if bm.Callback != nil {
+					bm.Callback(b, got)
+				}
+
 				// Clean the container in case we are doing another run.
 				if i < b.N-1 && len(bm.CleanCmd) != 0 {
 					if _, err = container.Exec(ctx, dockerutil.ExecOpts{
diff --git a/test/benchmarks/fs/rubydev_test.go b/test/benchmarks/fs/rubydev_test.go
index 340540dff..e0c376cf9 100644
--- a/test/benchmarks/fs/rubydev_test.go
+++ b/test/benchmarks/fs/rubydev_test.go
@@ -19,6 +19,8 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"regexp"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -52,6 +54,14 @@ func BenchmarkRubyNoOpTest(b *testing.B) {
 	}, nil)
 }
 
+func extractLoadTime(output string) (float64, error) {
+	submatches := regexp.MustCompile(`files took (\d[\d,]*[\.]?[\d]*) seconds to load`).FindStringSubmatch(output)
+	if len(submatches) != 2 {
+		return 0, fmt.Errorf("count not find load time in output = %q", output)
+	}
+	return strconv.ParseFloat(submatches[1], 64)
+}
+
 // BenchmarkRubySpecTest runs a complex test suite from the Fastlane project:
 // https://github.com/fastlane/fastlane
 func BenchmarkRubySpecTest(b *testing.B) {
@@ -60,6 +70,14 @@ func BenchmarkRubySpecTest(b *testing.B) {
 		WorkDir:    "/fastlane",
 		RunCmd:     []string{"bash", "/files/run_fastlane_tests.sh"},
 		WantOutput: "3613 examples, 0 failures",
+		Callback: func(b *testing.B, output string) {
+			loadTime, err := extractLoadTime(output)
+			if err != nil {
+				b.Errorf("failed to extract load time from fastlane test suite output: %v", err)
+				return
+			}
+			b.ReportMetric(loadTime, "load-sec")
+		},
 	}, []string{
 		// Fastlane tests pollute the filesystem a lot.
 		// To find out, run `find / -exec stat  -c "%n %y" {} \; | sort` before and after running tests