Allow runsc to generate coverage reports.

Add a coverage-report flag that will cause the sandbox to generate a coverage
report (with suffix .cov) in the debug log directory upon exiting. For the
report to be generated, runsc must have been built with the following Bazel
flags: `--collect_code_coverage --instrumentation_filter=...`.

With coverage reports, we should be able to aggregate results across all tests
to surface code coverage statistics for the project as a whole.

The report is simply a text file with each line representing a covered block
as `file:start_line.start_col,end_line.end_col`. Note that this is similar to
the format of coverage reports generated with `go test -coverprofile`,
although we omit the count and number of statements, which are not useful for
us.

Some simple ways of getting coverage reports:

bazel test <some_test> --collect_code_coverage \
  --instrumentation_filter=//pkg/...

bazel build //runsc --collect_code_coverage \
  --instrumentation_filter=//pkg/...
runsc -coverage-report=dir/ <other_flags> do ...

PiperOrigin-RevId: 368952911

pkg/coverage/coverage.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"sort"
+	"sync/atomic"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/hostarch"
@@ -34,12 +35,16 @@ import (
 	"github.com/bazelbuild/rules_go/go/tools/coverdata"
 )
 
-// coverageMu must be held while accessing coverdata.Cover. This prevents
-// concurrent reads/writes from multiple threads collecting coverage data.
-var coverageMu sync.RWMutex
+var (
+	// coverageMu must be held while accessing coverdata.Cover. This prevents
+	// concurrent reads/writes from multiple threads collecting coverage data.
+	coverageMu sync.RWMutex
 
-// once ensures that globalData is only initialized once.
-var once sync.Once
+	// reportOutput is the place to write out a coverage report. It should be
+	// closed after the report is written. It is protected by reportOutputMu.
+	reportOutput   io.WriteCloser
+	reportOutputMu sync.Mutex
+)
 
 // blockBitLength is the number of bits used to represent coverage block index
 // in a synthetic PC (the rest are used to represent the file index). Even
@@ -51,12 +56,26 @@ var once sync.Once
 // file and every block.
 const blockBitLength = 16
 
-// KcovAvailable returns whether the kcov coverage interface is available. It is
-// available as long as coverage is enabled for some files.
-func KcovAvailable() bool {
+// Available returns whether any coverage data is available.
+func Available() bool {
 	return len(coverdata.Cover.Blocks) > 0
 }
 
+// EnableReport sets up coverage reporting.
+func EnableReport(w io.WriteCloser) {
+	reportOutputMu.Lock()
+	defer reportOutputMu.Unlock()
+	reportOutput = w
+}
+
+// KcovSupported returns whether the kcov interface should be made available.
+//
+// If coverage reporting is on, do not turn on kcov, which will consume
+// coverage data.
+func KcovSupported() bool {
+	return (reportOutput == nil) && Available()
+}
+
 var globalData struct {
 	// files is the set of covered files sorted by filename. It is calculated at
 	// startup.
@@ -65,6 +84,9 @@ var globalData struct {
 	// syntheticPCs are a set of PCs calculated at startup, where the PC
 	// at syntheticPCs[i][j] corresponds to file i, block j.
 	syntheticPCs [][]uint64
+
+	// once ensures that globalData is only initialized once.
+	once sync.Once
 }
 
 // ClearCoverageData clears existing coverage data.
@@ -166,7 +188,7 @@ func ConsumeCoverageData(w io.Writer) int {
 // InitCoverageData initializes globalData. It should be called before any kcov
 // data is written.
 func InitCoverageData() {
-	once.Do(func() {
+	globalData.once.Do(func() {
 		// First, order all files. Then calculate synthetic PCs for every block
 		// (using the well-defined ordering for files as well).
 		for file := range coverdata.Cover.Blocks {
@@ -185,6 +207,38 @@ func InitCoverageData() {
 	})
 }
 
+// reportOnce ensures that a coverage report is written at most once. For a
+// complete coverage report, Report should be called during the sandbox teardown
+// process. Report is called from multiple places (which may overlap) so that a
+// coverage report is written in different sandbox exit scenarios.
+var reportOnce sync.Once
+
+// Report writes out a coverage report with all blocks that have been covered.
+//
+// TODO(b/144576401): Decide whether this should actually be in LCOV format
+func Report() error {
+	if reportOutput == nil {
+		return nil
+	}
+
+	var err error
+	reportOnce.Do(func() {
+		for file, counters := range coverdata.Cover.Counters {
+			blocks := coverdata.Cover.Blocks[file]
+			for i := 0; i < len(counters); i++ {
+				if atomic.LoadUint32(&counters[i]) > 0 {
+					err = writeBlock(reportOutput, file, blocks[i])
+					if err != nil {
+						return
+					}
+				}
+			}
+		}
+		reportOutput.Close()
+	})
+	return err
+}
+
 // Symbolize prints information about the block corresponding to pc.
 func Symbolize(out io.Writer, pc uint64) error {
 	fileNum, blockNum := syntheticPCToIndexes(pc)
@@ -196,18 +250,32 @@ func Symbolize(out io.Writer, pc uint64) error {
 	if err != nil {
 		return err
 	}
-	writeBlock(out, pc, file, block)
-	return nil
+	return writeBlockWithPC(out, pc, file, block)
 }
 
 // WriteAllBlocks prints all information about all blocks along with their
 // corresponding synthetic PCs.
-func WriteAllBlocks(out io.Writer) {
+func WriteAllBlocks(out io.Writer) error {
 	for fileNum, file := range globalData.files {
 		for blockNum, block := range coverdata.Cover.Blocks[file] {
-			writeBlock(out, calculateSyntheticPC(fileNum, blockNum), file, block)
+			if err := writeBlockWithPC(out, calculateSyntheticPC(fileNum, blockNum), file, block); err != nil {
+				return err
+			}
 		}
 	}
+	return nil
+}
+
+func writeBlockWithPC(out io.Writer, pc uint64, file string, block testing.CoverBlock) error {
+	if _, err := io.WriteString(out, fmt.Sprintf("%#x\n", pc)); err != nil {
+		return err
+	}
+	return writeBlock(out, file, block)
+}
+
+func writeBlock(out io.Writer, file string, block testing.CoverBlock) error {
+	_, err := io.WriteString(out, fmt.Sprintf("%s:%d.%d,%d.%d\n", file, block.Line0, block.Col0, block.Line1, block.Col1))
+	return err
 }
 
 func calculateSyntheticPC(fileNum int, blockNum int) uint64 {
@@ -239,8 +307,3 @@ func blockFromIndex(file string, i int) (testing.CoverBlock, error) {
 	}
 	return blocks[i], nil
 }
-
-func writeBlock(out io.Writer, pc uint64, file string, block testing.CoverBlock) {
-	io.WriteString(out, fmt.Sprintf("%#x\n", pc))
-	io.WriteString(out, fmt.Sprintf("%s:%d.%d,%d.%d\n", file, block.Line0, block.Col0, block.Line1, block.Col1))
-}

pkg/sentry/fsimpl/sys/sys.go

@@ -122,11 +122,11 @@ func cpuDir(ctx context.Context, fs *filesystem, creds *auth.Credentials) kernfs
 }
 
 func kernelDir(ctx context.Context, fs *filesystem, creds *auth.Credentials) kernfs.Inode {
-	// If kcov is available, set up /sys/kernel/debug/kcov. Technically, debugfs
-	// should be mounted at debug/, but for our purposes, it is sufficient to
-	// keep it in sys.
+	// Set up /sys/kernel/debug/kcov. Technically, debugfs should be
+	// mounted at debug/, but for our purposes, it is sufficient to keep it
+	// in sys.
 	var children map[string]kernfs.Inode
-	if coverage.KcovAvailable() {
+	if coverage.KcovSupported() {
 		log.Debugf("Set up /sys/kernel/debug/kcov")
 		children = map[string]kernfs.Inode{
 			"debug": fs.newDir(ctx, creds, linux.FileMode(0700), map[string]kernfs.Inode{

runsc/boot/BUILD

@@ -30,6 +30,7 @@ go_library(
         "//pkg/cleanup",
         "//pkg/context",
         "//pkg/control/server",
+        "//pkg/coverage",
         "//pkg/cpuid",
         "//pkg/eventchannel",
         "//pkg/fd",

runsc/boot/loader.go

@@ -29,6 +29,7 @@ import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/bpf"
 	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/coverage"
 	"gvisor.dev/gvisor/pkg/cpuid"
 	"gvisor.dev/gvisor/pkg/fd"
 	"gvisor.dev/gvisor/pkg/log"
@@ -1000,6 +1001,13 @@ func (l *Loader) waitContainer(cid string, waitStatus *uint32) error {
 	// consider the container exited.
 	ws := l.wait(tg)
 	*waitStatus = ws
+
+	// Write coverage report after the root container has exited. This guarantees
+	// that the report is written in cases where the sandbox is killed by a signal
+	// after the ContainerWait request is completed.
+	if l.root.procArgs.ContainerID == cid {
+		coverage.Report()
+	}
 	return nil
 }
 

runsc/cli/BUILD

@@ -10,6 +10,7 @@ go_library(
         "//runsc:__pkg__",
     ],
     deps = [
+        "//pkg/coverage",
         "//pkg/log",
         "//pkg/refs",
         "//pkg/sentry/platform",

runsc/cli/main.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/google/subcommands"
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/coverage"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/refs"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
@@ -50,6 +51,7 @@ var (
 	logFD      = flag.Int("log-fd", -1, "file descriptor to log to.  If set, the 'log' flag is ignored.")
 	debugLogFD = flag.Int("debug-log-fd", -1, "file descriptor to write debug logs to.  If set, the 'debug-log-dir' flag is ignored.")
 	panicLogFD = flag.Int("panic-log-fd", -1, "file descriptor to write Go's runtime messages.")
+	coverageFD = flag.Int("coverage-fd", -1, "file descriptor to write Go coverage output.")
 )
 
 // Main is the main entrypoint.
@@ -205,6 +207,10 @@ func Main(version string) {
 	} else if conf.AlsoLogToStderr {
 		e = &log.MultiEmitter{e, newEmitter(conf.DebugLogFormat, os.Stderr)}
 	}
+	if *coverageFD >= 0 {
+		f := os.NewFile(uintptr(*coverageFD), "coverage file")
+		coverage.EnableReport(f)
+	}
 
 	log.SetTarget(e)
 
@@ -234,6 +240,8 @@ func Main(version string) {
 	// Call the subcommand and pass in the configuration.
 	var ws unix.WaitStatus
 	subcmdCode := subcommands.Execute(context.Background(), conf, &ws)
+	// Write coverage report before os.Exit().
+	coverage.Report()
 	if subcmdCode == subcommands.ExitSuccess {
 		log.Infof("Exiting with status: %v", ws)
 		if ws.Signaled() {

runsc/cmd/symbolize.go

@@ -65,13 +65,15 @@ func (c *Symbolize) Execute(_ context.Context, f *flag.FlagSet, args ...interfac
 		f.Usage()
 		return subcommands.ExitUsageError
 	}
-	if !coverage.KcovAvailable() {
+	if !coverage.Available() {
 		return Errorf("symbolize can only be used when coverage is available.")
 	}
 	coverage.InitCoverageData()
 
 	if c.dumpAll {
-		coverage.WriteAllBlocks(os.Stdout)
+		if err := coverage.WriteAllBlocks(os.Stdout); err != nil {
+			return Errorf("Failed to write out blocks: %v", err)
+		}
 		return subcommands.ExitSuccess
 	}
 

runsc/config/config.go

@@ -55,6 +55,9 @@ type Config struct {
 	// PanicLog is the path to log GO's runtime messages, if not empty.
 	PanicLog string `flag:"panic-log"`
 
+	// CoverageReport is the path to write Go coverage information, if not empty.
+	CoverageReport string `flag:"coverage-report"`
+
 	// DebugLogFormat is the log format for debug.
 	DebugLogFormat string `flag:"debug-log-format"`
 

runsc/config/flags.go

@@ -44,7 +44,8 @@ func RegisterFlags() {
 
 		// Debugging flags.
 		flag.String("debug-log", "", "additional location for logs. If it ends with '/', log files are created inside the directory with default names. The following variables are available: %TIMESTAMP%, %COMMAND%.")
-		flag.String("panic-log", "", "file path were panic reports and other Go's runtime messages are written.")
+		flag.String("panic-log", "", "file path where panic reports and other Go's runtime messages are written.")
+		flag.String("coverage-report", "", "file path where Go coverage reports are written. Reports will only be generated if runsc is built with --collect_code_coverage and --instrumentation_filter Bazel flags.")
 		flag.Bool("log-packets", false, "enable network packet logging.")
 		flag.String("debug-log-format", "text", "log format: text (default), json, or json-k8s.")
 		flag.Bool("alsologtostderr", false, "send log messages to stderr.")

runsc/sandbox/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//pkg/cleanup",
         "//pkg/control/client",
         "//pkg/control/server",
+        "//pkg/coverage",
         "//pkg/log",
         "//pkg/sentry/control",
         "//pkg/sentry/platform",

runsc/sandbox/sandbox.go

@@ -34,6 +34,7 @@ import (
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/control/client"
 	"gvisor.dev/gvisor/pkg/control/server"
+	"gvisor.dev/gvisor/pkg/coverage"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/control"
 	"gvisor.dev/gvisor/pkg/sentry/platform"
@@ -399,15 +400,15 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 		cmd.Args = append(cmd.Args, "--log-fd="+strconv.Itoa(nextFD))
 		nextFD++
 	}
-	if conf.DebugLog != "" {
-		test := ""
-		if len(conf.TestOnlyTestNameEnv) != 0 {
-			// Fetch test name if one is provided and the test only flag was set.
-			if t, ok := specutils.EnvVar(args.Spec.Process.Env, conf.TestOnlyTestNameEnv); ok {
-				test = t
-			}
-		}
 
+	test := ""
+	if len(conf.TestOnlyTestNameEnv) != 0 {
+		// Fetch test name if one is provided and the test only flag was set.
+		if t, ok := specutils.EnvVar(args.Spec.Process.Env, conf.TestOnlyTestNameEnv); ok {
+			test = t
+		}
+	}
+	if conf.DebugLog != "" {
 		debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "boot", test)
 		if err != nil {
 			return fmt.Errorf("opening debug log file in %q: %v", conf.DebugLog, err)
@@ -418,23 +419,29 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 		nextFD++
 	}
 	if conf.PanicLog != "" {
-		test := ""
-		if len(conf.TestOnlyTestNameEnv) != 0 {
-			// Fetch test name if one is provided and the test only flag was set.
-			if t, ok := specutils.EnvVar(args.Spec.Process.Env, conf.TestOnlyTestNameEnv); ok {
-				test = t
-			}
-		}
-
 		panicLogFile, err := specutils.DebugLogFile(conf.PanicLog, "panic", test)
 		if err != nil {
-			return fmt.Errorf("opening debug log file in %q: %v", conf.PanicLog, err)
+			return fmt.Errorf("opening panic log file in %q: %v", conf.PanicLog, err)
 		}
 		defer panicLogFile.Close()
 		cmd.ExtraFiles = append(cmd.ExtraFiles, panicLogFile)
 		cmd.Args = append(cmd.Args, "--panic-log-fd="+strconv.Itoa(nextFD))
 		nextFD++
 	}
+	covFilename := conf.CoverageReport
+	if covFilename == "" {
+		covFilename = os.Getenv("GO_COVERAGE_FILE")
+	}
+	if covFilename != "" && coverage.Available() {
+		covFile, err := specutils.DebugLogFile(covFilename, "cov", test)
+		if err != nil {
+			return fmt.Errorf("opening debug log file in %q: %v", covFilename, err)
+		}
+		defer covFile.Close()
+		cmd.ExtraFiles = append(cmd.ExtraFiles, covFile)
+		cmd.Args = append(cmd.Args, "--coverage-fd="+strconv.Itoa(nextFD))
+		nextFD++
+	}
 
 	// Add the "boot" command to the args.
 	//

test/runner/runner.go

@@ -252,6 +252,7 @@ func runRunsc(spec *specs.Spec) error {
 		debugLogDir += "/"
 		log.Infof("runsc logs: %s", debugLogDir)
 		args = append(args, "-debug-log", debugLogDir)
+		args = append(args, "-coverage-report", debugLogDir)
 
 		// Default -log sends messages to stderr which makes reading the test log
 		// difficult. Instead, drop them when debug log is enabled given it's a