From 127262d21a210e6848cc600a5dcb5b4252f2753e Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Tue, 2 Jan 2024 13:33:57 -0800 Subject: [PATCH] Add annotation to send `runsc` debug logs to user logs. On Kubernetes, these are logged at the pod level. This makes it convenient to debug gVisor on Kubernetes clusters where SSH access to nodes is difficult or prohibited by policy. With this and other pod annotations, it is possible to do strace debugging with only pod annotations and no SSH. PiperOrigin-RevId: 595197543 --- runsc/cli/main.go | 55 +++++++++++++++++++++++++++++++++++++----- runsc/config/config.go | 8 ++++++ runsc/config/flags.go | 12 +++++---- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/runsc/cli/main.go b/runsc/cli/main.go index 9d9df0448..63f1e76ed 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -23,6 +23,8 @@ import ( "os" "os/signal" "runtime" + "strconv" + "strings" "time" "github.com/google/subcommands" @@ -133,23 +135,23 @@ func Main() { // case that does not occur. _ = time.Local.String() - var e log.Emitter + var emitters log.MultiEmitter if *debugLogFD > -1 { f := os.NewFile(uintptr(*debugLogFD), "debug log file") - e = newEmitter(conf.DebugLogFormat, f) + emitters = append(emitters, newEmitter(conf.DebugLogFormat, f)) } else if len(conf.DebugLog) > 0 && specutils.IsDebugCommand(conf, subcommand) { f, err := specutils.DebugLogFile(conf.DebugLog, subcommand, "" /* name */) if err != nil { util.Fatalf("error opening debug log file in %q: %v", conf.DebugLog, err) } - e = newEmitter(conf.DebugLogFormat, f) + emitters = append(emitters, newEmitter(conf.DebugLogFormat, f)) } else { // Stderr is reserved for the application, just discard the logs if no debug // log is specified. - e = newEmitter("text", ioutil.Discard) + emitters = append(emitters, newEmitter("text", ioutil.Discard)) } if *panicLogFD > -1 || *debugLogFD > -1 { @@ -171,7 +173,10 @@ func Main() { util.Fatalf("error dup'ing fd %d to stderr: %v", fd, err) } } else if conf.AlsoLogToStderr { - e = &log.MultiEmitter{e, newEmitter(conf.DebugLogFormat, os.Stderr)} + emitters = append(emitters, newEmitter(conf.DebugLogFormat, os.Stderr)) + } + if ulEmittter, add := userLogEmitter(conf, subcommand); add { + emitters = append(emitters, ulEmittter) } if *coverageFD >= 0 { f := os.NewFile(uintptr(*coverageFD), "coverage file") @@ -184,7 +189,16 @@ func Main() { } } - log.SetTarget(e) + switch len(emitters) { + case 0: + // Do nothing. + case 1: + // Use the singular emitter to avoid needless + // `for` loop overhead when logging to a single place. + log.SetTarget(emitters[0]) + default: + log.SetTarget(&emitters) + } log.Infof("***************************") log.Infof("Args: %s", os.Args) @@ -303,3 +317,32 @@ func newEmitter(format string, logFile io.Writer) log.Emitter { util.Fatalf("invalid log format %q, must be 'text', 'json', or 'json-k8s'", format) panic("unreachable") } + +// userLogEmitter returns an emitter to add logs to user logs if requested. +func userLogEmitter(conf *config.Config, subcommand string) (log.Emitter, bool) { + if subcommand != "boot" || !conf.DebugToUserLog { + return nil, false + } + // We need to manually scan for `--user-log-fd` since it is a flag of the + // `boot` subcommand. We know it is in `--user-log-fd=FD` format because + // we control how arguments to `runsc boot` are formatted. + const userLogFDFlagPrefix = "--user-log-fd=" + var userLog *os.File + for _, arg := range os.Args[1:] { + if !strings.HasPrefix(arg, userLogFDFlagPrefix) { + continue + } + if userLog != nil { + util.Fatalf("duplicate %q flag", userLogFDFlagPrefix) + } + userLogFD, err := strconv.Atoi(arg[len(userLogFDFlagPrefix):]) + if err != nil { + util.Fatalf("invalid user log FD flag %q: %v", arg, err) + } + userLog = os.NewFile(uintptr(userLogFD), "user log file") + } + if userLog == nil { + return nil, false + } + return log.K8sJSONEmitter{&log.Writer{Next: userLog}}, true +} diff --git a/runsc/config/config.go b/runsc/config/config.go index 910db74c6..a47d63f21 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -58,8 +58,16 @@ type Config struct { LogFormat string `flag:"log-format"` // DebugLog is the path to log debug information to, if not empty. + // If specified together with `DebugToUserLog`, debug logs are emitted + // to both. DebugLog string `flag:"debug-log"` + // DebugToUserLog indicates that Sentry debug logs should be emitted + // to user-visible logs. + // If specified together with `DebugLog`, debug logs are emitted + // to both. + DebugToUserLog bool `flag:"debug-to-user-log"` + // DebugCommand is a comma-separated list of commands to be debugged if // --debug-log is also set. Empty means debug all. "!" negates the expression. // E.g. "create,start" or "!boot,events". diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 0cbcdfce5..f87888d53 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -52,6 +52,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("log-packets", false, "enable network packet logging.") flagSet.String("pcap-log", "", "location of PCAP log file.") flagSet.String("debug-log-format", "text", "log format: text (default), json, or json-k8s.") + flagSet.Bool("debug-to-user-log", false, "also emit Sentry logs to user-visible logs") // Only register -alsologtostderr flag if it is not already defined on this flagSet. if flagSet.Lookup("alsologtostderr") == nil { flagSet.Bool("alsologtostderr", false, "send log messages to stderr.") @@ -144,11 +145,12 @@ func RegisterFlags(flagSet *flag.FlagSet) { var overrideAllowlist = map[string]struct { check func(name string, value string) error }{ - "debug": {}, - "strace": {}, - "strace-syscalls": {}, - "strace-log-size": {}, - "host-uds": {}, + "debug": {}, + "debug-to-user-log": {}, + "strace": {}, + "strace-syscalls": {}, + "strace-log-size": {}, + "host-uds": {}, "oci-seccomp": {check: checkOciSeccomp}, }