Add 'runsc debug' command

It prints sandbox stacks to the log to help debug stuckness. I expect
that many more options will be added in the future.

PiperOrigin-RevId: 201405931
Change-Id: I87e560800cd5a5a7b210dc25a5661363c8c3a16e

pkg/log/log.go

@@ -251,8 +251,8 @@ const defaultStackSize = 1 << 16 // 64KB
 // maxStackSize is the maximum buffer size to allocate for stack traces.
 const maxStackSize = 1 << 26 // 64MB
 
-// stacks returns goroutine stacks, like panic.
-func stacks(all bool) []byte {
+// Stacks returns goroutine stacks, like panic.
+func Stacks(all bool) []byte {
 	var trace []byte
 	for s := defaultStackSize; s <= maxStackSize; s *= 4 {
 		trace = make([]byte, s)
@@ -271,7 +271,7 @@ func stacks(all bool) []byte {
 //
 // This will be print a traceback, tb, as Warningf(format+":\n%s", v..., tb).
 func Traceback(format string, v ...interface{}) {
-	v = append(v, stacks(false))
+	v = append(v, Stacks(false))
 	Warningf(format+":\n%s", v...)
 }
 
@@ -279,7 +279,7 @@ func Traceback(format string, v ...interface{}) {
 //
 // This will be print a traceback, tb, as Warningf(format+":\n%s", v..., tb).
 func TracebackAll(format string, v ...interface{}) {
-	v = append(v, stacks(true))
+	v = append(v, Stacks(true))
 	Warningf(format+":\n%s", v...)
 }
 

runsc/boot/BUILD

@@ -7,6 +7,7 @@ go_library(
     srcs = [
         "config.go",
         "controller.go",
+        "debug.go",
         "events.go",
         "fds.go",
         "fs.go",

runsc/boot/controller.go

@@ -68,6 +68,9 @@ const (
 	// RootContainerStart is the URPC endpoint for starting a new sandbox
 	// with root container.
 	RootContainerStart = "containerManager.StartRoot"
+
+	// SandboxStacks collects sandbox stacks for debugging.
+	SandboxStacks = "debug.Stacks"
 )
 
 // ControlSocketAddr generates an abstract unix socket name for the given id.
@@ -107,6 +110,8 @@ func newController(fd int, k *kernel.Kernel, w *watchdog.Watchdog) (*controller,
 		srv.Register(net)
 	}
 
+	srv.Register(&debug{})
+
 	if err := srv.StartServing(); err != nil {
 		return nil, err
 	}

runsc/boot/debug.go

@@ -0,0 +1,29 @@
+// Copyright 2018 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boot
+
+import (
+	"gvisor.googlesource.com/gvisor/pkg/log"
+)
+
+type debug struct {
+}
+
+// Stacks collects all sandbox stacks and copies them to 'stacks'.
+func (*debug) Stacks(_ *struct{}, stacks *string) error {
+	buf := log.Stacks(true)
+	*stacks = string(buf)
+	return nil
+}

runsc/cmd/BUILD

@@ -10,6 +10,7 @@ go_library(
         "checkpoint.go",
         "cmd.go",
         "create.go",
+        "debug.go",
         "delete.go",
         "events.go",
         "exec.go",

runsc/cmd/debug.go

@@ -0,0 +1,108 @@
+// Copyright 2018 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"flag"
+	"github.com/google/subcommands"
+	"gvisor.googlesource.com/gvisor/pkg/log"
+	"gvisor.googlesource.com/gvisor/runsc/boot"
+	"gvisor.googlesource.com/gvisor/runsc/container"
+)
+
+// Debug implements subcommands.Command for the "debug" command.
+type Debug struct {
+	pid    int
+	stacks bool
+}
+
+// Name implements subcommands.Command.
+func (*Debug) Name() string {
+	return "debug"
+}
+
+// Synopsis implements subcommands.Command.
+func (*Debug) Synopsis() string {
+	return "shows a variety of debug information"
+}
+
+// Usage implements subcommands.Command.
+func (*Debug) Usage() string {
+	return `debug [flags] <container id>`
+}
+
+// SetFlags implements subcommands.Command.
+func (d *Debug) SetFlags(f *flag.FlagSet) {
+	f.IntVar(&d.pid, "pid", 0, "sandbox process ID. Container ID is not necessary if this is set")
+	f.BoolVar(&d.stacks, "stacks", false, "if true, dumps all sandbox stacks to the log")
+}
+
+// Execute implements subcommands.Command.Execute.
+func (d *Debug) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
+	var c *container.Container
+	conf := args[0].(*boot.Config)
+
+	if d.pid == 0 {
+		// No pid, container ID must have been provided.
+		if f.NArg() != 1 {
+			f.Usage()
+			return subcommands.ExitUsageError
+		}
+		var err error
+		c, err = container.Load(conf.RootDir, f.Arg(0))
+		if err != nil {
+			Fatalf("error loading container %q: %v", f.Arg(0), err)
+		}
+	} else {
+		if f.NArg() != 0 {
+			f.Usage()
+			return subcommands.ExitUsageError
+		}
+		// Go over all sandboxes and find the one that matches PID.
+		ids, err := container.List(conf.RootDir)
+		if err != nil {
+			Fatalf("error listing containers: %v", err)
+		}
+		for _, id := range ids {
+			candidate, err := container.Load(conf.RootDir, id)
+			if err != nil {
+				Fatalf("error loading container %q: %v", id, err)
+			}
+			if candidate.Pid() == d.pid {
+				c = candidate
+				break
+			}
+		}
+		if c == nil {
+			Fatalf("container with PID %d not found", d.pid)
+		}
+	}
+
+	log.Infof("Found sandbox %q, PID: %d", c.Sandbox.ID, c.Sandbox.Pid)
+	if !c.Sandbox.IsRunning() {
+		Fatalf("sandbox %q is not running", c.Sandbox.ID)
+	}
+
+	if d.stacks {
+		log.Infof("Retrieving sandbox stacks")
+		stacks, err := c.Sandbox.Stacks()
+		if err != nil {
+			Fatalf("error retrieving stacks: %v", err)
+		}
+		log.Infof("     *** Stack dump ***\n%s", stacks)
+	}
+	return subcommands.ExitSuccess
+}

runsc/main.go

@@ -88,6 +88,7 @@ func main() {
 	// The string below will be printed above the commands.
 	const internalGroup = "internal use only"
 	subcommands.Register(new(cmd.Boot), internalGroup)
+	subcommands.Register(new(cmd.Debug), internalGroup)
 	subcommands.Register(new(cmd.Gofer), internalGroup)
 
 	// All subcommands must be registered before flag parsing.

runsc/sandbox/sandbox.go

@@ -537,6 +537,22 @@ func (s *Sandbox) IsRunning() bool {
 	return false
 }
 
+// Stacks collects and returns all stacks for the sandbox.
+func (s *Sandbox) Stacks() (string, error) {
+	log.Debugf("Stacks sandbox %q", s.ID)
+	conn, err := s.connect()
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+
+	var stacks string
+	if err := conn.Call(boot.SandboxStacks, nil, &stacks); err != nil {
+		return "", fmt.Errorf("err getting sandbox %q stacks: %v", s.ID, err)
+	}
+	return stacks, nil
+}
+
 // killProcess sends a signal to the host process (i.e. a sandbox or gofer
 // process). Sandbox.Signal should be used to send a signal to a process
 // running inside the sandbox.