Push signal-delivery and wait into the sandbox.

This is another step towards multi-container support.

Previously, we delivered signals directly to the sandbox process (which then
forwarded the signal to PID 1 inside the sandbox). Similarly, we waited on a
container by waiting on the sandbox process itself. This approach will not work
when there are multiple containers inside the sandbox, and we need to
signal/wait on individual containers.

This CL adds two new messages, ContainerSignal and ContainerWait. These
messages include the id of the container to signal/wait. The controller inside
the sandbox receives these messages and signals/waits on the appropriate
process inside the sandbox.

The container id is plumbed into the sandbox, but it currently is not used. We
still end up signaling/waiting on PID 1 in all cases.  Once we actually have
multiple containers inside the sandbox, we will need to keep some sort of map
of container id -> pid (or possibly pid namespace), and signal/kill the
appropriate process for the container.

PiperOrigin-RevId: 197028366
Change-Id: I07b4d5dc91ecd2affc1447e6b4bdd6b0b7360895
This commit is contained in:
Nicolas Lacasse
2018-05-17 11:55:28 -07:00
committed by Shentubot
parent 8e1deb2ab8
commit 31386185fe
9 changed files with 204 additions and 131 deletions
+1
View File
@@ -25,6 +25,7 @@ go_library(
"//pkg/control/server",
"//pkg/cpuid",
"//pkg/log",
"//pkg/sentry/arch",
"//pkg/sentry/context",
"//pkg/sentry/control",
"//pkg/sentry/fs",
+80 -36
View File
@@ -18,30 +18,39 @@ import (
"fmt"
"gvisor.googlesource.com/gvisor/pkg/control/server"
"gvisor.googlesource.com/gvisor/pkg/sentry/arch"
"gvisor.googlesource.com/gvisor/pkg/sentry/control"
"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
"gvisor.googlesource.com/gvisor/pkg/sentry/socket/epsocket"
)
const (
// ApplicationStart is the URPC endpoint for starting a sandboxed app.
ApplicationStart = "application.Start"
// ApplicationProcesses is the URPC endpoint for getting the list of
// processes running in a sandbox.
ApplicationProcesses = "application.Processes"
// ApplicationExecute is the URPC endpoint for executing a command in a
// sandbox.
ApplicationExecute = "application.Execute"
// ApplicationEvent is the URPC endpoint for getting stats about the
// ContainerEvent is the URPC endpoint for getting stats about the
// container used by "runsc events".
ApplicationEvent = "application.Event"
ContainerEvent = "containerManager.Event"
// ContainerExecute is the URPC endpoint for executing a command in a
// container..
ContainerExecute = "containerManager.Execute"
// ContainerProcesses is the URPC endpoint for getting the list of
// processes running in a container.
ContainerProcesses = "containerManager.Processes"
// ContainerSignal is used to send a signal to a container.
ContainerSignal = "containerManager.Signal"
// ContainerWait is used to wait on the init process of the container
// and return its ExitStatus.
ContainerWait = "containerManager.Wait"
// NetworkCreateLinksAndRoutes is the URPC endpoint for creating links
// and routes in a network stack.
NetworkCreateLinksAndRoutes = "Network.CreateLinksAndRoutes"
// RootContainerStart is the URPC endpoint for starting a new sandbox
// with root container.
RootContainerStart = "containerManager.StartRoot"
)
// ControlSocketAddr generates an abstract unix socket name for the given id.
@@ -55,8 +64,8 @@ type controller struct {
// srv is the contorl server.
srv *server.Server
// app holds the application methods.
app *application
// manager holds the containerManager methods.
manager *containerManager
}
// newController creates a new controller and starts it listening.
@@ -66,12 +75,12 @@ func newController(fd int, k *kernel.Kernel) (*controller, error) {
return nil, err
}
app := &application{
manager := &containerManager{
startChan: make(chan struct{}),
startResultChan: make(chan error),
k: k,
}
srv.Register(app)
srv.Register(manager)
if eps, ok := k.NetworkStack().(*epsocket.Stack); ok {
net := &Network{
@@ -85,44 +94,79 @@ func newController(fd int, k *kernel.Kernel) (*controller, error) {
}
return &controller{
srv: srv,
app: app,
srv: srv,
manager: manager,
}, nil
}
// application contains methods that control the sandboxed application.
type application struct {
// startChan is used to signal when the application process should be
// started.
// containerManager manages sandboes containers.
type containerManager struct {
// startChan is used to signal when the root container process should
// be started.
startChan chan struct{}
// startResultChan is used to signal when the application has started. Any
// errors encountered during startup will be sent to the channel. A nil value
// indicates success.
// startResultChan is used to signal when the root container has
// started. Any errors encountered during startup will be sent to the
// channel. A nil value indicates success.
startResultChan chan error
// k is the emulated linux kernel on which the sandboxed
// application runs.
// containers run.
k *kernel.Kernel
}
// Start will start the application process.
func (a *application) Start(_, _ *struct{}) error {
// Tell the application to start and wait for the result.
a.startChan <- struct{}{}
return <-a.startResultChan
// StartRoot will start the root container process.
func (cm *containerManager) StartRoot(_, _ *struct{}) error {
// Tell the root container to start and wait for the result.
cm.startChan <- struct{}{}
return <-cm.startResultChan
}
// Processes retrieves information about processes running in the sandbox.
func (a *application) Processes(_, out *[]*control.Process) error {
return control.Processes(a.k, out)
func (cm *containerManager) Processes(_, out *[]*control.Process) error {
return control.Processes(cm.k, out)
}
// Execute runs a command on a created or running sandbox.
func (a *application) Execute(e *control.ExecArgs, waitStatus *uint32) error {
proc := control.Proc{Kernel: a.k}
func (cm *containerManager) Execute(e *control.ExecArgs, waitStatus *uint32) error {
proc := control.Proc{Kernel: cm.k}
if err := proc.Exec(e, waitStatus); err != nil {
return fmt.Errorf("error executing: %+v: %v", e, err)
}
return nil
}
// Wait waits for the init process in the given container.
func (cm *containerManager) Wait(cid *string, waitStatus *uint32) error {
// TODO: Use the cid and wait on the init process in that
// container. Currently we just wait on PID 1 in the sandbox.
tg := cm.k.TaskSet().Root.ThreadGroupWithID(1)
if tg == nil {
return fmt.Errorf("cannot wait: no thread group with id 1")
}
tg.WaitExited()
*waitStatus = tg.ExitStatus().Status()
return nil
}
// SignalArgs are arguments to the Signal method.
type SignalArgs struct {
// CID is the container id.
CID string
// Signo is the signal to send to the process.
Signo int32
}
// Signal sends a signal to the init process of the container.
func (cm *containerManager) Signal(args *SignalArgs, _ *struct{}) error {
// TODO: Use the cid and send the signal to the init
// process in theat container. Currently we just signal PID 1 in the
// sandbox.
si := arch.SignalInfo{Signo: args.Signo}
t := cm.k.TaskSet().Root.TaskWithID(1)
if t == nil {
return fmt.Errorf("cannot signal: no task with id 1")
}
return t.SendSignal(&si)
}
+4 -3
View File
@@ -59,10 +59,11 @@ type Memory struct {
Raw map[string]uint64 `json:"raw,omitempty"`
}
func (a *application) Event(_ *struct{}, out *Event) error {
// Event gets the events from the container.
func (cm *containerManager) Event(_ *struct{}, out *Event) error {
stats := &Stats{}
stats.populateMemory(a.k)
stats.populatePIDs(a.k)
stats.populateMemory(cm.k)
stats.populatePIDs(cm.k)
*out = Event{Type: "stats", Data: stats}
return nil
}
+10 -11
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package boot loads the kernel and runs the application.
// Package boot loads the kernel and runs a container..
package boot
import (
@@ -57,7 +57,7 @@ import (
_ "gvisor.googlesource.com/gvisor/pkg/sentry/socket/unix"
)
// Loader keeps state needed to start the kernel and run the application.
// Loader keeps state needed to start the kernel and run the container..
type Loader struct {
// k is the kernel.
k *kernel.Kernel
@@ -73,10 +73,10 @@ type Loader struct {
watchdog *watchdog.Watchdog
// stopSignalForwarding disables forwarding of signals to the sandboxed
// app. It should be called when a sandbox is destroyed.
// container. It should be called when a sandbox is destroyed.
stopSignalForwarding func()
// procArgs refers to the initial application task.
// procArgs refers to the root container task.
procArgs kernel.CreateProcessArgs
}
@@ -283,10 +283,10 @@ func createPlatform(conf *Config) (platform.Platform, error) {
}
}
// Run runs the application.
// Run runs the root container..
func (l *Loader) Run() error {
err := l.run()
l.ctrl.app.startResultChan <- err
l.ctrl.manager.startResultChan <- err
if err != nil {
// Give the controller some time to send the error to the
// runtime. If we return too quickly here the process will exit
@@ -321,7 +321,7 @@ func (l *Loader) run() error {
}
}
// Create the initial application task.
// Create the root container init task.
if _, err := l.k.CreateProcess(l.procArgs); err != nil {
return fmt.Errorf("failed to create init process: %v", err)
}
@@ -335,13 +335,12 @@ func (l *Loader) run() error {
// WaitForStartSignal waits for a start signal from the control server.
func (l *Loader) WaitForStartSignal() {
<-l.ctrl.app.startChan
<-l.ctrl.manager.startChan
}
// WaitExit waits for the application to exit, and returns the application's
// exit status.
// WaitExit waits for the root container to exit, and returns its exit status.
func (l *Loader) WaitExit() kernel.ExitStatus {
// Wait for application.
// Wait for container.
l.k.WaitExited()
return l.k.GlobalInit().ExitStatus()
+7 -7
View File
@@ -72,13 +72,13 @@ func TestRun(t *testing.T) {
var wg sync.WaitGroup
wg.Add(1)
go func() {
resultChanErr = <-s.ctrl.app.startResultChan
resultChanErr = <-s.ctrl.manager.startResultChan
wg.Done()
}()
// Run the application.
// Run the container..
if err := s.Run(); err != nil {
t.Errorf("error running application: %v", err)
t.Errorf("error running container: %v", err)
}
// We should have not gotten an error on the startResultChan.
@@ -112,7 +112,7 @@ func TestStartSignal(t *testing.T) {
go func() {
s.WaitForStartSignal()
// Pretend that Run() executed and returned no error.
s.ctrl.app.startResultChan <- nil
s.ctrl.manager.startResultChan <- nil
waitFinished <- struct{}{}
}()
@@ -126,9 +126,9 @@ func TestStartSignal(t *testing.T) {
// OK.
}
// Trigger the control server Start method.
if err := s.ctrl.app.Start(nil, nil); err != nil {
t.Errorf("error calling Start: %v", err)
// Trigger the control server StartRoot method.
if err := s.ctrl.manager.StartRoot(nil, nil); err != nil {
t.Errorf("error calling StartRoot: %v", err)
}
// Now WaitForStartSignal should return (within a short amount of
+5 -5
View File
@@ -111,21 +111,21 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...interface{})
}
// Create the loader.
s, err := boot.New(spec, conf, b.controllerFD, b.ioFDs.GetArray(), b.console)
l, err := boot.New(spec, conf, b.controllerFD, b.ioFDs.GetArray(), b.console)
if err != nil {
Fatalf("error creating loader: %v", err)
}
defer s.Destroy()
defer l.Destroy()
// Wait for the start signal from runsc.
s.WaitForStartSignal()
l.WaitForStartSignal()
// Run the application and wait for it to finish.
if err := s.Run(); err != nil {
if err := l.Run(); err != nil {
Fatalf("error running sandbox: %v", err)
}
ws := s.WaitExit()
ws := l.WaitExit()
log.Infof("application exiting with %+v", ws)
*waitStatus = syscall.WaitStatus(ws.Status())
return subcommands.ExitSuccess
+7 -3
View File
@@ -120,9 +120,13 @@ func Load(rootDir, id string) (*Container, error) {
//
// This is inherently racey.
if c.Status == Running || c.Status == Created {
// Send signal 0 to check if container still exists.
if err := c.Signal(0); err != nil {
// Container no longer exists.
// Check if the sandbox process is still running.
if c.Sandbox.IsRunning() {
// TODO: Send a message into the sandbox to
// see if this particular container is still running.
} else {
// Sandbox no longer exists, so this container
// definitly does not exist.
c.Status = Stopped
c.Sandbox = nil
}
+39 -33
View File
@@ -20,10 +20,10 @@ import (
"io"
"io/ioutil"
"os"
"os/signal"
"path/filepath"
"reflect"
"strings"
"sync"
"syscall"
"testing"
"time"
@@ -75,9 +75,6 @@ func newSpecWithArgs(args ...string) *specs.Spec {
return spec
}
// shutdownSignal will be sent to the sandbox in order to shut down cleanly.
const shutdownSignal = syscall.SIGUSR2
// setupContainer creates a bundle and root dir for the container, generates a
// test config, and writes the spec to config.json in the bundle dir.
func setupContainer(spec *specs.Spec) (rootDir, bundleDir string, conf *boot.Config, err error) {
@@ -201,15 +198,35 @@ func TestLifecycle(t *testing.T) {
t.Error(err)
}
// Send the container a signal, which we catch and use to cleanly
// shutdown.
if err := s.Signal(shutdownSignal); err != nil {
t.Fatalf("error sending signal %v to container: %v", shutdownSignal, err)
// Wait on the container.
var wg sync.WaitGroup
wg.Add(1)
go func() {
ws, err := s.Wait()
if err != nil {
t.Errorf("error waiting on container: %v", err)
}
if got, want := ws.Signal(), syscall.SIGTERM; got != want {
t.Errorf("got signal %v, want %v", got, want)
}
wg.Done()
}()
// Send the container a SIGTERM which will cause it to stop.
if err := s.Signal(syscall.SIGTERM); err != nil {
t.Fatalf("error sending signal %v to container: %v", syscall.SIGTERM, err)
}
// Wait for it to die.
if _, err := s.Wait(); err != nil {
t.Fatalf("error waiting on container: %v", err)
}
wg.Wait()
// The sandbox process should have exited by now, but it is a zombie.
// In normal runsc usage, it will be parented to init, and init will
// reap the sandbox. However, in this case the test runner is the
// parent and will not reap the sandbox process, so we must do it
// ourselves.
p, _ := os.FindProcess(s.Sandbox.Pid)
p.Wait()
// Load the container from disk and check the status.
s, err = container.Load(rootDir, id)
if err != nil {
@@ -640,28 +657,17 @@ func TestMain(m *testing.M) {
subcommands.Register(new(cmd.Gofer), "gofer")
switch flag.Arg(0) {
case "boot", "gofer":
// Run the command in a goroutine so we can block the main
// thread waiting for shutdownSignal.
go func() {
conf := &boot.Config{
RootDir: "unused-root-dir",
Network: boot.NetworkNone,
}
var ws syscall.WaitStatus
subcmdCode := subcommands.Execute(context.Background(), conf, &ws)
if subcmdCode != subcommands.ExitSuccess {
panic(fmt.Sprintf("command failed to execute, err: %v", subcmdCode))
}
// Container exited normally. Shut down this process.
os.Exit(ws.ExitStatus())
}()
// Shutdown cleanly when the shutdownSignal is received. This
// allows us to write coverage data before exiting.
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, shutdownSignal)
<-sigc
exit(0)
conf := &boot.Config{
RootDir: "unused-root-dir",
Network: boot.NetworkNone,
}
var ws syscall.WaitStatus
subcmdCode := subcommands.Execute(context.Background(), conf, &ws)
if subcmdCode != subcommands.ExitSuccess {
panic(fmt.Sprintf("command failed to execute, err: %v", subcmdCode))
}
// Container exited. Shut down this process.
exit(ws.ExitStatus())
default:
// Otherwise run the tests.
exit(m.Run())
+51 -33
View File
@@ -95,13 +95,12 @@ func (s *Sandbox) Start(cid string, spec *specs.Spec, conf *boot.Config) error {
return fmt.Errorf("error setting up network: %v", err)
}
// Send a message to the sandbox control server to start the
// application.
// Send a message to the sandbox control server to start the root
// container..
//
// TODO: Pass in the container id (cid) here. The sandbox
// should start only that container.
if err := conn.Call(boot.ApplicationStart, nil, nil); err != nil {
return fmt.Errorf("error starting application %v: %v", spec.Process.Args, err)
// TODO: We need a way to start non-root containers.
if err := conn.Call(boot.RootContainerStart, nil, nil); err != nil {
return fmt.Errorf("error starting root container %v: %v", spec.Process.Args, err)
}
return nil
@@ -110,6 +109,7 @@ func (s *Sandbox) Start(cid string, spec *specs.Spec, conf *boot.Config) error {
// Processes retrieves the list of processes and associated metadata for a
// given container in this sandbox.
func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {
log.Debugf("Getting processes for container %q in sandbox %q", cid, s.ID)
conn, err := s.connect()
if err != nil {
return nil, err
@@ -119,7 +119,7 @@ func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {
var pl []*control.Process
// TODO: Pass in the container id (cid) here. The sandbox
// should return process info for only that container.
if err := conn.Call(boot.ApplicationProcesses, nil, &pl); err != nil {
if err := conn.Call(boot.ContainerProcesses, nil, &pl); err != nil {
return nil, fmt.Errorf("error retrieving process data from sandbox: %v", err)
}
return pl, nil
@@ -127,17 +127,18 @@ func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {
// Execute runs the specified command in the container.
func (s *Sandbox) Execute(cid string, e *control.ExecArgs) (syscall.WaitStatus, error) {
log.Debugf("Executing new process in container %q in sandbox %q", cid, s.ID)
conn, err := s.connect()
if err != nil {
return 0, fmt.Errorf("error connecting to control server at pid %d: %v", s.Pid, err)
}
defer conn.Close()
// Send a message to the sandbox control server to start the application.
// Send a message to the sandbox control server to start the container..
var waitStatus uint32
// TODO: Pass in the container id (cid) here. The sandbox
// should execute in the context of that container.
if err := conn.Call(boot.ApplicationExecute, e, &waitStatus); err != nil {
if err := conn.Call(boot.ContainerExecute, e, &waitStatus); err != nil {
return 0, fmt.Errorf("error executing in sandbox: %v", err)
}
@@ -146,6 +147,7 @@ func (s *Sandbox) Execute(cid string, e *control.ExecArgs) (syscall.WaitStatus,
// Event retrieves stats about the sandbox such as memory and CPU utilization.
func (s *Sandbox) Event(cid string) (*boot.Event, error) {
log.Debugf("Getting events for container %q in sandbox %q", cid, s.ID)
conn, err := s.connect()
if err != nil {
return nil, err
@@ -155,7 +157,7 @@ func (s *Sandbox) Event(cid string) (*boot.Event, error) {
var e boot.Event
// TODO: Pass in the container id (cid) here. The sandbox
// should return events only for that container.
if err := conn.Call(boot.ApplicationEvent, nil, &e); err != nil {
if err := conn.Call(boot.ContainerEvent, nil, &e); err != nil {
return nil, fmt.Errorf("error retrieving event data from sandbox: %v", err)
}
e.ID = cid
@@ -163,7 +165,7 @@ func (s *Sandbox) Event(cid string) (*boot.Event, error) {
}
func (s *Sandbox) connect() (*urpc.Client, error) {
log.Debugf("Connecting to sandbox...")
log.Debugf("Connecting to sandbox %q", s.ID)
conn, err := client.ConnectTo(boot.ControlSocketAddr(s.ID))
if err != nil {
return nil, fmt.Errorf("error connecting to control server at pid %d: %v", s.Pid, err)
@@ -380,20 +382,18 @@ func (s *Sandbox) waitForCreated(timeout time.Duration) error {
// Wait waits for the containerized process to exit, and returns its WaitStatus.
func (s *Sandbox) Wait(cid string) (syscall.WaitStatus, error) {
// TODO: This waits on the sandbox process. We need a way
// to wait on an individual container in the sandbox.
log.Debugf("Waiting for container %q in sandbox %q", cid, s.ID)
var ws syscall.WaitStatus
conn, err := s.connect()
if err != nil {
return ws, err
}
defer conn.Close()
p, err := os.FindProcess(s.Pid)
if err != nil {
// "On Unix systems, FindProcess always succeeds and returns a
// Process for the given pid."
panic(err)
if err := conn.Call(boot.ContainerWait, &cid, &ws); err != nil {
return ws, fmt.Errorf("err waiting on container %q: %v", cid, err)
}
ps, err := p.Wait()
if err != nil {
return 0, err
}
return ps.Sys().(syscall.WaitStatus), nil
return ws, nil
}
// Stop stops the container in the sandbox.
@@ -409,12 +409,12 @@ func (s *Sandbox) Destroy() error {
if s.Pid != 0 {
// TODO: Too harsh?
log.Debugf("Killing sandbox %q", s.ID)
sendSignal(s.Pid, unix.SIGKILL)
killProcess(s.Pid, unix.SIGKILL)
s.Pid = 0
}
if s.GoferPid != 0 {
log.Debugf("Killing gofer for sandbox %q", s.ID)
sendSignal(s.GoferPid, unix.SIGKILL)
killProcess(s.GoferPid, unix.SIGKILL)
s.GoferPid = 0
}
@@ -424,17 +424,35 @@ func (s *Sandbox) Destroy() error {
// Signal sends the signal to a container in the sandbox.
func (s *Sandbox) Signal(cid string, sig syscall.Signal) error {
log.Debugf("Signal sandbox %q", s.ID)
conn, err := s.connect()
if err != nil {
return err
}
defer conn.Close()
// TODO: This sends a signal to the sandbox process, which
// will be forwarded to the first process in the sandbox. We need a way
// to send a signal to any container in the sandbox.
// to wait on an individual container in the sandbox.
return sendSignal(s.Pid, sig)
args := boot.SignalArgs{
CID: cid,
Signo: int32(sig),
}
if err := conn.Call(boot.ContainerSignal, &args, nil); err != nil {
return fmt.Errorf("err signaling container %q: %v", cid, err)
}
return nil
}
// sendSignal sends a signal to the sandbox process.
func sendSignal(pid int, sig syscall.Signal) error {
// IsRunning returns true iff the sandbox process is running.
func (s *Sandbox) IsRunning() bool {
// Send a signal 0 to the sandbox process.
if err := killProcess(s.Pid, 0); err != nil {
return false
}
return true
}
// killProcess sends a signal to the host process (i.e. a sandbox or gofer
// process). Sandbox.Signal should be used to send a signal to a process
// running inside the sandbox.
func killProcess(pid int, sig syscall.Signal) error {
if err := syscall.Kill(pid, sig); err != nil {
return fmt.Errorf("error sending signal %d to pid %d: %v", sig, pid, err)
}