From 0c09a59188fdae7481c0b652d5a4c96a6bf85975 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Thu, 12 Jan 2023 14:41:50 -0800 Subject: [PATCH] Use named UDS to connect to the sandbox The abstract namespace requires access to the host network namespace which is not always available from containers, making it hard to write daemonsets that communicate with runsc sandboxes. Using a named UDS in the root directory makes it easier, by mounting the root dir (which is already required to load state files) into the container to allow acccess. PiperOrigin-RevId: 501671432 --- runsc/boot/controller.go | 5 ---- runsc/boot/loader_test.go | 2 +- runsc/container/container_test.go | 41 +++++++++++++++++++++++++++++++ runsc/sandbox/sandbox.go | 37 ++++++++++++++++++++++++---- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index a989cb858..1906a4851 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -145,11 +145,6 @@ const ( CgroupsWriteControlFiles = "Cgroups.WriteControlFiles" ) -// ControlSocketAddr generates an abstract unix socket name for the given ID. -func ControlSocketAddr(id string) string { - return fmt.Sprintf("\x00runsc-sandbox.%s", id) -} - // controller holds the control server, and is used for communication into the // sandbox. type controller struct { diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go index 4ef88dacd..3aec336f6 100644 --- a/runsc/boot/loader_test.go +++ b/runsc/boot/loader_test.go @@ -104,7 +104,7 @@ func startGofer(root string) (int, func(), error) { } func createLoader(spec *specs.Spec) (*Loader, func(), error) { - fd, err := server.CreateSocket(ControlSocketAddr(fmt.Sprintf("%010d", rand.Int())[:10])) + fd, err := server.CreateSocket(fmt.Sprintf("\x00loader-test.%010d", rand.Int())[:10]) if err != nil { return nil, nil, err } diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index aaa1f00a9..665e13cae 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -24,6 +24,7 @@ import ( "path" "path/filepath" "reflect" + "runtime" "strconv" "strings" "testing" @@ -2713,3 +2714,43 @@ func TestSaveSystemdCgroup(t *testing.T) { t.Errorf("CompatCgroup not properly saved: want %v, got %v", cont.CompatCgroup, loadCont.CompatCgroup) } } + +// TestSandboxCommunicationUnshare checks that communication with sandboxes do +// not require being in the same network namespace. This is required to allow +// Kubernetes daemonsets/containers to communicate with sandboxes without the +// need to join the host network namespaces. +func TestSandboxCommunicationUnshare(t *testing.T) { + spec, conf := sleepSpecConf(t) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + + cont, err := New(conf, args) + if err != nil { + t.Fatalf("Creating container: %v", err) + } + defer cont.Destroy() + + if err := cont.Start(conf); err != nil { + t.Fatalf("starting container: %v", err) + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if err := unix.Unshare(unix.CLONE_NEWNET); err != nil { + t.Fatalf("unix.Unshare(): %v", err) + } + + // Send a simple command to test that the sandbox can be reached. + if err := cont.SignalContainer(0, true); err != nil { + t.Errorf("SignalContainer(): %v", err) + } +} diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 6d2d2d335..de0c5b0a4 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -24,6 +24,7 @@ import ( "math" "os" "os/exec" + "path/filepath" "strconv" "strings" "syscall" @@ -54,6 +55,24 @@ import ( "gvisor.dev/gvisor/runsc/specutils" ) +// createControlSocket finds a location and creates the socket used to +// communicate with the sandbox. +func createControlSocket(rootDir, id string) (string, int, error) { + name := fmt.Sprintf("runsc-%s.sock", id) + + // Only use absolute paths to guarantee resolution from anywhere. + for _, dir := range []string{rootDir, "/var/run", "/run", "/tmp"} { + path := filepath.Join(dir, name) + log.Debugf("Attempting to create socket file %q", path) + fd, err := server.CreateSocket(path) + if err == nil { + log.Debugf("Using socket file %q", path) + return path, fd, nil + } + } + return "", -1, fmt.Errorf("unable to find location to write socket file") +} + // pid is an atomic type that implements JSON marshal/unmarshal interfaces. type pid struct { val atomicbitops.Int64 @@ -113,6 +132,8 @@ type Sandbox struct { // started, before it may be modified. OriginalOOMScoreAdj int `json:"originalOomScoreAdj"` + ControlAddress string `json:"control_address"` + // child is set if a sandbox process is a child of the current process. // // This field isn't saved to json, because only a creator of sandbox @@ -189,6 +210,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) { UID: -1, // prevent usage before it's set. GID: -1, // prevent usage before it's set. } + // The Cleanup object cleans up partially created sandboxes when an error // occurs. Any errors occurring during cleanup itself are ignored. c := cleanup.Make(func() { @@ -522,7 +544,7 @@ func (s *Sandbox) Event(cid string) (*boot.EventOut, error) { func (s *Sandbox) sandboxConnect() (*urpc.Client, error) { log.Debugf("Connecting to sandbox %q", s.ID) - conn, err := client.ConnectTo(boot.ControlSocketAddr(s.ID)) + conn, err := client.ConnectTo(s.ControlAddress) if err != nil { return nil, s.connError(err) } @@ -620,12 +642,12 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn } // Create a socket for the control server and donate it to the sandbox. - addr := boot.ControlSocketAddr(s.ID) - sockFD, err := server.CreateSocket(addr) - log.Infof("Creating sandbox process with addr: %s", addr[1:]) // skip "\00". + controlAddress, sockFD, err := createControlSocket(conf.RootDir, s.ID) if err != nil { - return fmt.Errorf("creating control server socket for sandbox %q: %v", s.ID, err) + return fmt.Errorf("creating control socket %q: %v", s.ControlAddress, err) } + log.Infof("Control socket: %q", s.ControlAddress) + s.ControlAddress = controlAddress donations.DonateAndClose("controller-fd", os.NewFile(uintptr(sockFD), "control_server_socket")) specFile, err := specutils.OpenSpec(args.BundleDir) @@ -1004,6 +1026,11 @@ func (s *Sandbox) IsRootContainer(cid string) bool { // is idempotent. func (s *Sandbox) destroy() error { log.Debugf("Destroy sandbox %q", s.ID) + if len(s.ControlAddress) != 0 { + if err := os.Remove(s.ControlAddress); err != nil { + log.Warningf("failed to delete control socket file %q: %v", s.ControlAddress, err) + } + } pid := s.Pid.load() if pid != 0 { log.Debugf("Killing sandbox %q", s.ID)