Add self-backed overlay for disk-backed EmptyDir volumes.

This change adds a new mount annotation:
dev.gvisor.spec.mount.{volumeName}.lifecycle: shared | pod | container

For now runsc shim will be setting this annotation for all EmptyDir volumes.
Runsc looks at this annotation can adds a self-backed overlay on top of
EmptyDir volumes that have type=bind and share=container.

runsc gives precedence to mount annotations over --overlay2 configuration.
Added a container test for this feature.

This change also updates MountHint.fileAccessType() to be more generic using
all the available information about a mount.

PiperOrigin-RevId: 530746837
This commit is contained in:
Ayush Ranjan
2023-05-09 16:43:39 -07:00
committed by gVisor bot
parent 1ef6ee2e46
commit 0ca9dbc41c
7 changed files with 264 additions and 63 deletions
+24 -1
View File
@@ -28,8 +28,15 @@ const (
// devshmName is the volume name used for /dev/shm. Pick a name that is
// unlikely to be used.
devshmName = "gvisorinternaldevshm"
// emptyDirVolumesDir is the directory inside kubeletPodsDir/{uid}/volumes/
// that hosts all the EmptyDir volumes used by the pod.
emptyDirVolumesDir = "kubernetes.io~empty-dir"
)
// The directory structure for volumes is as follows:
// /var/lib/kubelet/pods/{uid}/volumes/{type} where `uid` is the pod UID and
// `type` is the volume type.
var kubeletPodsDir = "/var/lib/kubelet/pods"
// volumeName gets volume name from volume annotation key, example:
@@ -73,6 +80,11 @@ func volumeSourceKey(volume string) string {
return volumeKeyPrefix + volume + ".source"
}
// volumeLifecycleKey constructs the annotation key for volume lifecycle.
func volumeLifecycleKey(volume string) string {
return volumeKeyPrefix + volume + ".lifecycle"
}
// volumePath searches the volume path in the kubelet pod directory.
func volumePath(volume, uid string) (string, error) {
// TODO: Support subpath when gvisor supports pod volume bind mount.
@@ -117,12 +129,22 @@ func UpdateVolumeAnnotations(s *specs.Spec) (bool, error) {
}
volume := volumeName(k)
if uid != "" {
// This is a sandbox.
// This is a sandbox. Add source and lifecycle annotations for volumes.
path, err := volumePath(volume, uid)
if err != nil {
return false, fmt.Errorf("get volume path for %q: %w", volume, err)
}
s.Annotations[volumeSourceKey(volume)] = path
// TODO(b/142076984): Remove the lifecycle setting logic after it has
// been adopted in GKE admission plugin.
lifecycleKey := volumeLifecycleKey(volume)
if _, ok := s.Annotations[lifecycleKey]; !ok {
// Only set lifecycle annotation if not already set.
if strings.Contains(path, emptyDirVolumesDir) {
// Emptydir is created and destroyed with the pod.
s.Annotations[lifecycleKey] = "pod"
}
}
updated = true
} else {
// This is a container.
@@ -188,6 +210,7 @@ func configureShm(s *specs.Spec) (bool, error) {
s.Annotations[volumeKeyPrefix+devshmName+".source"] = m.Source
s.Annotations[volumeKeyPrefix+devshmName+".type"] = devshmType
s.Annotations[volumeKeyPrefix+devshmName+".share"] = "pod"
s.Annotations[volumeKeyPrefix+devshmName+".lifecycle"] = "pod"
// Given that we don't have visibility into mount options for all
// containers, assume broad access for the master mount (it's tmpfs
// inside the sandbox anyways) and apply options to subcontainers as
+34 -29
View File
@@ -38,7 +38,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
testLogDirPath = "/var/log/pods/testns_testname_" + testPodUID
testLegacyLogDirPath = "/var/log/pods/" + testPodUID
)
testVolumePath := fmt.Sprintf("%s/%s/volumes/kubernetes.io~empty-dir/%s", dir, testPodUID, testVolumeName)
testVolumePath := fmt.Sprintf("%s/%s/volumes/%s/%s", dir, testPodUID, emptyDirVolumesDir, testVolumeName)
if err := os.MkdirAll(testVolumePath, 0755); err != nil {
t.Fatalf("Create test volume: %v", err)
@@ -64,12 +64,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
},
expected: &specs.Spec{
Annotations: map[string]string{
sandboxLogDirAnnotation: testLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
sandboxLogDirAnnotation: testLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
volumeKeyPrefix + testVolumeName + ".lifecycle": "pod",
},
},
expectUpdate: true,
@@ -87,12 +88,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
},
expected: &specs.Spec{
Annotations: map[string]string{
sandboxLogDirAnnotation: testLegacyLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
sandboxLogDirAnnotation: testLegacyLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
volumeKeyPrefix + testVolumeName + ".lifecycle": "pod",
},
},
expectUpdate: true,
@@ -273,11 +275,12 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
name: "bind options removed",
spec: &specs.Spec{
Annotations: map[string]string{
ContainerTypeAnnotation: ContainerTypeContainer,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
ContainerTypeAnnotation: ContainerTypeContainer,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
volumeKeyPrefix + testVolumeName + ".lifecycle": "pod",
},
Mounts: []specs.Mount{
{
@@ -290,11 +293,12 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
},
expected: &specs.Spec{
Annotations: map[string]string{
ContainerTypeAnnotation: ContainerTypeContainer,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
ContainerTypeAnnotation: ContainerTypeContainer,
volumeKeyPrefix + testVolumeName + ".share": "pod",
volumeKeyPrefix + testVolumeName + ".type": "tmpfs",
volumeKeyPrefix + testVolumeName + ".options": "ro",
volumeKeyPrefix + testVolumeName + ".source": testVolumePath,
volumeKeyPrefix + testVolumeName + ".lifecycle": "pod",
},
Mounts: []specs.Mount{
{
@@ -325,12 +329,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
},
expected: &specs.Spec{
Annotations: map[string]string{
sandboxLogDirAnnotation: testLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + devshmName + ".share": "pod",
volumeKeyPrefix + devshmName + ".type": "tmpfs",
volumeKeyPrefix + devshmName + ".options": "rw",
volumeKeyPrefix + devshmName + ".source": testVolumePath,
sandboxLogDirAnnotation: testLogDirPath,
ContainerTypeAnnotation: containerTypeSandbox,
volumeKeyPrefix + devshmName + ".share": "pod",
volumeKeyPrefix + devshmName + ".type": "tmpfs",
volumeKeyPrefix + devshmName + ".options": "rw",
volumeKeyPrefix + devshmName + ".source": testVolumePath,
volumeKeyPrefix + devshmName + ".lifecycle": "pod",
},
Mounts: []specs.Mount{
{
+87 -25
View File
@@ -29,16 +29,18 @@ import (
// MountPrefix is the annotation prefix for mount hints.
const MountPrefix = "dev.gvisor.spec.mount."
// shareType indicates who can access/mutate the volume contents.
type shareType int
const (
invalid shareType = iota
// container shareType indicates that the mount is used by a single container.
// container shareType indicates that the mount is used by a single
// container. There are no external observers.
container
// pod shareType indicates that the mount is used by more than one container
// inside the pod.
// inside the pod. There are no external observers.
pod
// shared shareType indicates that the mount can also be shared with a process
@@ -46,19 +48,6 @@ const (
shared
)
func parseShare(val string) (shareType, error) {
switch val {
case "container":
return container, nil
case "pod":
return pod, nil
case "shared":
return shared, nil
default:
return 0, fmt.Errorf("invalid share value %q", val)
}
}
func (s shareType) String() string {
switch s {
case invalid:
@@ -74,6 +63,37 @@ func (s shareType) String() string {
}
}
// lifecycleType indicates whether creation/deletion of the volume is tied to
// the pod or container's lifecycle.
type lifecycleType int
const (
// sharedLife indicates that the volume's lifecycle is not tied to the pod.
// The volume persists beyond the pod's life. This is the safe default.
sharedLife lifecycleType = iota
// podLife indicates that the volume's lifecycle is tied to the pod's
// lifecycle. The volume is destroyed with the pod.
podLife
// containerLife indicates that the volume's lifecycle is tied to the
// container's lifecycle. The volume is destroyed with the container.
containerLife
)
func (o lifecycleType) String() string {
switch o {
case sharedLife:
return "shared"
case podLife:
return "pod"
case containerLife:
return "container"
default:
return fmt.Sprintf("invalid lifecycle value %d", o)
}
}
// PodMountHints contains a collection of mountHints for the pod.
type PodMountHints struct {
mounts map[string]*MountHint
@@ -129,9 +149,10 @@ func NewPodMountHints(spec *specs.Spec) (*PodMountHints, error) {
// annotations. They can override mount type, and provide sharing information
// so that mounts can be correctly shared inside the pod.
type MountHint struct {
name string
share shareType
mount specs.Mount
name string
share shareType
mount specs.Mount
lifecycle lifecycleType
// vfsMount is the master mount for the volume. For mounts with 'pod' share
// the master volume is bind mounted inside the containers.
@@ -148,13 +169,11 @@ func (m *MountHint) setField(key, val string) error {
case "type":
return m.setType(val)
case "share":
share, err := parseShare(val)
if err != nil {
return err
}
m.share = share
return m.setShare(val)
case "options":
m.mount.Options = specutils.FilterMountOptions(strings.Split(val, ","))
case "lifecycle":
return m.setLifecycle(val)
default:
return fmt.Errorf("invalid mount annotation: %s=%s", key, val)
}
@@ -171,13 +190,49 @@ func (m *MountHint) setType(val string) error {
return nil
}
// isShared returns true if this mount should be configured as a shared mount.
func (m *MountHint) isShared() bool {
func (m *MountHint) setShare(val string) error {
switch val {
case container.String():
m.share = container
case pod.String():
m.share = pod
case shared.String():
m.share = shared
default:
return fmt.Errorf("invalid share value %q", val)
}
return nil
}
func (m *MountHint) setLifecycle(val string) error {
switch val {
case containerLife.String():
m.lifecycle = containerLife
case podLife.String():
m.lifecycle = podLife
case sharedLife.String():
m.lifecycle = sharedLife
default:
return fmt.Errorf("invalid lifecycle %q", val)
}
return nil
}
// shouldShareMount returns true if this mount should be configured as a shared
// mount that is shared among multiple containers in a pod.
func (m *MountHint) shouldShareMount() bool {
// TODO(b/142076984): Only support tmpfs for now. Bind mounts require a
// common gofer to mount all shared volumes.
return m.mount.Type == tmpfs.Name && m.share == pod
}
// ShouldOverlay returns true if this mount should be overlaid.
func (m *MountHint) ShouldOverlay() bool {
// TODO(b/142076984): Only support share=container for now. Once shared gofer
// support is added, we can overlay shared bind mounts too.
return m.mount.Type == Bind && m.share == container && m.lifecycle != sharedLife
}
// checkCompatible verifies that shared mount is compatible with master.
// Master options must be the same or less restrictive than the container mount,
// e.g. master can be 'rw' while container mounts as 'ro'.
@@ -197,7 +252,14 @@ func (m *MountHint) checkCompatible(replica *specs.Mount) error {
return nil
}
// Precondition: m.mount.Type == Bind.
func (m *MountHint) fileAccessType() config.FileAccessType {
if m.share == shared {
return config.FileAccessShared
}
if m.shouldShareMount() {
return config.FileAccessExclusive
}
if m.share == container {
return config.FileAccessExclusive
}
+2 -2
View File
@@ -623,7 +623,7 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, conf *config.Conf
err error
)
if submount.hint != nil && submount.hint.isShared() {
if submount.hint != nil && submount.hint.shouldShareMount() {
mnt, err = c.mountSharedSubmount(ctx, conf, mns, creds, submount.mount, submount.hint)
if err != nil {
return fmt.Errorf("mount shared mount %q to %q: %v", submount.hint.name, submount.mount.Destination, err)
@@ -922,7 +922,7 @@ func (c *containerMounter) mountTmp(ctx context.Context, conf *config.Config, cr
func (c *containerMounter) processHints(conf *config.Config, creds *auth.Credentials) error {
ctx := c.k.SupervisorContext()
for _, hint := range c.hints.mounts {
if !hint.isShared() {
if !hint.shouldShareMount() {
continue
}
+16 -6
View File
@@ -285,7 +285,11 @@ func New(conf *config.Config, args Args) (*Container, error) {
}
}
c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup}
overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores()
mountHints, err := boot.NewPodMountHints(args.Spec)
if err != nil {
return nil, fmt.Errorf("error creating pod mount hints: %w", err)
}
overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(mountHints)
if err != nil {
return nil, err
}
@@ -310,6 +314,7 @@ func New(conf *config.Config, args Args) (*Container, error) {
Attached: args.Attached,
OverlayFilestoreFiles: overlayFilestoreFiles,
OverlayMediums: overlayMediums,
MountHints: mountHints,
PassFiles: args.PassFiles,
ExecFile: args.ExecFile,
}
@@ -426,7 +431,7 @@ func (c *Container) Start(conf *config.Config) error {
return err
}
} else {
overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores()
overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(c.Sandbox.MountHints)
if err != nil {
return err
}
@@ -861,13 +866,13 @@ func (c *Container) forEachSelfOverlay(fn func(mountSrc string)) {
// createOverlayFilestores creates the regular files that will back the tmpfs
// upper mount for overlay mounts. It also returns information about the
// overlay medium used for each bind mount.
func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium, error) {
func (c *Container) createOverlayFilestores(mountHints *boot.PodMountHints) ([]*os.File, []boot.OverlayMedium, error) {
var filestoreFiles []*os.File
var overlayMediums []boot.OverlayMedium
// Handle root mount first.
shouldOverlay := c.OverlayConf.RootMount && !c.Spec.Root.Readonly
filestore, medium, err := c.createOverlayFilestore(c.Spec.Root.Path, shouldOverlay)
filestore, medium, err := c.createOverlayFilestore(c.Spec.Root.Path, shouldOverlay, nil /* hint */)
if err != nil {
return nil, nil, err
}
@@ -881,8 +886,9 @@ func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium,
if !specutils.IsGoferMount(c.Spec.Mounts[i]) {
continue
}
hint := mountHints.FindMount(&c.Spec.Mounts[i])
shouldOverlay := c.OverlayConf.SubMounts && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options)
filestore, medium, err := c.createOverlayFilestore(c.Spec.Mounts[i].Source, shouldOverlay)
filestore, medium, err := c.createOverlayFilestore(c.Spec.Mounts[i].Source, shouldOverlay, hint)
if err != nil {
return nil, nil, err
}
@@ -899,7 +905,11 @@ func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium,
return filestoreFiles, overlayMediums, nil
}
func (c *Container) createOverlayFilestore(mountSrc string, shouldOverlay bool) (*os.File, boot.OverlayMedium, error) {
func (c *Container) createOverlayFilestore(mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.OverlayMedium, error) {
if hint != nil && hint.ShouldOverlay() {
// MountHint information takes precedence over shouldOverlay.
return c.createOverlayFilestoreInSelf(mountSrc)
}
switch {
case !shouldOverlay:
return nil, boot.NoOverlay, nil
+92
View File
@@ -42,6 +42,7 @@ import (
"gvisor.dev/gvisor/pkg/sentry/platform"
"gvisor.dev/gvisor/pkg/sync"
"gvisor.dev/gvisor/pkg/test/testutil"
"gvisor.dev/gvisor/runsc/boot"
"gvisor.dev/gvisor/runsc/cgroup"
"gvisor.dev/gvisor/runsc/config"
"gvisor.dev/gvisor/runsc/flag"
@@ -3032,3 +3033,94 @@ func TestExecFDExec(t *testing.T) {
t.Errorf("echo result, got: %q, want: %q", got, want)
}
}
// This test checks that a bind mount which is annotated to be fully owned by
// the sandbox is overlaid using "self" overlay medium.
func TestOverlayByMountAnnotation(t *testing.T) {
conf := testutil.TestConfig(t)
// Disable overlay settings.
conf.Overlay2.Set("none")
// We just sleep here because we want to test execution in an already
// running container.
spec := testutil.NewSpecWithArgs("bash", "-c", "sleep infinity")
// Set up a bind mount at "/submount".
subMount, err := ioutil.TempDir(testutil.TmpDir(), "submount")
if err != nil {
t.Fatalf("ioutil.TempDir failed: %v", err)
}
defer os.RemoveAll(subMount)
spec.Mounts = append(spec.Mounts, specs.Mount{
Destination: subMount,
Source: subMount,
Type: "bind",
})
// Add mount annotation to self-overlay the submount.
volumeName := "mount1"
if spec.Annotations == nil {
spec.Annotations = make(map[string]string)
}
spec.Annotations[boot.MountPrefix+volumeName+".source"] = subMount
spec.Annotations[boot.MountPrefix+volumeName+".type"] = "bind"
spec.Annotations[boot.MountPrefix+volumeName+".share"] = "container"
spec.Annotations[boot.MountPrefix+volumeName+".lifecycle"] = "pod"
_, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf)
if err != nil {
t.Fatalf("error setting up container: %v", err)
}
defer cleanup()
args := Args{
ID: testutil.RandomContainerID(),
Spec: spec,
BundleDir: bundleDir,
}
cont, err := New(conf, args)
if err != nil {
t.Fatalf("Creating container: %v", err)
}
destroyed := false
destroy := func() {
if destroyed {
return
}
destroyed = true
cont.Destroy()
}
defer destroy()
if err := cont.Start(conf); err != nil {
t.Fatalf("starting container: %v", err)
}
// Create a file in submount with a few bytes.
testFilePath := path.Join(subMount, "testfile")
if ws, err := execute(conf, cont, "/bin/sh", "-c", "echo hello > "+testFilePath); err != nil || ws != 0 {
t.Fatalf("exec command failed to write a file in submount, ws: %v, err: %v", ws, err)
}
// Check that the filestore file is created and is not empty.
filestoreFile := boot.SelfOverlayFilestorePath(subMount, cont.Sandbox.ID)
var stat unix.Stat_t
if err := unix.Stat(filestoreFile, &stat); err != nil {
t.Fatalf("unix.Stat(%q) failed for submount filestore: %v", filestoreFile, err)
}
if stat.Blocks == 0 {
t.Errorf("submount filestore file %q is empty", filestoreFile)
}
// Check that the file is not created on the host.
if err := unix.Stat(path.Join(subMount, testFilePath), &stat); err == nil {
t.Errorf("%q file created on the host in spite of overlay", testFilePath)
}
// Destroying the container should delete the filestore file.
destroy()
if err := unix.Stat(filestoreFile, &stat); err == nil {
t.Fatalf("overlay filestore at %q was not deleted after container.Destroy()", filestoreFile)
}
}
+9
View File
@@ -178,6 +178,10 @@ type Sandbox struct {
// ControlAddress is the uRPC address used to connect to the sandbox.
ControlAddress string `json:"control_address"`
// MountHints provides extra information about container mounts that apply
// to the entire pod.
MountHints *boot.PodMountHints `json:"mountHints"`
// child is set if a sandbox process is a child of the current process.
//
// This field isn't saved to json, because only a creator of sandbox
@@ -231,6 +235,10 @@ type Args struct {
// bind mounts in Spec.Mounts (in the same order).
OverlayMediums []boot.OverlayMedium
// MountHints provides extra information about containers mounts that apply
// to the entire pod.
MountHints *boot.PodMountHints
// MountsFile is a file container mount information from the spec. It's
// equivalent to the mounts from the spec, except that all paths have been
// resolved to their final absolute location.
@@ -267,6 +275,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) {
GID: -1, // prevent usage before it's set.
MetricMetadata: conf.MetricMetadata(),
MetricServerAddress: conf.MetricServer,
MountHints: args.MountHints,
}
if args.Spec != nil && args.Spec.Annotations != nil {
s.PodName = args.Spec.Annotations[podNameAnnotation]