From 0ca9dbc41c02d4150e847c00586cae1ed7fb717f Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Tue, 9 May 2023 16:40:23 -0700 Subject: [PATCH] Add self-backed overlay for disk-backed EmptyDir volumes. This change adds a new mount annotation: dev.gvisor.spec.mount.{volumeName}.lifecycle: shared | pod | container For now runsc shim will be setting this annotation for all EmptyDir volumes. Runsc looks at this annotation can adds a self-backed overlay on top of EmptyDir volumes that have type=bind and share=container. runsc gives precedence to mount annotations over --overlay2 configuration. Added a container test for this feature. This change also updates MountHint.fileAccessType() to be more generic using all the available information about a mount. PiperOrigin-RevId: 530746837 --- pkg/shim/utils/volumes.go | 25 ++++++- pkg/shim/utils/volumes_test.go | 63 +++++++++-------- runsc/boot/mount_hints.go | 112 +++++++++++++++++++++++------- runsc/boot/vfs.go | 4 +- runsc/container/container.go | 22 ++++-- runsc/container/container_test.go | 92 ++++++++++++++++++++++++ runsc/sandbox/sandbox.go | 9 +++ 7 files changed, 264 insertions(+), 63 deletions(-) diff --git a/pkg/shim/utils/volumes.go b/pkg/shim/utils/volumes.go index 1e0576ff9..8af2081da 100644 --- a/pkg/shim/utils/volumes.go +++ b/pkg/shim/utils/volumes.go @@ -28,8 +28,15 @@ const ( // devshmName is the volume name used for /dev/shm. Pick a name that is // unlikely to be used. devshmName = "gvisorinternaldevshm" + + // emptyDirVolumesDir is the directory inside kubeletPodsDir/{uid}/volumes/ + // that hosts all the EmptyDir volumes used by the pod. + emptyDirVolumesDir = "kubernetes.io~empty-dir" ) +// The directory structure for volumes is as follows: +// /var/lib/kubelet/pods/{uid}/volumes/{type} where `uid` is the pod UID and +// `type` is the volume type. var kubeletPodsDir = "/var/lib/kubelet/pods" // volumeName gets volume name from volume annotation key, example: @@ -73,6 +80,11 @@ func volumeSourceKey(volume string) string { return volumeKeyPrefix + volume + ".source" } +// volumeLifecycleKey constructs the annotation key for volume lifecycle. +func volumeLifecycleKey(volume string) string { + return volumeKeyPrefix + volume + ".lifecycle" +} + // volumePath searches the volume path in the kubelet pod directory. func volumePath(volume, uid string) (string, error) { // TODO: Support subpath when gvisor supports pod volume bind mount. @@ -117,12 +129,22 @@ func UpdateVolumeAnnotations(s *specs.Spec) (bool, error) { } volume := volumeName(k) if uid != "" { - // This is a sandbox. + // This is a sandbox. Add source and lifecycle annotations for volumes. path, err := volumePath(volume, uid) if err != nil { return false, fmt.Errorf("get volume path for %q: %w", volume, err) } s.Annotations[volumeSourceKey(volume)] = path + // TODO(b/142076984): Remove the lifecycle setting logic after it has + // been adopted in GKE admission plugin. + lifecycleKey := volumeLifecycleKey(volume) + if _, ok := s.Annotations[lifecycleKey]; !ok { + // Only set lifecycle annotation if not already set. + if strings.Contains(path, emptyDirVolumesDir) { + // Emptydir is created and destroyed with the pod. + s.Annotations[lifecycleKey] = "pod" + } + } updated = true } else { // This is a container. @@ -188,6 +210,7 @@ func configureShm(s *specs.Spec) (bool, error) { s.Annotations[volumeKeyPrefix+devshmName+".source"] = m.Source s.Annotations[volumeKeyPrefix+devshmName+".type"] = devshmType s.Annotations[volumeKeyPrefix+devshmName+".share"] = "pod" + s.Annotations[volumeKeyPrefix+devshmName+".lifecycle"] = "pod" // Given that we don't have visibility into mount options for all // containers, assume broad access for the master mount (it's tmpfs // inside the sandbox anyways) and apply options to subcontainers as diff --git a/pkg/shim/utils/volumes_test.go b/pkg/shim/utils/volumes_test.go index cc6ab44fc..2e09098e5 100644 --- a/pkg/shim/utils/volumes_test.go +++ b/pkg/shim/utils/volumes_test.go @@ -38,7 +38,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) { testLogDirPath = "/var/log/pods/testns_testname_" + testPodUID testLegacyLogDirPath = "/var/log/pods/" + testPodUID ) - testVolumePath := fmt.Sprintf("%s/%s/volumes/kubernetes.io~empty-dir/%s", dir, testPodUID, testVolumeName) + testVolumePath := fmt.Sprintf("%s/%s/volumes/%s/%s", dir, testPodUID, emptyDirVolumesDir, testVolumeName) if err := os.MkdirAll(testVolumePath, 0755); err != nil { t.Fatalf("Create test volume: %v", err) @@ -64,12 +64,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) { }, expected: &specs.Spec{ Annotations: map[string]string{ - sandboxLogDirAnnotation: testLogDirPath, - ContainerTypeAnnotation: containerTypeSandbox, - volumeKeyPrefix + testVolumeName + ".share": "pod", - volumeKeyPrefix + testVolumeName + ".type": "tmpfs", - volumeKeyPrefix + testVolumeName + ".options": "ro", - volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + sandboxLogDirAnnotation: testLogDirPath, + ContainerTypeAnnotation: containerTypeSandbox, + volumeKeyPrefix + testVolumeName + ".share": "pod", + volumeKeyPrefix + testVolumeName + ".type": "tmpfs", + volumeKeyPrefix + testVolumeName + ".options": "ro", + volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + volumeKeyPrefix + testVolumeName + ".lifecycle": "pod", }, }, expectUpdate: true, @@ -87,12 +88,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) { }, expected: &specs.Spec{ Annotations: map[string]string{ - sandboxLogDirAnnotation: testLegacyLogDirPath, - ContainerTypeAnnotation: containerTypeSandbox, - volumeKeyPrefix + testVolumeName + ".share": "pod", - volumeKeyPrefix + testVolumeName + ".type": "tmpfs", - volumeKeyPrefix + testVolumeName + ".options": "ro", - volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + sandboxLogDirAnnotation: testLegacyLogDirPath, + ContainerTypeAnnotation: containerTypeSandbox, + volumeKeyPrefix + testVolumeName + ".share": "pod", + volumeKeyPrefix + testVolumeName + ".type": "tmpfs", + volumeKeyPrefix + testVolumeName + ".options": "ro", + volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + volumeKeyPrefix + testVolumeName + ".lifecycle": "pod", }, }, expectUpdate: true, @@ -273,11 +275,12 @@ func TestUpdateVolumeAnnotations(t *testing.T) { name: "bind options removed", spec: &specs.Spec{ Annotations: map[string]string{ - ContainerTypeAnnotation: ContainerTypeContainer, - volumeKeyPrefix + testVolumeName + ".share": "pod", - volumeKeyPrefix + testVolumeName + ".type": "tmpfs", - volumeKeyPrefix + testVolumeName + ".options": "ro", - volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + ContainerTypeAnnotation: ContainerTypeContainer, + volumeKeyPrefix + testVolumeName + ".share": "pod", + volumeKeyPrefix + testVolumeName + ".type": "tmpfs", + volumeKeyPrefix + testVolumeName + ".options": "ro", + volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + volumeKeyPrefix + testVolumeName + ".lifecycle": "pod", }, Mounts: []specs.Mount{ { @@ -290,11 +293,12 @@ func TestUpdateVolumeAnnotations(t *testing.T) { }, expected: &specs.Spec{ Annotations: map[string]string{ - ContainerTypeAnnotation: ContainerTypeContainer, - volumeKeyPrefix + testVolumeName + ".share": "pod", - volumeKeyPrefix + testVolumeName + ".type": "tmpfs", - volumeKeyPrefix + testVolumeName + ".options": "ro", - volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + ContainerTypeAnnotation: ContainerTypeContainer, + volumeKeyPrefix + testVolumeName + ".share": "pod", + volumeKeyPrefix + testVolumeName + ".type": "tmpfs", + volumeKeyPrefix + testVolumeName + ".options": "ro", + volumeKeyPrefix + testVolumeName + ".source": testVolumePath, + volumeKeyPrefix + testVolumeName + ".lifecycle": "pod", }, Mounts: []specs.Mount{ { @@ -325,12 +329,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) { }, expected: &specs.Spec{ Annotations: map[string]string{ - sandboxLogDirAnnotation: testLogDirPath, - ContainerTypeAnnotation: containerTypeSandbox, - volumeKeyPrefix + devshmName + ".share": "pod", - volumeKeyPrefix + devshmName + ".type": "tmpfs", - volumeKeyPrefix + devshmName + ".options": "rw", - volumeKeyPrefix + devshmName + ".source": testVolumePath, + sandboxLogDirAnnotation: testLogDirPath, + ContainerTypeAnnotation: containerTypeSandbox, + volumeKeyPrefix + devshmName + ".share": "pod", + volumeKeyPrefix + devshmName + ".type": "tmpfs", + volumeKeyPrefix + devshmName + ".options": "rw", + volumeKeyPrefix + devshmName + ".source": testVolumePath, + volumeKeyPrefix + devshmName + ".lifecycle": "pod", }, Mounts: []specs.Mount{ { diff --git a/runsc/boot/mount_hints.go b/runsc/boot/mount_hints.go index f89a5d471..f1e8cc73a 100644 --- a/runsc/boot/mount_hints.go +++ b/runsc/boot/mount_hints.go @@ -29,16 +29,18 @@ import ( // MountPrefix is the annotation prefix for mount hints. const MountPrefix = "dev.gvisor.spec.mount." +// shareType indicates who can access/mutate the volume contents. type shareType int const ( invalid shareType = iota - // container shareType indicates that the mount is used by a single container. + // container shareType indicates that the mount is used by a single + // container. There are no external observers. container // pod shareType indicates that the mount is used by more than one container - // inside the pod. + // inside the pod. There are no external observers. pod // shared shareType indicates that the mount can also be shared with a process @@ -46,19 +48,6 @@ const ( shared ) -func parseShare(val string) (shareType, error) { - switch val { - case "container": - return container, nil - case "pod": - return pod, nil - case "shared": - return shared, nil - default: - return 0, fmt.Errorf("invalid share value %q", val) - } -} - func (s shareType) String() string { switch s { case invalid: @@ -74,6 +63,37 @@ func (s shareType) String() string { } } +// lifecycleType indicates whether creation/deletion of the volume is tied to +// the pod or container's lifecycle. +type lifecycleType int + +const ( + // sharedLife indicates that the volume's lifecycle is not tied to the pod. + // The volume persists beyond the pod's life. This is the safe default. + sharedLife lifecycleType = iota + + // podLife indicates that the volume's lifecycle is tied to the pod's + // lifecycle. The volume is destroyed with the pod. + podLife + + // containerLife indicates that the volume's lifecycle is tied to the + // container's lifecycle. The volume is destroyed with the container. + containerLife +) + +func (o lifecycleType) String() string { + switch o { + case sharedLife: + return "shared" + case podLife: + return "pod" + case containerLife: + return "container" + default: + return fmt.Sprintf("invalid lifecycle value %d", o) + } +} + // PodMountHints contains a collection of mountHints for the pod. type PodMountHints struct { mounts map[string]*MountHint @@ -129,9 +149,10 @@ func NewPodMountHints(spec *specs.Spec) (*PodMountHints, error) { // annotations. They can override mount type, and provide sharing information // so that mounts can be correctly shared inside the pod. type MountHint struct { - name string - share shareType - mount specs.Mount + name string + share shareType + mount specs.Mount + lifecycle lifecycleType // vfsMount is the master mount for the volume. For mounts with 'pod' share // the master volume is bind mounted inside the containers. @@ -148,13 +169,11 @@ func (m *MountHint) setField(key, val string) error { case "type": return m.setType(val) case "share": - share, err := parseShare(val) - if err != nil { - return err - } - m.share = share + return m.setShare(val) case "options": m.mount.Options = specutils.FilterMountOptions(strings.Split(val, ",")) + case "lifecycle": + return m.setLifecycle(val) default: return fmt.Errorf("invalid mount annotation: %s=%s", key, val) } @@ -171,13 +190,49 @@ func (m *MountHint) setType(val string) error { return nil } -// isShared returns true if this mount should be configured as a shared mount. -func (m *MountHint) isShared() bool { +func (m *MountHint) setShare(val string) error { + switch val { + case container.String(): + m.share = container + case pod.String(): + m.share = pod + case shared.String(): + m.share = shared + default: + return fmt.Errorf("invalid share value %q", val) + } + return nil +} + +func (m *MountHint) setLifecycle(val string) error { + switch val { + case containerLife.String(): + m.lifecycle = containerLife + case podLife.String(): + m.lifecycle = podLife + case sharedLife.String(): + m.lifecycle = sharedLife + default: + return fmt.Errorf("invalid lifecycle %q", val) + } + return nil +} + +// shouldShareMount returns true if this mount should be configured as a shared +// mount that is shared among multiple containers in a pod. +func (m *MountHint) shouldShareMount() bool { // TODO(b/142076984): Only support tmpfs for now. Bind mounts require a // common gofer to mount all shared volumes. return m.mount.Type == tmpfs.Name && m.share == pod } +// ShouldOverlay returns true if this mount should be overlaid. +func (m *MountHint) ShouldOverlay() bool { + // TODO(b/142076984): Only support share=container for now. Once shared gofer + // support is added, we can overlay shared bind mounts too. + return m.mount.Type == Bind && m.share == container && m.lifecycle != sharedLife +} + // checkCompatible verifies that shared mount is compatible with master. // Master options must be the same or less restrictive than the container mount, // e.g. master can be 'rw' while container mounts as 'ro'. @@ -197,7 +252,14 @@ func (m *MountHint) checkCompatible(replica *specs.Mount) error { return nil } +// Precondition: m.mount.Type == Bind. func (m *MountHint) fileAccessType() config.FileAccessType { + if m.share == shared { + return config.FileAccessShared + } + if m.shouldShareMount() { + return config.FileAccessExclusive + } if m.share == container { return config.FileAccessExclusive } diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 3b19a3291..19f598d84 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -623,7 +623,7 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, conf *config.Conf err error ) - if submount.hint != nil && submount.hint.isShared() { + if submount.hint != nil && submount.hint.shouldShareMount() { mnt, err = c.mountSharedSubmount(ctx, conf, mns, creds, submount.mount, submount.hint) if err != nil { return fmt.Errorf("mount shared mount %q to %q: %v", submount.hint.name, submount.mount.Destination, err) @@ -922,7 +922,7 @@ func (c *containerMounter) mountTmp(ctx context.Context, conf *config.Config, cr func (c *containerMounter) processHints(conf *config.Config, creds *auth.Credentials) error { ctx := c.k.SupervisorContext() for _, hint := range c.hints.mounts { - if !hint.isShared() { + if !hint.shouldShareMount() { continue } diff --git a/runsc/container/container.go b/runsc/container/container.go index 79407be2b..4e3a5a619 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -285,7 +285,11 @@ func New(conf *config.Config, args Args) (*Container, error) { } } c.CompatCgroup = cgroup.CgroupJSON{Cgroup: subCgroup} - overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores() + mountHints, err := boot.NewPodMountHints(args.Spec) + if err != nil { + return nil, fmt.Errorf("error creating pod mount hints: %w", err) + } + overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(mountHints) if err != nil { return nil, err } @@ -310,6 +314,7 @@ func New(conf *config.Config, args Args) (*Container, error) { Attached: args.Attached, OverlayFilestoreFiles: overlayFilestoreFiles, OverlayMediums: overlayMediums, + MountHints: mountHints, PassFiles: args.PassFiles, ExecFile: args.ExecFile, } @@ -426,7 +431,7 @@ func (c *Container) Start(conf *config.Config) error { return err } } else { - overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores() + overlayFilestoreFiles, overlayMediums, err := c.createOverlayFilestores(c.Sandbox.MountHints) if err != nil { return err } @@ -861,13 +866,13 @@ func (c *Container) forEachSelfOverlay(fn func(mountSrc string)) { // createOverlayFilestores creates the regular files that will back the tmpfs // upper mount for overlay mounts. It also returns information about the // overlay medium used for each bind mount. -func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium, error) { +func (c *Container) createOverlayFilestores(mountHints *boot.PodMountHints) ([]*os.File, []boot.OverlayMedium, error) { var filestoreFiles []*os.File var overlayMediums []boot.OverlayMedium // Handle root mount first. shouldOverlay := c.OverlayConf.RootMount && !c.Spec.Root.Readonly - filestore, medium, err := c.createOverlayFilestore(c.Spec.Root.Path, shouldOverlay) + filestore, medium, err := c.createOverlayFilestore(c.Spec.Root.Path, shouldOverlay, nil /* hint */) if err != nil { return nil, nil, err } @@ -881,8 +886,9 @@ func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium, if !specutils.IsGoferMount(c.Spec.Mounts[i]) { continue } + hint := mountHints.FindMount(&c.Spec.Mounts[i]) shouldOverlay := c.OverlayConf.SubMounts && !specutils.IsReadonlyMount(c.Spec.Mounts[i].Options) - filestore, medium, err := c.createOverlayFilestore(c.Spec.Mounts[i].Source, shouldOverlay) + filestore, medium, err := c.createOverlayFilestore(c.Spec.Mounts[i].Source, shouldOverlay, hint) if err != nil { return nil, nil, err } @@ -899,7 +905,11 @@ func (c *Container) createOverlayFilestores() ([]*os.File, []boot.OverlayMedium, return filestoreFiles, overlayMediums, nil } -func (c *Container) createOverlayFilestore(mountSrc string, shouldOverlay bool) (*os.File, boot.OverlayMedium, error) { +func (c *Container) createOverlayFilestore(mountSrc string, shouldOverlay bool, hint *boot.MountHint) (*os.File, boot.OverlayMedium, error) { + if hint != nil && hint.ShouldOverlay() { + // MountHint information takes precedence over shouldOverlay. + return c.createOverlayFilestoreInSelf(mountSrc) + } switch { case !shouldOverlay: return nil, boot.NoOverlay, nil diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 1a3e34a46..b52c0babc 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -42,6 +42,7 @@ import ( "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/cgroup" "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/flag" @@ -3032,3 +3033,94 @@ func TestExecFDExec(t *testing.T) { t.Errorf("echo result, got: %q, want: %q", got, want) } } + +// This test checks that a bind mount which is annotated to be fully owned by +// the sandbox is overlaid using "self" overlay medium. +func TestOverlayByMountAnnotation(t *testing.T) { + conf := testutil.TestConfig(t) + // Disable overlay settings. + conf.Overlay2.Set("none") + + // We just sleep here because we want to test execution in an already + // running container. + spec := testutil.NewSpecWithArgs("bash", "-c", "sleep infinity") + + // Set up a bind mount at "/submount". + subMount, err := ioutil.TempDir(testutil.TmpDir(), "submount") + if err != nil { + t.Fatalf("ioutil.TempDir failed: %v", err) + } + defer os.RemoveAll(subMount) + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: subMount, + Source: subMount, + Type: "bind", + }) + + // Add mount annotation to self-overlay the submount. + volumeName := "mount1" + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + spec.Annotations[boot.MountPrefix+volumeName+".source"] = subMount + spec.Annotations[boot.MountPrefix+volumeName+".type"] = "bind" + spec.Annotations[boot.MountPrefix+volumeName+".share"] = "container" + spec.Annotations[boot.MountPrefix+volumeName+".lifecycle"] = "pod" + + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + + cont, err := New(conf, args) + if err != nil { + t.Fatalf("Creating container: %v", err) + } + destroyed := false + destroy := func() { + if destroyed { + return + } + destroyed = true + cont.Destroy() + } + defer destroy() + + if err := cont.Start(conf); err != nil { + t.Fatalf("starting container: %v", err) + } + + // Create a file in submount with a few bytes. + testFilePath := path.Join(subMount, "testfile") + if ws, err := execute(conf, cont, "/bin/sh", "-c", "echo hello > "+testFilePath); err != nil || ws != 0 { + t.Fatalf("exec command failed to write a file in submount, ws: %v, err: %v", ws, err) + } + + // Check that the filestore file is created and is not empty. + filestoreFile := boot.SelfOverlayFilestorePath(subMount, cont.Sandbox.ID) + var stat unix.Stat_t + if err := unix.Stat(filestoreFile, &stat); err != nil { + t.Fatalf("unix.Stat(%q) failed for submount filestore: %v", filestoreFile, err) + } + if stat.Blocks == 0 { + t.Errorf("submount filestore file %q is empty", filestoreFile) + } + + // Check that the file is not created on the host. + if err := unix.Stat(path.Join(subMount, testFilePath), &stat); err == nil { + t.Errorf("%q file created on the host in spite of overlay", testFilePath) + } + + // Destroying the container should delete the filestore file. + destroy() + if err := unix.Stat(filestoreFile, &stat); err == nil { + t.Fatalf("overlay filestore at %q was not deleted after container.Destroy()", filestoreFile) + } +} diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 0227fe100..5b6036586 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -178,6 +178,10 @@ type Sandbox struct { // ControlAddress is the uRPC address used to connect to the sandbox. ControlAddress string `json:"control_address"` + // MountHints provides extra information about container mounts that apply + // to the entire pod. + MountHints *boot.PodMountHints `json:"mountHints"` + // child is set if a sandbox process is a child of the current process. // // This field isn't saved to json, because only a creator of sandbox @@ -231,6 +235,10 @@ type Args struct { // bind mounts in Spec.Mounts (in the same order). OverlayMediums []boot.OverlayMedium + // MountHints provides extra information about containers mounts that apply + // to the entire pod. + MountHints *boot.PodMountHints + // MountsFile is a file container mount information from the spec. It's // equivalent to the mounts from the spec, except that all paths have been // resolved to their final absolute location. @@ -267,6 +275,7 @@ func New(conf *config.Config, args *Args) (*Sandbox, error) { GID: -1, // prevent usage before it's set. MetricMetadata: conf.MetricMetadata(), MetricServerAddress: conf.MetricServer, + MountHints: args.MountHints, } if args.Spec != nil && args.Spec.Annotations != nil { s.PodName = args.Spec.Annotations[podNameAnnotation]