mirror of
https://github.com/netbirdio/gvisor.git
synced 2026-05-22 17:12:49 -07:00
Support volume annotations (#40)
* Add volume annotation support Signed-off-by: Lantao Liu <lantaol@google.com> * Add unit test. Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu
+3
-1
@@ -24,4 +24,6 @@ before_install:
|
||||
- sudo apt-get -q update
|
||||
- sudo apt-get install -y libseccomp-dev/trusty-backports
|
||||
|
||||
script: ./test/e2e/${TEST}/test.sh
|
||||
script:
|
||||
- make test
|
||||
- ./test/e2e/${TEST}/test.sh
|
||||
|
||||
@@ -35,3 +35,8 @@ push: binaries
|
||||
gsutil cp ./bin/gvisor-containerd-shim gs://$(DEPLOY_PATH)/gvisor-containerd-shim-$(VERSION)
|
||||
gsutil cp ./bin/containerd-shim-runsc-v1 gs://$(DEPLOY_PATH)/containerd-shim-runsc-v1-$(VERSION)
|
||||
echo "$(VERSION)" | gsutil cp - "gs://$(DEPLOY_PATH)/latest"
|
||||
|
||||
.PHONY: test
|
||||
|
||||
test:
|
||||
GO111MODULE=${GO_MODULE} CGO_ENABLED=0 ${GC} test -v ./pkg/... ${SHIM_GO_LDFLAGS} ${GO_TAGS}
|
||||
|
||||
@@ -555,6 +555,10 @@ func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace string,
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "read oci spec")
|
||||
}
|
||||
if err := utils.UpdateVolumeAnnotations(r.Bundle, spec); err != nil {
|
||||
return nil, errors.Wrap(err, "update volume annotations")
|
||||
}
|
||||
|
||||
runsc.FormatLogPath(r.ID, config)
|
||||
rootfs := filepath.Join(path, "rootfs")
|
||||
runtime := proc.NewRunsc(runtimeRoot, path, namespace, r.Runtime, config)
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
/*
|
||||
Copyright 2019 Google LLC.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/containerd/cri/pkg/annotations"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const volumeKeyPrefix = "gvisor.dev/spec/mount/"
|
||||
|
||||
var kubeletPodsDir = "/var/lib/kubelet/pods"
|
||||
|
||||
// volumeName gets volume name from volume annotation key, example:
|
||||
// gvisor.dev/spec/mount/NAME/share
|
||||
func volumeName(k string) string {
|
||||
return strings.SplitN(strings.TrimPrefix(k, volumeKeyPrefix), "/", 2)[0]
|
||||
}
|
||||
|
||||
// volumeFieldName gets volume field name from volume annotation key, example:
|
||||
// `type` is the field of gvisor.dev/spec/mount/NAME/type
|
||||
func volumeFieldName(k string) string {
|
||||
parts := strings.Split(strings.TrimPrefix(k, volumeKeyPrefix), "/")
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// podUID gets pod UID from the pod log path.
|
||||
func podUID(s *specs.Spec) (string, error) {
|
||||
sandboxLogDir := s.Annotations[annotations.SandboxLogDir]
|
||||
if sandboxLogDir == "" {
|
||||
return "", errors.New("no sandbox log path annotation")
|
||||
}
|
||||
fields := strings.Split(filepath.Base(sandboxLogDir), "_")
|
||||
switch len(fields) {
|
||||
case 1: // This is the old CRI logging path
|
||||
return fields[0], nil
|
||||
case 3: // This is the new CRI logging path
|
||||
return fields[2], nil
|
||||
}
|
||||
return "", errors.Errorf("unexpected sandbox log path %q", sandboxLogDir)
|
||||
}
|
||||
|
||||
// isVolumeKey checks whether an annotation key is for volume.
|
||||
func isVolumeKey(k string) bool {
|
||||
return strings.HasPrefix(k, volumeKeyPrefix)
|
||||
}
|
||||
|
||||
// volumeSourceKey constructs the annotation key for volume source.
|
||||
func volumeSourceKey(volume string) string {
|
||||
return volumeKeyPrefix + volume + "/source"
|
||||
}
|
||||
|
||||
// volumePath searches the volume path in the kubelet pod directory.
|
||||
func volumePath(volume, uid string) (string, error) {
|
||||
// TODO: Support subpath when gvisor supports pod volume bind mount.
|
||||
volumeSearchPath := fmt.Sprintf("%s/%s/volumes/*/%s", kubeletPodsDir, uid, volume)
|
||||
dirs, err := filepath.Glob(volumeSearchPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
return "", errors.Errorf("unexpected matched volume list %v", dirs)
|
||||
}
|
||||
return dirs[0], nil
|
||||
}
|
||||
|
||||
// isVolumePath checks whether a string is the volume path.
|
||||
func isVolumePath(volume, path string) (bool, error) {
|
||||
// TODO: Support subpath when gvisor supports pod volume bind mount.
|
||||
volumeSearchPath := fmt.Sprintf("%s/*/volumes/*/%s", kubeletPodsDir, volume)
|
||||
return filepath.Match(volumeSearchPath, path)
|
||||
}
|
||||
|
||||
// UpdateVolumeAnnotations add necessary OCI annotations for gvisor
|
||||
// volume optimization.
|
||||
func UpdateVolumeAnnotations(bundle string, s *specs.Spec) error {
|
||||
var (
|
||||
uid string
|
||||
err error
|
||||
)
|
||||
if IsSandbox(s) {
|
||||
uid, err = podUID(s)
|
||||
if err != nil {
|
||||
// Skip if we can't get pod UID, because this doesn't work
|
||||
// for containerd 1.1.
|
||||
logrus.WithError(err).Error("Can't get pod uid")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
var updated bool
|
||||
for k, v := range s.Annotations {
|
||||
if !isVolumeKey(k) {
|
||||
continue
|
||||
}
|
||||
if volumeFieldName(k) != "type" {
|
||||
continue
|
||||
}
|
||||
if v != "tmpfs" {
|
||||
// Only tmpfs is supported now.
|
||||
continue
|
||||
}
|
||||
volume := volumeName(k)
|
||||
if uid != "" {
|
||||
// This is a sandbox
|
||||
path, err := volumePath(volume, uid)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "get volume path for %q", volume)
|
||||
}
|
||||
s.Annotations[volumeSourceKey(volume)] = path
|
||||
updated = true
|
||||
} else {
|
||||
// This is a container
|
||||
for i := range s.Mounts {
|
||||
// An error is returned for sandbox if source annotation
|
||||
// is not successfully applied, so it is guaranteed that
|
||||
// the source annotation for sandbox has already been
|
||||
// successfully applied at this point.
|
||||
// The volume name is unique inside a pod, so matching without
|
||||
// podUID is fine here.
|
||||
// TODO: Pass podUID down to shim for containers to do
|
||||
// more accurate matching.
|
||||
if yes, _ := isVolumePath(volume, s.Mounts[i].Source); yes {
|
||||
// gVisor requires the container mount type to match
|
||||
// sandbox mount type for tmpfs.
|
||||
s.Mounts[i].Type = "tmpfs"
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !updated {
|
||||
return nil
|
||||
}
|
||||
// Update bundle
|
||||
b, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(filepath.Join(bundle, "config.json"), b, 0666)
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
/*
|
||||
Copyright 2019 Google LLC.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/cri/pkg/annotations"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
)
|
||||
|
||||
func TestUpdateVolumeAnnotations(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "test-update-volume-annotations")
|
||||
if err != nil {
|
||||
t.Fatalf("create tempdir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
kubeletPodsDir = dir
|
||||
|
||||
const (
|
||||
testPodUID = "testuid"
|
||||
testVolumeName = "testvolume"
|
||||
testLogDirPath = "/var/log/pods/testns_testname_" + testPodUID
|
||||
testLegacyLogDirPath = "/var/log/pods/" + testPodUID
|
||||
)
|
||||
testVolumePath := fmt.Sprintf("%s/%s/volumes/kubernetes.io~empty-dir/%s", dir, testPodUID, testVolumeName)
|
||||
|
||||
if err := os.MkdirAll(testVolumePath, 0755); err != nil {
|
||||
t.Fatalf("Create test volume: %v", err)
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
desc string
|
||||
spec *specs.Spec
|
||||
expected *specs.Spec
|
||||
expectErr bool
|
||||
expectUpdate bool
|
||||
}{
|
||||
{
|
||||
desc: "volume annotations for sandbox",
|
||||
spec: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/source": testVolumePath,
|
||||
},
|
||||
},
|
||||
expectUpdate: true,
|
||||
},
|
||||
{
|
||||
desc: "volume annotations for sandbox with legacy log path",
|
||||
spec: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLegacyLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLegacyLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/source": testVolumePath,
|
||||
},
|
||||
},
|
||||
expectUpdate: true,
|
||||
},
|
||||
{
|
||||
desc: "volume annotations for container",
|
||||
spec: &specs.Spec{
|
||||
Mounts: []specs.Mount{
|
||||
{
|
||||
Destination: "/test",
|
||||
Type: "bind",
|
||||
Source: testVolumePath,
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
{
|
||||
Destination: "/random",
|
||||
Type: "bind",
|
||||
Source: "/random",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeContainer,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Mounts: []specs.Mount{
|
||||
{
|
||||
Destination: "/test",
|
||||
Type: "tmpfs",
|
||||
Source: testVolumePath,
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
{
|
||||
Destination: "/random",
|
||||
Type: "bind",
|
||||
Source: "/random",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeContainer,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
expectUpdate: true,
|
||||
},
|
||||
{
|
||||
desc: "should not return error without pod log directory",
|
||||
spec: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/share": "pod",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/" + testVolumeName + "/options": "ro",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "should return error if volume path does not exist",
|
||||
spec: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
"gvisor.dev/spec/mount/notexist/share": "pod",
|
||||
"gvisor.dev/spec/mount/notexist/type": "tmpfs",
|
||||
"gvisor.dev/spec/mount/notexist/options": "ro",
|
||||
},
|
||||
},
|
||||
expectErr: true,
|
||||
},
|
||||
{
|
||||
desc: "no volume annotations for sandbox",
|
||||
spec: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Annotations: map[string]string{
|
||||
annotations.SandboxLogDir: testLogDirPath,
|
||||
annotations.ContainerType: annotations.ContainerTypeSandbox,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "no volume annotations for container",
|
||||
spec: &specs.Spec{
|
||||
Mounts: []specs.Mount{
|
||||
{
|
||||
Destination: "/test",
|
||||
Type: "bind",
|
||||
Source: "/test",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
{
|
||||
Destination: "/random",
|
||||
Type: "bind",
|
||||
Source: "/random",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeContainer,
|
||||
},
|
||||
},
|
||||
expected: &specs.Spec{
|
||||
Mounts: []specs.Mount{
|
||||
{
|
||||
Destination: "/test",
|
||||
Type: "bind",
|
||||
Source: "/test",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
{
|
||||
Destination: "/random",
|
||||
Type: "bind",
|
||||
Source: "/random",
|
||||
Options: []string{"ro"},
|
||||
},
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
annotations.ContainerType: annotations.ContainerTypeContainer,
|
||||
},
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
bundle, err := ioutil.TempDir(dir, "test-bundle")
|
||||
if err != nil {
|
||||
t.Fatalf("Create test bundle: %v", err)
|
||||
}
|
||||
err = UpdateVolumeAnnotations(bundle, test.spec)
|
||||
if test.expectErr {
|
||||
if err == nil {
|
||||
t.Fatal("Expected error, but got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(test.expected, test.spec) {
|
||||
t.Fatalf("Expected %+v, got %+v", test.expected, test.spec)
|
||||
}
|
||||
if test.expectUpdate {
|
||||
b, err := ioutil.ReadFile(filepath.Join(bundle, "config.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("Read spec from bundle: %v", err)
|
||||
}
|
||||
var spec specs.Spec
|
||||
if err := json.Unmarshal(b, &spec); err != nil {
|
||||
t.Fatalf("Unmarshal spec: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(test.expected, &spec) {
|
||||
t.Fatalf("Expected %+v, got %+v", test.expected, &spec)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -709,6 +709,9 @@ func newInit(ctx context.Context, path, workDir, namespace string, platform rpro
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "read oci spec")
|
||||
}
|
||||
if err := utils.UpdateVolumeAnnotations(r.Bundle, spec); err != nil {
|
||||
return nil, errors.Wrap(err, "update volume annotations")
|
||||
}
|
||||
runsc.FormatLogPath(r.ID, options.RunscConfig)
|
||||
runtime := proc.NewRunsc(options.Root, path, namespace, options.BinaryName, options.RunscConfig)
|
||||
p := proc.New(r.ID, runtime, rproc.Stdio{
|
||||
|
||||
Reference in New Issue
Block a user