diff --git a/nogo.yaml b/nogo.yaml index 27f5c4e2b..d0722439b 100644 --- a/nogo.yaml +++ b/nogo.yaml @@ -180,6 +180,7 @@ analyzers: unreachable: external: # Enabled. exclude: + - ".*jsonschema/reader.go" - ".*protobuf/.*.go" unsafeptr: internal: diff --git a/runsc/cmd/util/BUILD b/runsc/cmd/util/BUILD index d3dea9f8a..8f11ae494 100644 --- a/runsc/cmd/util/BUILD +++ b/runsc/cmd/util/BUILD @@ -14,6 +14,7 @@ go_library( visibility = [ "//runsc/cli:__subpackages__", "//runsc/cmd:__subpackages__", + "//tools:__subpackages__", ], deps = [ "//pkg/log", diff --git a/tools/gvisor_k8s_tool/BUILD b/tools/gvisor_k8s_tool/BUILD new file mode 100644 index 000000000..1c73a86ec --- /dev/null +++ b/tools/gvisor_k8s_tool/BUILD @@ -0,0 +1,16 @@ +load("//tools:defs.bzl", "go_binary") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_binary( + name = "gvisor_k8s_tool", + srcs = ["main.go"], + deps = [ + "//runsc/flag", + "//tools/gvisor_k8s_tool/cmd/install", + "@com_github_google_subcommands//:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/README.md b/tools/gvisor_k8s_tool/README.md new file mode 100644 index 000000000..8c0ef5825 --- /dev/null +++ b/tools/gvisor_k8s_tool/README.md @@ -0,0 +1,19 @@ +# gVisor Kubernetes tool + +This tool is meant to make deploying `runsc` in a Kubernetes cluster easier. + +## Usage + +```shell +# Install using default kubectl context: +$ ./gvisor_k8s_tool install --cluster=kube: --image=my-runsc-installer + +# Install using custom kubectl config and context: +$ KUBECONFIG=/tmp/myconfig ./gvisor_k8s_tool \ + install --cluster=kube:mycontext --image=my-runsc-installer + +# Install in a GKE cluster: +$ ./gvisor_k8s_tool install \ + --cluster=gke:projects/myproject/locations/us-central1-a/clusters/mylittlecluster \ + --image=my-runsc-installer +``` diff --git a/tools/gvisor_k8s_tool/cluster/BUILD b/tools/gvisor_k8s_tool/cluster/BUILD new file mode 100644 index 000000000..62408ea14 --- /dev/null +++ b/tools/gvisor_k8s_tool/cluster/BUILD @@ -0,0 +1,19 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "cluster", + srcs = ["cluster.go"], + deps = [ + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/fields:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", + "@io_k8s_client_go//rest:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/cluster/cluster.go b/tools/gvisor_k8s_tool/cluster/cluster.go new file mode 100644 index 000000000..434e5cca7 --- /dev/null +++ b/tools/gvisor_k8s_tool/cluster/cluster.go @@ -0,0 +1,85 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cluster provides functions for dealing with Kubernetes clusters. +package cluster + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + // NamespaceDefault is the name of the default Kubernetes namespace. + NamespaceDefault = "default" +) + +// Cluster presents Kubernetes API method over a Kubernetes cluster. +type Cluster struct { + client kubernetes.Interface +} + +// New initializes a new Cluster from the given client REST config. +func New(config *rest.Config) (*Cluster, error) { + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("kubernetes.NewForConfig: %w", err) + } + return &Cluster{clientSet}, nil +} + +// CreateDaemonset creates a daemonset with default options. +func (c *Cluster) CreateDaemonset(ctx context.Context, ds *appsv1.DaemonSet) (*appsv1.DaemonSet, error) { + if ds.GetObjectMeta().GetNamespace() == "" { + ds.SetNamespace(NamespaceDefault) + } + return c.client.AppsV1().DaemonSets(ds.GetNamespace()).Create(ctx, ds, v1.CreateOptions{}) +} + +// DeleteDaemonset deletes a daemonset from this cluster. +func (c *Cluster) DeleteDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + return c.client.AppsV1().DaemonSets(ds.GetNamespace()).Delete(ctx, ds.GetName(), v1.DeleteOptions{}) +} + +// WaitForDaemonset waits until a daemonset has propagated containers across the affected nodes. +func (c *Cluster) WaitForDaemonset(ctx context.Context, ds *appsv1.DaemonSet) error { + w, err := c.client.AppsV1().DaemonSets(ds.GetNamespace()).Watch(ctx, v1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{v1.ObjectNameField: ds.ObjectMeta.Name}).String(), + }) + + if err != nil { + return fmt.Errorf("failed to watch DaemonSet: %w", err) + } + defer w.Stop() + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context canceled before DaemonSet was healthy") + case e, ok := <-w.ResultChan(): + d, ok := e.Object.(*appsv1.DaemonSet) + if !ok { + return fmt.Errorf("invalid object type: %T", d) + } + if d.Status.NumberReady == d.Status.DesiredNumberScheduled && d.Status.NumberUnavailable == 0 { + return nil + } + } + } +} diff --git a/tools/gvisor_k8s_tool/cmd/install/BUILD b/tools/gvisor_k8s_tool/cmd/install/BUILD new file mode 100644 index 000000000..54bacdb98 --- /dev/null +++ b/tools/gvisor_k8s_tool/cmd/install/BUILD @@ -0,0 +1,21 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "install", + srcs = ["install.go"], + deps = [ + "//pkg/log", + "//runsc/cmd/util", + "//runsc/flag", + "//tools/gvisor_k8s_tool/cluster", + "//tools/gvisor_k8s_tool/provider/clusterflag", + "//tools/gvisor_k8s_tool/spec", + "@com_github_google_subcommands//:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/cmd/install/install.go b/tools/gvisor_k8s_tool/cmd/install/install.go new file mode 100644 index 000000000..ef035b666 --- /dev/null +++ b/tools/gvisor_k8s_tool/cmd/install/install.go @@ -0,0 +1,137 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package install provides a function to install gVisor in a k8s cluster. +package install + +import ( + "context" + "fmt" + "strings" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/cluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/clusterflag" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/spec" +) + +// Install installs runsc from the given image in the given cluster. +func Install(ctx context.Context, c *cluster.Cluster, image string, options spec.InstallOptions) error { + ds := spec.RunscInstallDaemonSet(image, options) + // Delete a daemonset of the same name in the same namespace in case there is a collision. + if err := c.DeleteDaemonset(ctx, ds); err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to delete DaemonSet %q in namespace %q: %w", ds.Name, ds.Namespace, err) + } + + // Create the daemonset, but don't delete it so that we can get logs if there + // is a problem. + log.Infof("Creating DaemonSet %q in namespace %q...", ds.Name, ds.Namespace) + ds, err := c.CreateDaemonset(ctx, ds) + if err != nil { + return fmt.Errorf("failed to create DaemonSet %q in namespace %q: %w", ds.Name, ds.Namespace, err) + } + + log.Infof("Waiting for DaemonSet %q in namespace %q...", ds.Name, ds.Namespace) + if err := c.WaitForDaemonset(ctx, ds); err != nil { + return fmt.Errorf("failed to wait for daemonset: %v", err) + } + + log.Infof("DaemonSet %q in namespace %q complete.", ds.Name, ds.Namespace) + return nil +} + +// Command implements subcommands.Command. +type Command struct { + Image string + Cluster clusterflag.Flag + DaemonSetName string + DaemonSetNamespace string + PauseContainerImage string +} + +// Name implements subcommands.Command.Name. +func (*Command) Name() string { + return "install" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Command) Synopsis() string { + return "install gVisor in a kubernetes cluster" +} + +// Usage implements subcommands.Command.Usage. +func (*Command) Usage() string { + return `install --image= --cluster= + +Where "" is the name of the runsc installer image, +and contains information on how to connect +to the Kubernetes cluster to install to. + + can take the form of: + * --cluster=kube: + ... where "" is the name of a context in the + kubectl config file at $KUBECONFIG. + If $KUBECONFIG is not defined, it defaults to + $HOME/.kube/config. + If the context_name is empty, the default ("current") + context in the config file is used. + * --cluster=gke:projects//locations//clusters/ + ... where , and identify the project, + location, and name of the Google Kubernetes Engine cluster. + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (c *Command) SetFlags(f *flag.FlagSet) { + f.StringVar(&c.Image, "image", "", "runsc installer image") + f.Var(&c.Cluster, "cluster", "Kubernetes cluster to install runsc into") + f.StringVar(&c.DaemonSetName, "daemonset-name", "gvisor-runsc-installer", "name of the runsc installer DaemonSet; any previously-existing DaemonSet under this name will be deleted") + f.StringVar(&c.DaemonSetNamespace, "daemonset-namespace", spec.SystemNamespace, "namespace of the runsc installer DaemonSet") + f.StringVar(&c.PauseContainerImage, "pause-container-image", spec.PauseContainerImage, "container image that does nothing, used as placeholder in the DaemonSet") +} + +// Execute implements subcommands.Command.Execute. +// It installs gVisor in a Kubernetes cluster. +func (c *Command) Execute(ctx context.Context, f *flag.FlagSet, _ ...any) subcommands.ExitStatus { + if err := c.Cluster.Valid(); err != nil { + f.Usage() + return subcommands.ExitUsageError + } + clusterClient, err := c.Cluster.Cluster(ctx) + if err != nil { + util.Fatalf("Cannot initialize cluster client: %v", err) + } + var labels map[string]string + var nodeSelector map[string]string + switch c.Cluster.Provider { + case clusterflag.GKE: + labels = spec.GKESandboxNodeSelector + nodeSelector = spec.GKESandboxNodeSelector + default: + } + if err := Install(ctx, clusterClient, c.Image, spec.InstallOptions{ + DaemonSetName: c.DaemonSetName, + DaemonSetNamespace: c.DaemonSetNamespace, + PauseContainerImage: c.PauseContainerImage, + Labels: labels, + NodeSelector: nodeSelector, + }); err != nil { + util.Fatalf("Install failed: %v", err) + } + return subcommands.ExitSuccess +} diff --git a/tools/gvisor_k8s_tool/main.go b/tools/gvisor_k8s_tool/main.go new file mode 100644 index 000000000..6b335ea2f --- /dev/null +++ b/tools/gvisor_k8s_tool/main.go @@ -0,0 +1,43 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// gvisor_k8s_tool is a command-line tool to interact with gVisor in +// Kubernetes clusters. +package main + +import ( + "context" + "os" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/cmd/install" +) + +func registerCommands() { + subcommands.Register(subcommands.HelpCommand(), "") + subcommands.Register(subcommands.FlagsCommand(), "") + subcommands.Register(new(install.Command), "install") +} + +func main() { + registerCommands() + flag.Parse() + switch subcommands.Execute(context.Background()) { + case subcommands.ExitSuccess: + os.Exit(0) + default: + os.Exit(128) + } +} diff --git a/tools/gvisor_k8s_tool/provider/clusterflag/BUILD b/tools/gvisor_k8s_tool/provider/clusterflag/BUILD new file mode 100644 index 000000000..f5149eab3 --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/clusterflag/BUILD @@ -0,0 +1,17 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "clusterflag", + srcs = ["clusterflag.go"], + deps = [ + "//tools/gvisor_k8s_tool/cluster", + "//tools/gvisor_k8s_tool/provider/gke", + "//tools/gvisor_k8s_tool/provider/kubectl", + ], +) diff --git a/tools/gvisor_k8s_tool/provider/clusterflag/clusterflag.go b/tools/gvisor_k8s_tool/provider/clusterflag/clusterflag.go new file mode 100644 index 000000000..426442569 --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/clusterflag/clusterflag.go @@ -0,0 +1,132 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package clusterflag implements a flag.Value which can be used in commands +// to represent a Kubernetes cluster. +package clusterflag + +import ( + "context" + "fmt" + "strings" + + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/cluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/gke" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl" +) + +// Provider is a cluster provider. +type Provider string + +const ( + // Kubectl is a provider using a local kubectl config. + Kubectl Provider = "kube" + + // GKE is a provider using GKE. + GKE Provider = "gke" +) + +// String returns the provider name. +func (p Provider) String() string { + return string(p) +} + +// Valid returns whether the Provider is valid. +func (p Provider) Valid() bool { + switch p { + case Kubectl, GKE: + return true + default: + return false + } +} + +// ValidInfo validates whether the given info is valid for this provider. +func (p Provider) ValidInfo(info string) error { + switch p { + case Kubectl: + return nil + case GKE: + _, err := gke.NewClusterURL(info) + return err + default: + return fmt.Errorf("invalid provider: %q", p) + } +} + +// Flag contains the necessary information to connect to a Kubernetes cluster. +// Flag implements flag.Value. +type Flag struct { + StringVal string + Provider Provider + Info string +} + +// Valid checks if the flag values are valid. +func (f *Flag) Valid() error { + if !f.Provider.Valid() { + return fmt.Errorf("invalid provider: %q", f.Provider) + } + if err := f.Provider.ValidInfo(f.Info); err != nil { + return fmt.Errorf("invalid info for provider %q: %w", f.Provider, err) + } + return nil +} + +// String implements flag.Value.String. +func (f *Flag) String() string { + return f.StringVal +} + +// Get implements flag.Value.Get. +func (f *Flag) Get() any { + return f +} + +// Set implements flag.Value.Set. +// Set(String()) should be idempotent. +func (f *Flag) Set(s string) error { + f.StringVal = s + parts := strings.SplitN(s, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid format: %q (expected :)", s) + } + f2 := Flag{ + StringVal: s, + Provider: Provider(parts[0]), + Info: parts[1], + } + if err := f2.Valid(); err != nil { + return err + } + f.Provider = f2.Provider + f.Info = f2.Info + return nil +} + +// Cluster creates a cluster client. +func (f *Flag) Cluster(ctx context.Context) (*cluster.Cluster, error) { + switch f.Provider { + case Kubectl: + return kubectl.NewCluster(f.Info) + case GKE: + clusterURL, err := gke.NewClusterURL(f.Info) + if err != nil { + return nil, err + } + return gke.GetCluster(ctx, clusterURL) + default: + return nil, fmt.Errorf("invalid provider: %q", f.Provider) + } +} diff --git a/tools/gvisor_k8s_tool/provider/gke/BUILD b/tools/gvisor_k8s_tool/provider/gke/BUILD new file mode 100644 index 000000000..3671c2728 --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/gke/BUILD @@ -0,0 +1,17 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "gke", + srcs = ["gke.go"], + deps = [ + "//tools/gvisor_k8s_tool/cluster", + "//tools/gvisor_k8s_tool/util", + "@io_k8s_client_go//tools/clientcmd:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/provider/gke/gke.go b/tools/gvisor_k8s_tool/provider/gke/gke.go new file mode 100644 index 000000000..d398b53ff --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/gke/gke.go @@ -0,0 +1,95 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gke contains functions to interact with Google Kubernetes Engine. +package gke + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/cluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/util" + "k8s.io/client-go/tools/clientcmd" +) + +// ClusterURL represents a GKE cluster URL of the format: +// "projects/$MYPROJECT/locations/$CONTINENT-$LOCATION/clusters/$CLUSTER" +type ClusterURL struct { + ProjectID string + Location string + ClusterName string +} + +// String returns the cluster URL string. +func (c ClusterURL) String() string { + return fmt.Sprintf("projects/%s/locations/%s/clusters/%s", c.ProjectID, c.Location, c.ClusterName) +} + +// NewClusterURL parses the cluster URL. +func NewClusterURL(url string) (ClusterURL, error) { + if !strings.HasPrefix(url, "projects/") { + return ClusterURL{}, fmt.Errorf("invalid GKE cluster URL (expecting 'projects/MYPROJECT/locations/LOCATION/clusters/MYCLUSTER'): %q", url) + } + parts := strings.Split(url, "/") + if len(parts) != 6 { + return ClusterURL{}, fmt.Errorf("invalid GKE cluster URL (expecting 6 slash-delimited parts, got %d): %q", len(parts), url) + } + if parts[0] != "projects" || parts[2] != "locations" || parts[4] != "clusters" { + return ClusterURL{}, fmt.Errorf("invalid GKE cluster URL (expecting 'projects/MYPROJECT/locations/LOCATION/clusters/MYCLUSTER'): %q", url) + } + return ClusterURL{ + ProjectID: parts[1], + Location: parts[3], + ClusterName: parts[5], + }, nil +} + +// GetCluster returns a Kubernetes client for the given named cluster. +func GetCluster(ctx context.Context, clusterURL ClusterURL) (*cluster.Cluster, error) { + tmpDir, cleanTmp, err := util.TempDir() + defer cleanTmp() + if err != nil { + return nil, fmt.Errorf("failed to create temp dir: %w", err) + } + credFilePath := path.Join(tmpDir, fmt.Sprintf("%s.credential", clusterURL.ClusterName)) + f, err := os.Create(credFilePath) + if err != nil { + return nil, fmt.Errorf("failed to create cred file: %v", err) + } + f.Close() + cmd := exec.CommandContext(ctx, "gcloud", "--project", clusterURL.ProjectID, "container", "clusters", "get-credentials", clusterURL.ClusterName, "--location", clusterURL.Location) + cmd.Env = append(cmd.Environ(), fmt.Sprintf("KUBECONFIG=%s", credFilePath)) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to set credentials: %v; output: %s", err, string(out)) + } + configBytes, err := os.ReadFile(credFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read kubectl config file: %w", err) + } + kubeCfg, err := clientcmd.RESTConfigFromKubeConfig(configBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse kubectl config file: %w", err) + } + gkeCluster, err := cluster.New(kubeCfg) + if err != nil { + return nil, fmt.Errorf("failed to instantiate GKE cluster client: %w", err) + } + return gkeCluster, nil +} diff --git a/tools/gvisor_k8s_tool/provider/kubectl/BUILD b/tools/gvisor_k8s_tool/provider/kubectl/BUILD new file mode 100644 index 000000000..681d73888 --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/kubectl/BUILD @@ -0,0 +1,17 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "kubectl", + srcs = ["kubectl.go"], + deps = [ + "//pkg/log", + "//tools/gvisor_k8s_tool/cluster", + "@io_k8s_client_go//tools/clientcmd:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/provider/kubectl/kubectl.go b/tools/gvisor_k8s_tool/provider/kubectl/kubectl.go new file mode 100644 index 000000000..3a7b11988 --- /dev/null +++ b/tools/gvisor_k8s_tool/provider/kubectl/kubectl.go @@ -0,0 +1,63 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kubectl contains functions to interact with Kubernetes clusters +// controlled using kubectl configurations. +package kubectl + +import ( + "fmt" + "os" + "os/user" + "path" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/cluster" + "k8s.io/client-go/tools/clientcmd" +) + +// getKubeConfigPath returns the path to the kubectl config. +func getKubeConfigPath() (string, error) { + if envPath, ok := os.LookupEnv("KUBECONFIG"); ok { + return envPath, nil + } + me, err := user.Current() + if err != nil { + return "", fmt.Errorf("cannot get current user information: %w", err) + } + return path.Join(me.HomeDir, ".kube/config"), nil +} + +// NewCluster creates a new cluster client for the given context name and +// using the kubectl config defined in the KUBECONFIG environment variable. +// If the context name is empty, the default ("current") context is used. +func NewCluster(contextName string) (*cluster.Cluster, error) { + cfgPath, err := getKubeConfigPath() + if err != nil { + return nil, err + } + cfg, err := clientcmd.LoadFromFile(cfgPath) + if err != nil { + return nil, fmt.Errorf("cannot load kubectl config at %q: %w", cfgPath, err) + } + if contextName == "" { + contextName = cfg.CurrentContext + log.Infof("Using default kubectl context: %q", contextName) + } + restClient, err := clientcmd.NewNonInteractiveClientConfig(*cfg, contextName, nil, clientcmd.NewDefaultClientConfigLoadingRules()).ClientConfig() + if err != nil { + return nil, fmt.Errorf("cannot create REST client: %w", err) + } + return cluster.New(restClient) +} diff --git a/tools/gvisor_k8s_tool/spec/BUILD b/tools/gvisor_k8s_tool/spec/BUILD new file mode 100644 index 000000000..81c381c09 --- /dev/null +++ b/tools/gvisor_k8s_tool/spec/BUILD @@ -0,0 +1,19 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "spec", + srcs = ["spec.go"], + deps = [ + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + ], +) diff --git a/tools/gvisor_k8s_tool/spec/spec.go b/tools/gvisor_k8s_tool/spec/spec.go new file mode 100644 index 000000000..9a715f13f --- /dev/null +++ b/tools/gvisor_k8s_tool/spec/spec.go @@ -0,0 +1,128 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package spec contains Kubernetes object specifications for gVisor setup. +package spec + +import ( + "google.golang.org/protobuf/proto" + appsv1 "k8s.io/api/apps/v1" + v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // SystemNamespace is the name of the Kubernetes system namespace. + SystemNamespace = "kube-system" + // PauseContainerImage is the name of a container image that does nothing. + PauseContainerImage = "gcr.io/google-containers/pause" + // gvisorNodepoolKey the key for the label given to GKE Sandbox nodepools. + gvisorNodepoolKey = "sandbox.gke.io/runtime" + // gvisorRuntimeClass the runtimeClassName used for GKE Sandbox pods. + gvisorRuntimeClass = "gvisor" +) + +var ( + // GKESandboxNodeSelector selects GKE Sandbox nodes on GKE. + GKESandboxNodeSelector = map[string]string{gvisorNodepoolKey: gvisorRuntimeClass} +) + +// InstallOptions is the set of options to install runsc. +type InstallOptions struct { + DaemonSetNamespace string + DaemonSetName string + Labels map[string]string + NodeSelector map[string]string + PauseContainerImage string +} + +// RunscInstallDaemonSet returns a DaemonSet spec that installs runsc in +// Kubernetes. +func RunscInstallDaemonSet(image string, options InstallOptions) *appsv1.DaemonSet { + hpType := v13.HostPathDirectory + return &appsv1.DaemonSet{ + TypeMeta: v1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: options.DaemonSetName, + Namespace: options.DaemonSetNamespace, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &v1.LabelSelector{ + MatchLabels: options.Labels, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + Template: v13.PodTemplateSpec{ + ObjectMeta: v1.ObjectMeta{ + Labels: options.Labels, + }, + Spec: v13.PodSpec{ + Tolerations: []v13.Toleration{ + { + Operator: v13.TolerationOpExists, + }, + }, + HostPID: true, + InitContainers: []v13.Container{ + { + Name: options.DaemonSetName, + Image: image, + VolumeMounts: []v13.VolumeMount{ + { + Name: "host", + MountPath: "/host", + }, + }, + Resources: v13.ResourceRequirements{ + Requests: v13.ResourceList{ + v13.ResourceCPU: resource.MustParse("5m"), + v13.ResourceMemory: resource.MustParse("5Mi"), + }, + }, + SecurityContext: &v13.SecurityContext{ + Capabilities: &v13.Capabilities{ + Add: []v13.Capability{"CAP_SYS_ADMIN"}, + }, + Privileged: proto.Bool(true), + }, + }, + }, + Containers: []v13.Container{ + { + Name: "pause", + Image: options.PauseContainerImage, + }, + }, + NodeSelector: options.NodeSelector, + Volumes: []v13.Volume{ + { + Name: "host", + VolumeSource: v13.VolumeSource{ + HostPath: &v13.HostPathVolumeSource{ + Path: "/", + Type: &hpType, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/tools/gvisor_k8s_tool/util/BUILD b/tools/gvisor_k8s_tool/util/BUILD new file mode 100644 index 000000000..89b5cc348 --- /dev/null +++ b/tools/gvisor_k8s_tool/util/BUILD @@ -0,0 +1,12 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +go_library( + name = "util", + srcs = ["util.go"], +) diff --git a/tools/gvisor_k8s_tool/util/util.go b/tools/gvisor_k8s_tool/util/util.go new file mode 100644 index 000000000..bf146b060 --- /dev/null +++ b/tools/gvisor_k8s_tool/util/util.go @@ -0,0 +1,38 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package util contains utility functions for gvisor_k8s_tools. +package util + +import ( + "fmt" + "os" + "path" +) + +// TempDir creates and returns the path to a private temporary directory. +// The caller must call the given cleanup function to clean up the directory. +func TempDir() (string, func(), error) { + tempDir, err := os.MkdirTemp("", "gvisor_k8s_tool.*.tmp") + if err != nil { + return "", nil, err + } + // Create a private subdirectory (0700) within the temporary directory. + privateSubdir := path.Join(tempDir, "tmp") + if err := os.Mkdir(privateSubdir, 0700); err != nil { + os.RemoveAll(tempDir) + return "", nil, fmt.Errorf("cannot create subdir %q: %w", privateSubdir, err) + } + return privateSubdir, func() { os.RemoveAll(tempDir) }, nil +}