diff --git a/runsc/config/config.go b/runsc/config/config.go index a7cecb85e..41cd11a6e 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -108,6 +108,9 @@ type Config struct { // HostFifo controls permission to access host FIFO (or named pipes). HostFifo HostFifo `flag:"host-fifo"` + // HostSettings controls how host settings are handled. + HostSettings HostSettingsPolicy `flag:"host-settings"` + // Network indicates what type of network to use. Network NetworkType `flag:"network"` @@ -971,6 +974,86 @@ func (o *Overlay2) SubMountOverlayMedium() OverlayMedium { return o.medium } +// HostSettingsPolicy dictates how host settings should be handled. +type HostSettingsPolicy int + +// HostSettingsPolicy values. +const ( + // HostSettingsCheck checks the host settings. If any are not optimal, it + // will fail if any of them are mandatory, but will otherwise only log + // warnings. It never attempts to modify host settings. + HostSettingsCheck HostSettingsPolicy = iota + + // HostSettingsCheck checks the host settings. If any are not optimal, it + // will fail if any of them are mandatory, but will otherwise not log + // anything about non-mandatory settings. + // It never attempts to modify host settings. + HostSettingsCheckMandatory + + // HostSettingsIgnore does not check nor adjust any host settings. + // This is useful in case the host settings are already known to be + // optimal, or to avoid errors if `runsc` is running within a seccomp + // or AppArmor policy that prevents it from checking host settings. + HostSettingsIgnore + + // HostSettingsAdjust automatically adjusts host settings if they are not + // optimal. It will fail if any setting is mandatory but cannot be adjusted. + // For non-mandatory settings, it logs a warning if adjustment fails. + HostSettingsAdjust + + // HostSettingsEnforce automatically adjusts host settings if they are not + // optimal, and fails if adjustment of any setting fails. + HostSettingsEnforce +) + +// Set implements flag.Value. Set(String()) should be idempotent. +func (p *HostSettingsPolicy) Set(v string) error { + switch v { + case "check": + *p = HostSettingsCheck + case "check_mandatory": + *p = HostSettingsCheckMandatory + case "ignore": + *p = HostSettingsIgnore + case "adjust": + *p = HostSettingsAdjust + case "enforce": + *p = HostSettingsEnforce + default: + return fmt.Errorf("invalid host settings policy %q", v) + } + return nil +} + +// Ptr returns a pointer to `p`. +// Useful in flag declaration line. +func (p HostSettingsPolicy) Ptr() *HostSettingsPolicy { + return &p +} + +// Get implements flag.Get. +func (p *HostSettingsPolicy) Get() any { + return *p +} + +// String implements flag.String. +func (p HostSettingsPolicy) String() string { + switch p { + case HostSettingsCheck: + return "check" + case HostSettingsCheckMandatory: + return "check_mandatory" + case HostSettingsAdjust: + return "adjust" + case HostSettingsIgnore: + return "ignore" + case HostSettingsEnforce: + return "enforce" + default: + panic(fmt.Sprintf("Invalid host settings policy %d", p)) + } +} + // XDP holds configuration for whether and how to use XDP. type XDP struct { Mode XDPMode diff --git a/runsc/config/flags.go b/runsc/config/flags.go index d801a8b9b..2783a61c6 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -89,6 +89,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("oci-seccomp", false, "Enables loading OCI seccomp filters inside the sandbox.") flagSet.Bool("enable-core-tags", false, "enables core tagging. Requires host linux kernel >= 5.14.") flagSet.String("pod-init-config", "", "path to configuration file with additional steps to take during pod creation.") + flagSet.Var(HostSettingsCheck.Ptr(), "host-settings", "how to handle non-optimal host kernel settings: check (default, advisory-only), ignore (do not check), adjust (best-effort auto-adjustment), or enforce (auto-adjustment must succeed).") // Flags that control sandbox runtime behavior: MM related. flagSet.Bool("app-huge-pages", true, "enable use of huge pages for application memory; requires /sys/kernel/mm/transparent_hugepage/shmem_enabled = advise") diff --git a/runsc/hostsettings/BUILD b/runsc/hostsettings/BUILD new file mode 100644 index 000000000..a0a4fbdd0 --- /dev/null +++ b/runsc/hostsettings/BUILD @@ -0,0 +1,18 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "hostsettings", + srcs = [ + "hostsettings.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/log", + "//runsc/config", + ], +) diff --git a/runsc/hostsettings/hostsettings.go b/runsc/hostsettings/hostsettings.go new file mode 100644 index 000000000..4f9032048 --- /dev/null +++ b/runsc/hostsettings/hostsettings.go @@ -0,0 +1,307 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hostsettings provides suggestions or adjustments for host kernel +// settings to improve runsc performance, stability, or security. +package hostsettings + +import ( + "fmt" + "os" + "strconv" + "strings" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/runsc/config" +) + +// Handle deals with host settings according to the given policy. +func Handle(conf *config.Config) error { + switch conf.HostSettings { + case config.HostSettingsIgnore: + return nil + case config.HostSettingsCheck, config.HostSettingsCheckMandatory: + deltas, errs := check(conf) + for _, err := range errs { + log.Warningf("Checking host settings: error: %v", err) + } + for _, delta := range deltas { + if delta.Mandatory { + return fmt.Errorf("host setting %q (currently: %q) is not supported (%s); must change it to %q for runsc to work", delta.Name, delta.FromValue, delta.Purpose, delta.ToValue) + } + if conf.HostSettings == config.HostSettingsCheck { + log.Warningf("Host setting %q (currently: %q) is not optimal (%s); it is recommended to change it to %q", delta.Name, delta.FromValue, delta.Purpose, delta.ToValue) + } + } + case config.HostSettingsAdjust, config.HostSettingsEnforce: + deltas, errs := check(conf) + for _, err := range errs { + log.Warningf("Host settings: error: %v", err) + } + for _, delta := range deltas { + log.Warningf("Host setting %q (currently: %q) is not optimal (%s); attempting to change it to %q...", delta.Name, delta.FromValue, delta.Purpose, delta.ToValue) + err := delta.Apply() + // We handle the nil error case first here as it simplifies control flow. + if err == nil { + log.Warningf("Host setting %q changed from %q to %q (%s)", delta.Name, delta.FromValue, delta.ToValue, delta.Purpose) + continue + } + if delta.Mandatory { + return fmt.Errorf("host setting %q (currently: %q) is not supported (%s), and trying to change it to %q failed (%w); bailing out as this is necessary for runsc to work", delta.Name, delta.FromValue, delta.Purpose, delta.ToValue, err) + } + if conf.HostSettings == config.HostSettingsEnforce { + return fmt.Errorf("failed to adjust %s to %q: %w", delta.Name, delta.ToValue, err) + } + log.Warningf("Host settings: failed to adjust %s to %q: %v; continuing anyway.", delta.Name, delta.ToValue, err) + } + default: + return fmt.Errorf("invalid host settings policy %d", conf.HostSettings) + } + return nil +} + +// Setting is a host setting to check or adjust. +type Setting interface { + // Name is the name of the setting. In most cases this should be the + // full path of the setting, either under /sys or /proc/sys. + // This can be a non-path for settings that are not controlled via files, + // such as kernel command-line parameters or SELinux labels. + Name() string + + // Delta checks whether the current value of the setting is optimal. + // If already optimal, it returns a nil Delta. + Delta() (*Delta, error) +} + +// Delta is a change to make to a host setting. +type Delta struct { + // Name of the setting being changed. + Name string + + // FromValue is the value of the setting before applying the delta. + FromValue string + + // ToValue is the value of the setting after applying the delta. + ToValue string + + // Mandatory indicates whether the delta *must* be applied. + // This should be set in cases where `runsc` will fail completely + // if the delta is not applied. + Mandatory bool + + // Purpose achieved by applying the delta. + Purpose string + + // Apply applies the delta. + Apply func() error +} + +// pathSetting implements `Setting` for kernel settings controlled via files. +type pathSetting struct { + // path is the full path of the setting, either under /sys or /proc/sys. + path string + + // mightNotExist indicates that the file may not exist on all systems. + // If the path does not exist, no delta is returned for this setting. + mightNotExist bool + + // mightLackReadPerm indicates that the file may not be readable. + // If the file cannot be read with a permission error, and we are running + // in rootless mode, a message is logged but no delta is returned for this + // setting. + mightLackReadPerm bool + + // purpose describes the intent behind setting this value. + purpose string + + // `delta` returns an empty string if the current value is optimal, + // or a value to write to `path` if not optimal. + // It also returns whether the delta is mandatory. + delta func(conf *config.Config, current string) (string, bool, error) +} + +// Name implements `Setting.Name`. +func (s pathSetting) Name() string { + return s.path +} + +// Delta implements `Setting.Delta`. +func (s pathSetting) Delta(conf *config.Config) (*Delta, error) { + currentBytes, err := os.ReadFile(s.path) + if err != nil { + if s.mightNotExist && os.IsNotExist(err) { + return nil, nil + } + if s.mightLackReadPerm && os.IsPermission(err) { + log.Infof("Host settings: Cannot check if %q is optimal (%s): %v; continuing anyway.", s.path, s.purpose, err) + return nil, nil + } + return nil, fmt.Errorf("cannot read %q: %w", s.path, err) + } + currentValue := strings.TrimSpace(string(currentBytes)) + newValue, mandatory, err := s.delta(conf, currentValue) + if err != nil { + return nil, err + } + if newValue == "" { + return nil, nil + } + return &Delta{ + Name: s.path, + FromValue: currentValue, + ToValue: newValue, + Mandatory: mandatory, + Purpose: s.purpose, + Apply: func() error { + if err := os.WriteFile(s.path, []byte(newValue), 0644); err != nil { + return fmt.Errorf("cannot write %q: %w", s.path, err) + } + // Double-check that the value has been written properly. + newDelta, err := s.Delta(conf) + if err != nil { + return fmt.Errorf("cannot re-read %q after writing: %w", s.path, err) + } + if newDelta != nil { + return fmt.Errorf("writing to %q did not change the observed value: wrote %q but read back %q", s.path, newValue, newDelta.FromValue) + } + return nil + }, + }, nil +} + +// checks checks the host settings and returns any deltas to do or errors +// occurred while checking. +func check(conf *config.Config) ([]*Delta, []error) { + log.Debugf("Checking host settings") + settings := []pathSetting{ + { + path: "/sys/kernel/mm/transparent_hugepage/shmem_enabled", + purpose: "turning on transparent hugepages support in shmem increases memory allocation performance", + delta: func(conf *config.Config, current string) (string, bool, error) { + // /sys/kernel/mm/transparent_hugepage/shmem_enabled is formatted like: + // `always within_size advise [never] deny force`. + if strings.Contains(current, "[always]") || strings.Contains(current, "[advise]") || strings.Contains(current, "[force]") || strings.Contains(current, "[within_size]") { + return "", false, nil + } + return "advise", false, nil + }, + }, + { + path: "/proc/sys/vm/max_map_count", + purpose: "increasing max_map_count decreases the likelihood of host VMA exhaustion", + delta: func(conf *config.Config, current string) (string, bool, error) { + const recommendedMaxMapCount = 4194304 + currentVal, err := strconv.Atoi(strings.TrimSpace(current)) + if err != nil { + return "", false, fmt.Errorf("failed to parse %q as an integer: %v", current, err) + } + if currentVal >= recommendedMaxMapCount { + return "", false, nil + } + return strconv.Itoa(recommendedMaxMapCount), false, nil + }, + }, + { + path: "/proc/sys/kernel/yama/ptrace_scope", + mightNotExist: true, + purpose: "ptrace_scope=1 enables the systrap and ptrace platforms to work, as well as --directfs=false", + delta: func(conf *config.Config, current string) (string, bool, error) { + // systrap and ptrace require this because they use ptrace. + // DirectFS=false requires this as well: https://github.com/google/gvisor/issues/9006 + if conf.Platform != "systrap" && conf.Platform != "ptrace" && conf.DirectFS { + // Setting not required. + return "", false, nil + } + current = strings.TrimSpace(current) + if current == "0" || current == "1" { + return "", false, nil + } + return "1", true, nil + }, + }, + { + path: "/proc/sys/user/max_user_namespaces", + purpose: "runsc requires creating at least 2 new user namespaces and may run into the limit when creating multiple containers", + delta: func(conf *config.Config, current string) (string, bool, error) { + const recommendedMaxUserNamespaces = 256 + currentVal, err := strconv.Atoi(strings.TrimSpace(current)) + if err != nil { + return "", false, fmt.Errorf("failed to parse %q as an integer: %v", current, err) + } + if currentVal >= recommendedMaxUserNamespaces { + return "", false, nil + } + if currentVal < 2 { + // If less than two, runsc will definitely fail (it needs one for + // the gofer and one for the sandbox), so mark as mandatory. + return strconv.Itoa(recommendedMaxUserNamespaces), true, nil + } + return strconv.Itoa(recommendedMaxUserNamespaces), false, nil + }, + }, + { + path: "/proc/sys/kernel/unprivileged_userns_clone", + mightNotExist: true, + purpose: "in rootless mode, runsc requires the ability to create new user namespaces without privileges", + delta: func(conf *config.Config, current string) (string, bool, error) { + if !conf.Rootless { + // Setting not required. + return "", false, nil + } + currentVal, err := strconv.Atoi(strings.TrimSpace(current)) + if err != nil { + return "", false, fmt.Errorf("failed to parse %q as an integer: %v", current, err) + } + if currentVal != 0 { + return "", false, nil + } + return "1", true, nil + }, + }, + { + path: "/proc/sys/kernel/unprivileged_userns_apparmor_policy", + mightNotExist: true, + mightLackReadPerm: true, + purpose: "in rootless mode, runsc requires the ability to create new user namespaces without privileges", + delta: func(conf *config.Config, current string) (string, bool, error) { + if !conf.Rootless { + // Setting not required. + return "", false, nil + } + currentVal, err := strconv.Atoi(strings.TrimSpace(current)) + if err != nil { + return "", false, fmt.Errorf("failed to parse %q as an integer: %v", current, err) + } + if currentVal != 0 { + return "", false, nil + } + return "1", true, nil + }, + }, + } + var deltas []*Delta + var errs []error + for _, setting := range settings { + log.Debugf("Checking host setting: %s", setting.Name()) + delta, err := setting.Delta(conf) + if err != nil { + errs = append(errs, fmt.Errorf("failed to check %s: %v", setting.Name(), err)) + continue + } + if delta != nil { + deltas = append(deltas, delta) + } + } + return deltas, errs +} diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index 5cea466e7..e926ce5cb 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -50,6 +50,7 @@ go_library( "//runsc/config", "//runsc/console", "//runsc/donation", + "//runsc/hostsettings", "//runsc/profile", "//runsc/sandbox/bpf", "//runsc/specutils", diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 0b53d4f45..ee5ef3ce1 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -60,6 +60,7 @@ import ( "gvisor.dev/gvisor/runsc/config" "gvisor.dev/gvisor/runsc/console" "gvisor.dev/gvisor/runsc/donation" + "gvisor.dev/gvisor/runsc/hostsettings" "gvisor.dev/gvisor/runsc/profile" "gvisor.dev/gvisor/runsc/specutils" "gvisor.dev/gvisor/runsc/starttime" @@ -401,6 +402,9 @@ func (s *Sandbox) CreateSubcontainer(conf *config.Config, cid string, tty *os.Fi // StartRoot starts running the root container process inside the sandbox. func (s *Sandbox) StartRoot(conf *config.Config) error { + if err := hostsettings.Handle(conf); err != nil { + return fmt.Errorf("host settings: %w (use --host-settings=ignore to bypass)", err) + } pid := s.Pid.load() log.Debugf("Start root sandbox %q, PID: %d", s.ID, pid) conn, err := s.sandboxConnect() @@ -462,6 +466,10 @@ func (s *Sandbox) StartSubcontainer(spec *specs.Spec, conf *config.Config, cid s // Restore sends the restore call for a container in the sandbox. func (s *Sandbox) Restore(conf *config.Config, cid string, imagePath string, direct, background bool) error { + if err := hostsettings.Handle(conf); err != nil { + return fmt.Errorf("host settings: %w (use --host-settings=ignore to bypass)", err) + } + log.Debugf("Restore sandbox %q from path %q", s.ID, imagePath) stateFileName := path.Join(imagePath, boot.CheckpointStateFileName) diff --git a/test/runner/main.go b/test/runner/main.go index b8f4b5659..b208d9ca3 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -329,6 +329,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { "-platform", *platform, "-file-access", *fileAccess, "-gvisor-gro", + "-host-settings=check_mandatory", } if *network == "host" && !testutil.TestEnvSupportsNetAdmin {