From 28688eb1a5ab1531eedf7561bc072b3376923dfb Mon Sep 17 00:00:00 2001 From: Lucas Manning Date: Thu, 24 Feb 2022 11:27:11 -0800 Subject: [PATCH] Support cgroup Install() through systemd. This change implements Install() though systemd and adds associated tests. The properties saved by the systemdCgroup object will be applied during Join(), which will be added in a later change. PiperOrigin-RevId: 430751223 --- runsc/cgroup/BUILD | 8 ++ runsc/cgroup/systemd.go | 172 +++++++++++++++++++++++++++++ runsc/cgroup/systemd_test.go | 202 +++++++++++++++++++++++++++++++++++ 3 files changed, 382 insertions(+) create mode 100644 runsc/cgroup/systemd.go create mode 100644 runsc/cgroup/systemd_test.go diff --git a/runsc/cgroup/BUILD b/runsc/cgroup/BUILD index c0148ac52..a8a085761 100644 --- a/runsc/cgroup/BUILD +++ b/runsc/cgroup/BUILD @@ -7,12 +7,15 @@ go_library( srcs = [ "cgroup.go", "cgroup_v2.go", + "systemd.go", ], visibility = ["//:sandbox"], deps = [ "//pkg/cleanup", "//pkg/log", "@com_github_cenkalti_backoff//:go_default_library", + "@com_github_coreos_go_systemd_v22//dbus:go_default_library", + "@com_github_godbus_dbus_v5//:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", "@org_golang_x_sync//errgroup:go_default_library", "@org_golang_x_sys//unix:go_default_library", @@ -25,11 +28,16 @@ go_test( srcs = [ "cgroup_test.go", "cgroup_v2_test.go", + "systemd_test.go", ], library = ":cgroup", tags = ["local"], deps = [ "//pkg/test/testutil", + "@com_github_coreos_go_systemd_v22//dbus:go_default_library", + "@com_github_godbus_dbus_v5//:go_default_library", + "@com_github_google_go_cmp//cmp:go_default_library", + "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", ], ) diff --git a/runsc/cgroup/systemd.go b/runsc/cgroup/systemd.go new file mode 100644 index 000000000..8e974324a --- /dev/null +++ b/runsc/cgroup/systemd.go @@ -0,0 +1,172 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cgroup + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + dbus "github.com/godbus/dbus/v5" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// ErrBadResourceSpec indicates that a cgroupSystemd function was +// passed a specs.LinuxResources object that is impossible or illegal +// to process. +var ErrBadResourceSpec = errors.New("misconfigured resource spec") + +// cgroupSystemd represents a cgroup managed by systemd. +type cgroupSystemd struct { + // Name is the name of the of the systemd scope that controls the cgroups. + Name string `json:"name"` + // Mountpoint is the unified mount point of cgroupV2. + Mountpoint string `json:"mountpoint"` + // Path is the relative path to the unified mountpoint. + Path string `json:"path"` + // Controllers is the list of supported controllers. + Controllers []string `json:"controllers"` + // OwnedPaths is the list of owned paths created when installing this cgroup. + OwnedPaths []string `json:"owned_paths"` + + properties []systemdDbus.Property + dbusConn *systemdDbus.Conn +} + +// Install creates and configures a scope unit with the specified resource +// limits. +func (c *cgroupSystemd) Install(res *specs.LinuxResources) error { + slice := path.Base(c.Path) + ext := path.Ext(slice) + if ext != ".slice" { + return fmt.Errorf("invalid systemd path %s does not end in a parent slice: %w", c.Path, ErrInvalidGroupPath) + } + c.properties = append(c.properties, systemdDbus.PropSlice(slice)) + c.properties = append(c.properties, systemdDbus.PropDescription("runsc container "+c.Name)) + pid := os.Getpid() + c.properties = append(c.properties, systemdDbus.PropPids(uint32(pid))) + // We always want proper accounting for the container for reporting resource + // usage. + c.addProp("MemoryAccounting", true) + c.addProp("CPUAccounting", true) + c.addProp("TasksAccounting", true) + c.addProp("IOAccounting", true) + // Delegate must be true so that the container can manage its own cgroups. + c.addProp("Delegate", true) + return c.genResourceControl(res) +} + +// MakePath builds a path to the given controller. +func (c *cgroupSystemd) MakePath(string) string { + return filepath.Join(c.Mountpoint, c.Path) +} + +func (c *cgroupSystemd) genResourceControl(res *specs.LinuxResources) error { + if res == nil { + return nil + } + var ( + mem = res.Memory + cpu = res.CPU + io = res.BlockIO + ) + if res.Pids != nil { + c.addProp("TasksMax", res.Pids.Limit) + } + if mem != nil { + if mem.Swap != nil { + if mem.Limit == nil { + return ErrBadResourceSpec + } + swap, err := convertMemorySwapToCgroupV2Value(*mem.Swap, *mem.Limit) + if err != nil { + return err + } + c.addProp("MemorySwapMax", strconv.FormatInt(swap, 10)) + } + if mem.Limit != nil { + c.addProp("MemoryMax", *mem.Limit) + } + if mem.Reservation != nil { + c.addProp("MemoryLow", *mem.Reservation) + } + } + if cpu != nil { + if cpu.Shares != nil { + weight := convertCPUSharesToCgroupV2Value(*cpu.Shares) + if weight != 0 { + c.addProp("CPUShares", weight) + } + } + + if cpu.Quota != nil && *cpu.Quota > 0 { + c.addProp("CPUQuota", strconv.FormatInt(*cpu.Quota, 10)+"%") + } + var period uint64 + if cpu.Period != nil && *cpu.Period != 0 { + period = *cpu.Period + } else { + period = defaultPeriod + } + // period is in microseconds, so we have to divide by 10 to convert + // to the milliseconds that systemd expects. + c.addProp("CPUQuotaPeriodSec", strconv.FormatUint(period/10, 10)+"ms") + if cpu.Cpus != "" { + c.addProp("AllowedCPUs", cpu.Cpus) + } + if cpu.Mems != "" { + c.addProp("AllowedMemoryNodes", cpu.Mems) + } + } + if io != nil { + if io.Weight != nil { + c.addProp("IOWeight", *io.Weight) + } + for _, dev := range io.WeightDevice { + val := fmt.Sprintf("%d:%d %d", dev.Major, dev.Minor, *dev.Weight) + c.addProp("IODevice", val) + } + c.addIOProps("IOReadBandwidth", io.ThrottleReadBpsDevice) + c.addIOProps("IOWriteBandwidth", io.ThrottleWriteBpsDevice) + c.addIOProps("IOReadIOPS", io.ThrottleReadIOPSDevice) + c.addIOProps("IOWriteIOPS", io.ThrottleWriteIOPSDevice) + } + return nil +} + +func (c *cgroupSystemd) addIOProps(name string, devs []specs.LinuxThrottleDevice) { + for _, dev := range devs { + val := fmt.Sprintf("%d:%d %d", dev.Major, dev.Minor, dev.Rate) + c.addProp(name, val) + } +} + +func (c *cgroupSystemd) addProp(name string, value interface{}) { + if value == nil { + return + } + c.properties = append(c.properties, newProp(name, value)) +} + +func newProp(name string, units interface{}) systemdDbus.Property { + return systemdDbus.Property{ + Name: name, + Value: dbus.MakeVariant(units), + } +} diff --git a/runsc/cgroup/systemd_test.go b/runsc/cgroup/systemd_test.go new file mode 100644 index 000000000..0ce39cd04 --- /dev/null +++ b/runsc/cgroup/systemd_test.go @@ -0,0 +1,202 @@ +// Copyright The runc Authors. +// Copyright The containerd Authors. +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cgroup + +import ( + "errors" + "path/filepath" + "strconv" + "testing" + + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + dbus "github.com/godbus/dbus/v5" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +var defaultProps = []systemdDbus.Property{} + +func TestInstall(t *testing.T) { + for _, tc := range []struct { + name string + res *specs.LinuxResources + cgroupPath string + wantProps []systemdDbus.Property + err error + }{ + { + name: "bad parent", + res: nil, + cgroupPath: "not_a_slice", + err: ErrInvalidGroupPath, + }, + { + name: "no limits", + res: nil, + wantProps: []systemdDbus.Property{ + {"Slice", dbus.MakeVariant("parent.slice")}, + {Name: "Description", Value: dbus.MakeVariant("runsc container ")}, + {Name: "MemoryAccounting", Value: dbus.MakeVariant(true)}, + {Name: "CPUAccounting", Value: dbus.MakeVariant(true)}, + {Name: "TasksAccounting", Value: dbus.MakeVariant(true)}, + {Name: "IOAccounting", Value: dbus.MakeVariant(true)}, + {Name: "Delegate", Value: dbus.MakeVariant(true)}, + }, + cgroupPath: "parent.slice", + }, + { + name: "memory", + res: &specs.LinuxResources{ + Memory: &specs.LinuxMemory{ + Limit: int64Ptr(1), + Swap: int64Ptr(2), + Reservation: int64Ptr(3), + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"MemoryMax", dbus.MakeVariant(int64(1))}, + {"MemoryLow", dbus.MakeVariant(int64(3))}, + {"MemorySwapMax", dbus.MakeVariant("1")}, + }, + }, + { + name: "memory no limit", + res: &specs.LinuxResources{ + Memory: &specs.LinuxMemory{ + Swap: int64Ptr(1), + }, + }, + err: ErrBadResourceSpec, + cgroupPath: "parent.slice", + }, + { + name: "cpu defaults", + res: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Shares: uint64Ptr(0), + Quota: int64Ptr(0), + Period: uint64Ptr(0), + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"CPUQuotaPeriodSec", dbus.MakeVariant(strconv.FormatUint(defaultPeriod/10, 10) + "ms")}, + }, + }, + { + name: "cpu", + res: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Shares: uint64Ptr(1), + Period: uint64Ptr(20), + Quota: int64Ptr(3), + Cpus: "4", + Mems: "5", + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"CPUShares", dbus.MakeVariant(convertCPUSharesToCgroupV2Value(1))}, + {"CPUQuotaPeriodSec", dbus.MakeVariant("2ms")}, + {"CPUQuota", dbus.MakeVariant("3%")}, + {"AllowedCPUs", dbus.MakeVariant("4")}, + {"AllowedMemoryNodes", dbus.MakeVariant("5")}, + }, + }, + { + name: "io", + res: &specs.LinuxResources{ + BlockIO: &specs.LinuxBlockIO{ + Weight: uint16Ptr(1), + WeightDevice: []specs.LinuxWeightDevice{ + makeLinuxWeightDevice(2, 3, uint16Ptr(4), uint16Ptr(0)), + makeLinuxWeightDevice(5, 6, uint16Ptr(7), uint16Ptr(0)), + }, + ThrottleReadBpsDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(8, 9, 10), + makeLinuxThrottleDevice(11, 12, 13), + }, + ThrottleWriteBpsDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(14, 15, 16), + }, + ThrottleReadIOPSDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(17, 18, 19), + }, + ThrottleWriteIOPSDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(20, 21, 22), + }, + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"IOWeight", dbus.MakeVariant(uint16(1))}, + {"IODevice", dbus.MakeVariant("2:3 4")}, + {"IODevice", dbus.MakeVariant("5:6 7")}, + {"IOReadBandwidth", dbus.MakeVariant("8:9 10")}, + {"IOReadBandwidth", dbus.MakeVariant("11:12 13")}, + {"IOWriteBandwidth", dbus.MakeVariant("14:15 16")}, + {"IOReadIOPS", dbus.MakeVariant("17:18 19")}, + {"IOWriteIOPS", dbus.MakeVariant("20:21 22")}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := testutil.TmpDir() + testPath := filepath.Join(dir, tc.cgroupPath) + + cg := cgroupSystemd{ + Path: testPath, + } + err := cg.Install(tc.res) + if !errors.Is(err, tc.err) { + t.Fatalf("Wrong error, got: %s, want: %s", tc.err, err) + } + cmper := cmp.Comparer(func(a dbus.Variant, b dbus.Variant) bool { + return a.String() == b.String() + }) + sorter := cmpopts.SortSlices(func(a systemdDbus.Property, b systemdDbus.Property) bool { + return (a.Name + a.Value.String()) > (b.Name + b.Value.String()) + }) + filteredProps := filterProperties(cg.properties, tc.wantProps) + if diff := cmp.Diff(filteredProps, tc.wantProps, cmper, sorter); diff != "" { + t.Errorf("cgroup properties list diff %s", diff) + } + }) + } +} + +// filterProperties filters the list of properties in got to ones with +// the names of properties specified in want. +func filterProperties(got []systemdDbus.Property, want []systemdDbus.Property) []systemdDbus.Property { + if want == nil { + return nil + } + filterMap := map[string]interface{}{} + for _, prop := range want { + filterMap[prop.Name] = nil + } + filtered := []systemdDbus.Property{} + for _, prop := range got { + if _, ok := filterMap[prop.Name]; ok { + filtered = append(filtered, prop) + } + } + return filtered +}