diff --git a/runsc/cgroup/BUILD b/runsc/cgroup/BUILD index c0148ac52..a8a085761 100644 --- a/runsc/cgroup/BUILD +++ b/runsc/cgroup/BUILD @@ -7,12 +7,15 @@ go_library( srcs = [ "cgroup.go", "cgroup_v2.go", + "systemd.go", ], visibility = ["//:sandbox"], deps = [ "//pkg/cleanup", "//pkg/log", "@com_github_cenkalti_backoff//:go_default_library", + "@com_github_coreos_go_systemd_v22//dbus:go_default_library", + "@com_github_godbus_dbus_v5//:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", "@org_golang_x_sync//errgroup:go_default_library", "@org_golang_x_sys//unix:go_default_library", @@ -25,11 +28,16 @@ go_test( srcs = [ "cgroup_test.go", "cgroup_v2_test.go", + "systemd_test.go", ], library = ":cgroup", tags = ["local"], deps = [ "//pkg/test/testutil", + "@com_github_coreos_go_systemd_v22//dbus:go_default_library", + "@com_github_godbus_dbus_v5//:go_default_library", + "@com_github_google_go_cmp//cmp:go_default_library", + "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", ], ) diff --git a/runsc/cgroup/systemd.go b/runsc/cgroup/systemd.go new file mode 100644 index 000000000..8e974324a --- /dev/null +++ b/runsc/cgroup/systemd.go @@ -0,0 +1,172 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cgroup + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + dbus "github.com/godbus/dbus/v5" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// ErrBadResourceSpec indicates that a cgroupSystemd function was +// passed a specs.LinuxResources object that is impossible or illegal +// to process. +var ErrBadResourceSpec = errors.New("misconfigured resource spec") + +// cgroupSystemd represents a cgroup managed by systemd. +type cgroupSystemd struct { + // Name is the name of the of the systemd scope that controls the cgroups. + Name string `json:"name"` + // Mountpoint is the unified mount point of cgroupV2. + Mountpoint string `json:"mountpoint"` + // Path is the relative path to the unified mountpoint. + Path string `json:"path"` + // Controllers is the list of supported controllers. + Controllers []string `json:"controllers"` + // OwnedPaths is the list of owned paths created when installing this cgroup. + OwnedPaths []string `json:"owned_paths"` + + properties []systemdDbus.Property + dbusConn *systemdDbus.Conn +} + +// Install creates and configures a scope unit with the specified resource +// limits. +func (c *cgroupSystemd) Install(res *specs.LinuxResources) error { + slice := path.Base(c.Path) + ext := path.Ext(slice) + if ext != ".slice" { + return fmt.Errorf("invalid systemd path %s does not end in a parent slice: %w", c.Path, ErrInvalidGroupPath) + } + c.properties = append(c.properties, systemdDbus.PropSlice(slice)) + c.properties = append(c.properties, systemdDbus.PropDescription("runsc container "+c.Name)) + pid := os.Getpid() + c.properties = append(c.properties, systemdDbus.PropPids(uint32(pid))) + // We always want proper accounting for the container for reporting resource + // usage. + c.addProp("MemoryAccounting", true) + c.addProp("CPUAccounting", true) + c.addProp("TasksAccounting", true) + c.addProp("IOAccounting", true) + // Delegate must be true so that the container can manage its own cgroups. + c.addProp("Delegate", true) + return c.genResourceControl(res) +} + +// MakePath builds a path to the given controller. +func (c *cgroupSystemd) MakePath(string) string { + return filepath.Join(c.Mountpoint, c.Path) +} + +func (c *cgroupSystemd) genResourceControl(res *specs.LinuxResources) error { + if res == nil { + return nil + } + var ( + mem = res.Memory + cpu = res.CPU + io = res.BlockIO + ) + if res.Pids != nil { + c.addProp("TasksMax", res.Pids.Limit) + } + if mem != nil { + if mem.Swap != nil { + if mem.Limit == nil { + return ErrBadResourceSpec + } + swap, err := convertMemorySwapToCgroupV2Value(*mem.Swap, *mem.Limit) + if err != nil { + return err + } + c.addProp("MemorySwapMax", strconv.FormatInt(swap, 10)) + } + if mem.Limit != nil { + c.addProp("MemoryMax", *mem.Limit) + } + if mem.Reservation != nil { + c.addProp("MemoryLow", *mem.Reservation) + } + } + if cpu != nil { + if cpu.Shares != nil { + weight := convertCPUSharesToCgroupV2Value(*cpu.Shares) + if weight != 0 { + c.addProp("CPUShares", weight) + } + } + + if cpu.Quota != nil && *cpu.Quota > 0 { + c.addProp("CPUQuota", strconv.FormatInt(*cpu.Quota, 10)+"%") + } + var period uint64 + if cpu.Period != nil && *cpu.Period != 0 { + period = *cpu.Period + } else { + period = defaultPeriod + } + // period is in microseconds, so we have to divide by 10 to convert + // to the milliseconds that systemd expects. + c.addProp("CPUQuotaPeriodSec", strconv.FormatUint(period/10, 10)+"ms") + if cpu.Cpus != "" { + c.addProp("AllowedCPUs", cpu.Cpus) + } + if cpu.Mems != "" { + c.addProp("AllowedMemoryNodes", cpu.Mems) + } + } + if io != nil { + if io.Weight != nil { + c.addProp("IOWeight", *io.Weight) + } + for _, dev := range io.WeightDevice { + val := fmt.Sprintf("%d:%d %d", dev.Major, dev.Minor, *dev.Weight) + c.addProp("IODevice", val) + } + c.addIOProps("IOReadBandwidth", io.ThrottleReadBpsDevice) + c.addIOProps("IOWriteBandwidth", io.ThrottleWriteBpsDevice) + c.addIOProps("IOReadIOPS", io.ThrottleReadIOPSDevice) + c.addIOProps("IOWriteIOPS", io.ThrottleWriteIOPSDevice) + } + return nil +} + +func (c *cgroupSystemd) addIOProps(name string, devs []specs.LinuxThrottleDevice) { + for _, dev := range devs { + val := fmt.Sprintf("%d:%d %d", dev.Major, dev.Minor, dev.Rate) + c.addProp(name, val) + } +} + +func (c *cgroupSystemd) addProp(name string, value interface{}) { + if value == nil { + return + } + c.properties = append(c.properties, newProp(name, value)) +} + +func newProp(name string, units interface{}) systemdDbus.Property { + return systemdDbus.Property{ + Name: name, + Value: dbus.MakeVariant(units), + } +} diff --git a/runsc/cgroup/systemd_test.go b/runsc/cgroup/systemd_test.go new file mode 100644 index 000000000..0ce39cd04 --- /dev/null +++ b/runsc/cgroup/systemd_test.go @@ -0,0 +1,202 @@ +// Copyright The runc Authors. +// Copyright The containerd Authors. +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cgroup + +import ( + "errors" + "path/filepath" + "strconv" + "testing" + + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + dbus "github.com/godbus/dbus/v5" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +var defaultProps = []systemdDbus.Property{} + +func TestInstall(t *testing.T) { + for _, tc := range []struct { + name string + res *specs.LinuxResources + cgroupPath string + wantProps []systemdDbus.Property + err error + }{ + { + name: "bad parent", + res: nil, + cgroupPath: "not_a_slice", + err: ErrInvalidGroupPath, + }, + { + name: "no limits", + res: nil, + wantProps: []systemdDbus.Property{ + {"Slice", dbus.MakeVariant("parent.slice")}, + {Name: "Description", Value: dbus.MakeVariant("runsc container ")}, + {Name: "MemoryAccounting", Value: dbus.MakeVariant(true)}, + {Name: "CPUAccounting", Value: dbus.MakeVariant(true)}, + {Name: "TasksAccounting", Value: dbus.MakeVariant(true)}, + {Name: "IOAccounting", Value: dbus.MakeVariant(true)}, + {Name: "Delegate", Value: dbus.MakeVariant(true)}, + }, + cgroupPath: "parent.slice", + }, + { + name: "memory", + res: &specs.LinuxResources{ + Memory: &specs.LinuxMemory{ + Limit: int64Ptr(1), + Swap: int64Ptr(2), + Reservation: int64Ptr(3), + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"MemoryMax", dbus.MakeVariant(int64(1))}, + {"MemoryLow", dbus.MakeVariant(int64(3))}, + {"MemorySwapMax", dbus.MakeVariant("1")}, + }, + }, + { + name: "memory no limit", + res: &specs.LinuxResources{ + Memory: &specs.LinuxMemory{ + Swap: int64Ptr(1), + }, + }, + err: ErrBadResourceSpec, + cgroupPath: "parent.slice", + }, + { + name: "cpu defaults", + res: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Shares: uint64Ptr(0), + Quota: int64Ptr(0), + Period: uint64Ptr(0), + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"CPUQuotaPeriodSec", dbus.MakeVariant(strconv.FormatUint(defaultPeriod/10, 10) + "ms")}, + }, + }, + { + name: "cpu", + res: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Shares: uint64Ptr(1), + Period: uint64Ptr(20), + Quota: int64Ptr(3), + Cpus: "4", + Mems: "5", + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"CPUShares", dbus.MakeVariant(convertCPUSharesToCgroupV2Value(1))}, + {"CPUQuotaPeriodSec", dbus.MakeVariant("2ms")}, + {"CPUQuota", dbus.MakeVariant("3%")}, + {"AllowedCPUs", dbus.MakeVariant("4")}, + {"AllowedMemoryNodes", dbus.MakeVariant("5")}, + }, + }, + { + name: "io", + res: &specs.LinuxResources{ + BlockIO: &specs.LinuxBlockIO{ + Weight: uint16Ptr(1), + WeightDevice: []specs.LinuxWeightDevice{ + makeLinuxWeightDevice(2, 3, uint16Ptr(4), uint16Ptr(0)), + makeLinuxWeightDevice(5, 6, uint16Ptr(7), uint16Ptr(0)), + }, + ThrottleReadBpsDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(8, 9, 10), + makeLinuxThrottleDevice(11, 12, 13), + }, + ThrottleWriteBpsDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(14, 15, 16), + }, + ThrottleReadIOPSDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(17, 18, 19), + }, + ThrottleWriteIOPSDevice: []specs.LinuxThrottleDevice{ + makeLinuxThrottleDevice(20, 21, 22), + }, + }, + }, + cgroupPath: "parent.slice", + wantProps: []systemdDbus.Property{ + {"IOWeight", dbus.MakeVariant(uint16(1))}, + {"IODevice", dbus.MakeVariant("2:3 4")}, + {"IODevice", dbus.MakeVariant("5:6 7")}, + {"IOReadBandwidth", dbus.MakeVariant("8:9 10")}, + {"IOReadBandwidth", dbus.MakeVariant("11:12 13")}, + {"IOWriteBandwidth", dbus.MakeVariant("14:15 16")}, + {"IOReadIOPS", dbus.MakeVariant("17:18 19")}, + {"IOWriteIOPS", dbus.MakeVariant("20:21 22")}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := testutil.TmpDir() + testPath := filepath.Join(dir, tc.cgroupPath) + + cg := cgroupSystemd{ + Path: testPath, + } + err := cg.Install(tc.res) + if !errors.Is(err, tc.err) { + t.Fatalf("Wrong error, got: %s, want: %s", tc.err, err) + } + cmper := cmp.Comparer(func(a dbus.Variant, b dbus.Variant) bool { + return a.String() == b.String() + }) + sorter := cmpopts.SortSlices(func(a systemdDbus.Property, b systemdDbus.Property) bool { + return (a.Name + a.Value.String()) > (b.Name + b.Value.String()) + }) + filteredProps := filterProperties(cg.properties, tc.wantProps) + if diff := cmp.Diff(filteredProps, tc.wantProps, cmper, sorter); diff != "" { + t.Errorf("cgroup properties list diff %s", diff) + } + }) + } +} + +// filterProperties filters the list of properties in got to ones with +// the names of properties specified in want. +func filterProperties(got []systemdDbus.Property, want []systemdDbus.Property) []systemdDbus.Property { + if want == nil { + return nil + } + filterMap := map[string]interface{}{} + for _, prop := range want { + filterMap[prop.Name] = nil + } + filtered := []systemdDbus.Property{} + for _, prop := range got { + if _, ok := filterMap[prop.Name]; ok { + filtered = append(filtered, prop) + } + } + return filtered +}