netbird/gvisor
0
0
Fork 0
mirror of https://github.com/netbirdio/gvisor.git synced 2026-05-22 17:12:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6821 from dqminh:feature/cgroupv2

Browse Source 
PiperOrigin-RevId: 413032543
This commit is contained in:
gVisor bot
2021-11-29 18:37:42 -08:00
parent 5e984d5aa2 32205d7a94
commit fa4e2fff8a
17 changed files with 1391 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/hooks/pre-command
+7 -4
View File
@@ -1,14 +1,16 @@
# Install packages we need. Docker must be installed and configured,
# as should Go itself. We just install some extra bits and pieces.
function install_pkgs() {
export DEBIAN_FRONTEND=noninteractive
while true; do
if sudo apt-get update && sudo apt-get install -y "$@"; then
if sudo -E apt-get update && sudo -E apt-get install -y "$@"; then
break
fi
done
}
install_pkgs make linux-libc-dev graphviz jq curl binutils gnupg gnupg-agent \
gcc pkg-config apt-transport-https ca-certificates software-properties-common
gcc pkg-config apt-transport-https ca-certificates software-properties-common \
jq
# Install headers, only if available.
if test -n "$(apt-cache search --names-only "^linux-headers-$(uname -r)$")"; then
@@ -22,8 +24,9 @@ export TOTAL_PARTITIONS=${BUILDKITE_PARALLEL_JOB_COUNT:-1}
# Ensure Docker has experimental enabled.
EXPERIMENTAL=$(sudo docker version --format='{{.Server.Experimental}}')
if test "${EXPERIMENTAL}" != "true"; then
make sudo TARGETS=//runsc:runsc ARGS="install --experimental=true"
CGDRIVER=$(sudo docker info --format='{{.CgroupDriver}}')
if test "${EXPERIMENTAL}" != "true" || test "${CGDRIVER}" != "cgroupfs"; then
make sudo TARGETS=//runsc:runsc ARGS="install --experimental=true --cgroupdriver=cgroupfs"
sudo systemctl restart docker
fi
.buildkite/pipeline.yaml
+20
View File
@@ -111,9 +111,19 @@ steps:
- <<: *common
label: ":test_tube: Unit tests"
command: make unit-tests
- <<: *common
label: ":test_tube: Unit tests (cgroupv2)"
command: make unit-tests
agents:
queue: "cgroupv2"
- <<: *common
label: ":test_tube: Container tests"
command: make container-tests
- <<: *common
label: ":test_tube: Container tests (cgroupv2)"
command: make container-tests
agents:
queue: "cgroupv2"
# All system call tests.
- <<: *common
@@ -125,6 +135,11 @@ steps:
- <<: *common
label: ":docker: Docker tests"
command: make docker-tests
- <<: *common
label: ":docker: Docker tests (cgroupv2)"
command: make docker-tests
agents:
queue: "cgroupv2"
- <<: *common
label: ":goggles: Overlay tests"
command: make overlay-tests
@@ -152,6 +167,11 @@ steps:
- <<: *common
label: ":docker: Containerd 1.5.4 tests"
command: make containerd-test-1.5.4
- <<: *common
label: ":docker: Containerd 1.5.4 tests (cgroupv2)"
command: make containerd-test-1.5.4
agents:
queue: "cgroupv2"
# Check the website builds.
- <<: *common
Makefile
+16 -1
View File
@@ -113,6 +113,12 @@ RUNTIME_BIN := $(RUNTIME_DIR)/runsc
RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs
RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
ifeq ($(shell stat -f -c "%T" /sys/fs/cgroup 2>/dev/null),cgroup2fs)
CGROUPV2 := true
else
CGROUPV2 := false
endif
$(RUNTIME_BIN): # See below.
@mkdir -p "$(RUNTIME_DIR)"
ifeq (,$(STAGED_BINARIES))
@@ -204,7 +210,7 @@ tests: unit-tests nogo-tests container-tests syscall-tests
integration-tests: ## Run all standard integration tests.
integration-tests: docker-tests overlay-tests hostnet-tests swgso-tests
integration-tests: do-tests kvm-tests containerd-test-1.3.9
integration-tests: do-tests kvm-tests containerd-tests-min
.PHONY: integration-tests
network-tests: ## Run all networking integration tests.
@@ -320,10 +326,19 @@ else
endif
@$(call sudo,test/root:root_test,--runtime=$(RUNTIME) -test.v)
ifeq ($(CGROUPV2),false)
containerd-tests-min: containerd-test-1.3.9
else
containerd-tests-min: containerd-test-1.4.3
endif
# The shim builds with containerd 1.3.9 and it's not backward compatible. Test
# with 1.3.9 and newer versions.
# When run under cgroupv2 environment, skip 1.3.9 as it does not support cgroupv2
containerd-tests: ## Runs all supported containerd version tests.
ifeq ($(CGROUPV2),false)
containerd-tests: containerd-test-1.3.9
endif
containerd-tests: containerd-test-1.4.3
containerd-tests: containerd-test-1.5.4
pkg/shim/BUILD
+2
View File
@@ -8,6 +8,7 @@ go_library(
"api.go",
"debug.go",
"epoll.go",
"oom_v2.go",
"options.go",
"service.go",
"service_linux.go",
@@ -25,6 +26,7 @@ go_library(
"@com_github_burntsushi_toml//:go_default_library",
"@com_github_containerd_cgroups//:go_default_library",
"@com_github_containerd_cgroups//stats/v1:go_default_library",
"@com_github_containerd_cgroups//v2:go_default_library",
"@com_github_containerd_console//:go_default_library",
"@com_github_containerd_containerd//api/events:go_default_library",
"@com_github_containerd_containerd//api/types/task:go_default_library",
pkg/shim/epoll.go
+5 -1
View File
@@ -81,9 +81,13 @@ func (e *epoller) run(ctx context.Context) {
}
}
func (e *epoller) add(id string, cg cgroups.Cgroup) error {
func (e *epoller) add(id string, cgx interface{}) error {
e.mu.Lock()
defer e.mu.Unlock()
cg, ok := cgx.(cgroups.Cgroup)
if !ok {
return fmt.Errorf("expected cgroups.Cgroup, got: %T", cgx)
}
fd, err := cg.OOMEventFD()
if err != nil {
return err
pkg/shim/oom_v2.go
+112
View File
@@ -0,0 +1,112 @@
// Copyright The containerd Authors.
// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
// +build linux
package shim
import (
"context"
"fmt"
cgroupsv2 "github.com/containerd/cgroups/v2"
"github.com/containerd/containerd/runtime"
"github.com/containerd/containerd/runtime/v2/shim"
"github.com/sirupsen/logrus"
)
// newOOMv2Epoller returns an implementation that listens to OOM events
// from a container's cgroups v2. This is copied from containerd to avoid
// having to upgrade containerd package just to get it
func newOOMv2Poller(publisher shim.Publisher) (oomPoller, error) {
return &watcherV2{
itemCh: make(chan itemV2),
publisher: publisher,
}, nil
}
// watcher implementation for handling OOM events from a container's cgroup
type watcherV2 struct {
itemCh chan itemV2
publisher shim.Publisher
}
type itemV2 struct {
id string
ev cgroupsv2.Event
err error
}
// Close closes the watcher
func (w *watcherV2) Close() error {
return nil
}
// Run the loop
func (w *watcherV2) run(ctx context.Context) {
lastOOMMap := make(map[string]uint64) // key: id, value: ev.OOM
for {
select {
case <-ctx.Done():
w.Close()
return
case i := <-w.itemCh:
if i.err != nil {
delete(lastOOMMap, i.id)
continue
}
lastOOM := lastOOMMap[i.id]
if i.ev.OOM > lastOOM {
if err := w.publisher.Publish(ctx, runtime.TaskOOMEventTopic, &TaskOOM{
ContainerID: i.id,
}); err != nil {
logrus.WithError(err).Error("publish OOM event")
}
}
if i.ev.OOM > 0 {
lastOOMMap[i.id] = i.ev.OOM
}
}
}
}
// Add cgroups.Cgroup to the epoll monitor
func (w *watcherV2) add(id string, cgx interface{}) error {
cg, ok := cgx.(*cgroupsv2.Manager)
if !ok {
return fmt.Errorf("expected *cgroupsv2.Manager, got: %T", cgx)
}
// NOTE: containerd/cgroups/v2 does not support closing eventCh routine currently.
// The routine shuts down when an error happens, mostly when the cgroup is deleted.
eventCh, errCh := cg.EventChan()
go func() {
for {
i := itemV2{id: id}
select {
case ev := <-eventCh:
i.ev = ev
w.itemCh <- i
case err := <-errCh:
i.err = err
w.itemCh <- i
// we no longer get any event/err when we got an err
logrus.WithError(err).Warn("error from eventChan")
return
}
}
}()
return nil
}
pkg/shim/service.go
+33 -3
View File
@@ -29,6 +29,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/containerd/cgroups"
cgroupsstats "github.com/containerd/cgroups/stats/v1"
cgroupsv2 "github.com/containerd/cgroups/v2"
"github.com/containerd/console"
"github.com/containerd/containerd/api/events"
"github.com/containerd/containerd/api/types/task"
@@ -82,6 +83,15 @@ const (
cgroupParentAnnotation = "dev.gvisor.spec.cgroup-parent"
)
type oomPoller interface {
io.Closer
// add adds `cg` cgroup to oom poller. `cg` is cgroups.Cgroup in v1 and
// `cgroupsv2.Manager` in v2
add(id string, cg interface{}) error
// run monitors oom event and notifies the shim about them
run(ctx context.Context)
}
// New returns a new shim service that can be used via GRPC.
func New(ctx context.Context, id string, publisher shim.Publisher, cancel func()) (shim.Shim, error) {
var opts shim.Opts
@@ -89,7 +99,15 @@ func New(ctx context.Context, id string, publisher shim.Publisher, cancel func()
opts = ctxOpts.(shim.Opts)
}
ep, err := newOOMEpoller(publisher)
var (
ep oomPoller
err error
)
if cgroups.Mode() == cgroups.Unified {
ep, err = newOOMv2Poller(publisher)
} else {
ep, err = newOOMEpoller(publisher)
}
if err != nil {
return nil, err
}
@@ -161,7 +179,7 @@ type service struct {
ec chan proc.Exit
// oomPoller monitors the sandbox's cgroup for OOM notifications.
oomPoller *epoller
oomPoller oomPoller
// cancel is a function that needs to be called before the shim stops. The
// function is provided by the caller to New().
@@ -473,7 +491,19 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (*ta
// sandbox create since the sandbox process will be created here.
pid := process.Pid()
if pid > 0 {
cg, err := cgroups.Load(cgroups.V1, cgroups.PidPath(pid))
var (
cg interface{}
err error
)
if cgroups.Mode() == cgroups.Unified {
var cgPath string
cgPath, err = cgroupsv2.PidGroupPath(pid)
if err == nil {
cg, err = cgroupsv2.LoadManager("/sys/fs/cgroup", cgPath)
}
} else {
cg, err = cgroups.Load(cgroups.V1, cgroups.PidPath(pid))
}
if err != nil {
return nil, fmt.Errorf("loading cgroup for %d: %w", pid, err)
}
runsc/cgroup/BUILD
+8 -2
View File
@@ -4,7 +4,10 @@ package(licenses = ["notice"])
go_library(
name = "cgroup",
srcs = ["cgroup.go"],
srcs = [
"cgroup.go",
"cgroup_v2.go",
],
visibility = ["//:sandbox"],
deps = [
"//pkg/cleanup",
@@ -19,7 +22,10 @@ go_library(
go_test(
name = "cgroup_test",
size = "small",
srcs = ["cgroup_test.go"],
srcs = [
"cgroup_test.go",
"cgroup_v2_test.go",
],
library = ":cgroup",
tags = ["local"],
deps = [
runsc/cgroup/cgroup.go
+74 -25
View File
@@ -217,10 +217,10 @@ func loadPaths(pid string) (map[string]string, error) {
}
defer mountinfo.Close()
return loadPathsHelper(procCgroup, mountinfo)
return loadPathsHelper(procCgroup, mountinfo, IsOnlyV2())
}
func loadPathsHelper(cgroup, mountinfo io.Reader) (map[string]string, error) {
func loadPathsHelper(cgroup, mountinfo io.Reader, unified bool) (map[string]string, error) {
paths := make(map[string]string)
scanner := bufio.NewScanner(cgroup)
@@ -231,6 +231,10 @@ func loadPathsHelper(cgroup, mountinfo io.Reader) (map[string]string, error) {
if len(tokens) != 3 {
return nil, fmt.Errorf("invalid cgroups file, line: %q", scanner.Text())
}
if len(tokens[1]) == 0 && unified {
paths[cgroup2Key] = tokens[2]
continue
}
if len(tokens[1]) == 0 {
continue
}
@@ -255,30 +259,42 @@ func loadPathsHelper(cgroup, mountinfo io.Reader) (map[string]string, error) {
// Format: ID parent major:minor root mount-point options opt-fields - fs-type source super-options
// Example: 39 32 0:34 / /sys/fs/cgroup/devices rw,noexec shared:18 - cgroup cgroup rw,devices
fields := strings.Fields(mountScanner.Text())
if len(fields) < 9 || fields[len(fields)-3] != "cgroup" {
if len(fields) < 9 {
// Skip mounts that are not cgroup mounts.
continue
}
// Cgroup controller type is in the super-options field.
superOptions := strings.Split(fields[len(fields)-1], ",")
for _, opt := range superOptions {
// Remove prefix for cgroups with no controller, eg. systemd.
opt = strings.TrimPrefix(opt, "name=")
switch fields[len(fields)-3] {
case "cgroup":
// Cgroup controller type is in the super-options field.
superOptions := strings.Split(fields[len(fields)-1], ",")
for _, opt := range superOptions {
// Remove prefix for cgroups with no controller, eg. systemd.
opt = strings.TrimPrefix(opt, "name=")
// Only considers cgroup controllers that are registered, and skip other
// irrelevant options, e.g. rw.
if cgroupPath, ok := paths[opt]; ok {
rootDir := fields[3]
if rootDir != "/" {
// When cgroup is in submount, remove repeated path components from
// cgroup path to avoid duplicating them.
relCgroupPath, err := filepath.Rel(rootDir, cgroupPath)
if err != nil {
return nil, err
// Only considers cgroup controllers that are registered, and skip other
// irrelevant options, e.g. rw.
if cgroupPath, ok := paths[opt]; ok {
rootDir := fields[3]
if rootDir != "/" {
// When cgroup is in submount, remove repeated path components from
// cgroup path to avoid duplicating them.
relCgroupPath, err := filepath.Rel(rootDir, cgroupPath)
if err != nil {
return nil, err
}
paths[opt] = relCgroupPath
}
paths[opt] = relCgroupPath
}
}
case "cgroup2":
if cgroupPath, ok := paths[cgroup2Key]; ok {
root := fields[3]
relCgroupPath, err := filepath.Rel(root, cgroupPath)
if err != nil {
return nil, err
}
paths[cgroup2Key] = relCgroupPath
}
}
}
if err := mountScanner.Err(); err != nil {
@@ -335,21 +351,38 @@ func NewFromPid(pid int) (Cgroup, error) {
}
func new(pid, cgroupsPath string) (Cgroup, error) {
var parents map[string]string
var (
parents map[string]string
err error
cg Cgroup
)
// If path is relative, load cgroup paths for the process to build the
// relative paths.
if !filepath.IsAbs(cgroupsPath) {
var err error
parents, err = loadPaths(pid)
if err != nil {
return nil, fmt.Errorf("finding current cgroups: %w", err)
}
}
cg := &cgroupV1{
Name: cgroupsPath,
Parents: parents,
Own: make(map[string]bool),
if IsOnlyV2() {
if p, ok := parents[cgroup2Key]; ok {
// The cgroup of current pid will have tasks in it and we can't use
// that, instead, use the its parent which should not have tasks in it.
cgroupsPath = filepath.Join(filepath.Dir(p), cgroupsPath)
}
// Assume that for v2, cgroup is always mounted at cgroupRoot.
cg, err = newCgroupV2(cgroupRoot, cgroupsPath)
if err != nil {
return nil, err
}
} else {
cg = &cgroupV1{
Name: cgroupsPath,
Parents: parents,
Own: make(map[string]bool),
}
}
log.Debugf("New cgroup for pid: %s, %+v", pid, cg)
return cg, nil
@@ -364,8 +397,20 @@ type cgroupJSONv1 struct {
Cgroup *cgroupV1 `json:"cgroup"`
}
type cgroupJSONv2 struct {
Cgroup *cgroupV2 `json:"cgroup"`
}
// UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON
func (c *CgroupJSON) UnmarshalJSON(data []byte) error {
if IsOnlyV2() {
v2 := cgroupJSONv2{}
err := json.Unmarshal(data, &v2)
if v2.Cgroup != nil {
c.Cgroup = v2.Cgroup
}
return err
}
v1 := cgroupJSONv1{}
err := json.Unmarshal(data, &v1)
if v1.Cgroup != nil {
@@ -380,6 +425,10 @@ func (c *CgroupJSON) MarshalJSON() ([]byte, error) {
v1 := cgroupJSONv1{}
return json.Marshal(&v1)
}
if IsOnlyV2() {
v2 := cgroupJSONv2{Cgroup: c.Cgroup.(*cgroupV2)}
return json.Marshal(&v2)
}
v1 := cgroupJSONv1{Cgroup: c.Cgroup.(*cgroupV1)}
return json.Marshal(&v1)
}
runsc/cgroup/cgroup_test.go
+1 -1
View File
@@ -831,7 +831,7 @@ func TestLoadPaths(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
r := strings.NewReader(tc.cgroups)
mountinfo := strings.NewReader(tc.mountinfo)
got, err := loadPathsHelper(r, mountinfo)
got, err := loadPathsHelper(r, mountinfo, false)
if len(tc.err) == 0 {
if err != nil {
t.Fatalf("Unexpected error: %v", err)
runsc/cgroup/cgroup_v2.go
+685
View File
File diff suppressed because it is too large Load Diff
runsc/cgroup/cgroup_v2_test.go
+242
View File
@@ -0,0 +1,242 @@
// Copyright The runc Authors.
// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cgroup
import (
"strings"
"testing"
)
var cgroupv2MountInfo = `29 22 0:26 / /sys/fs/cgroup rw shared:4 - cgroup2 cgroup2 rw,seclabel,nsdelegate`
func TestLoadPathsCgroupv2(t *testing.T) {
for _, tc := range []struct {
name string
cgroups string
mountinfo string
want map[string]string
err string
}{
{
name: "cgroupv2",
cgroups: "0::/docker/123",
mountinfo: cgroupv2MountInfo,
want: map[string]string{
"cgroup2": "docker/123",
},
},
{
name: "cgroupv2-nested",
cgroups: "0::/",
mountinfo: cgroupv2MountInfo,
want: map[string]string{
"cgroup2": ".",
},
},
} {
t.Run(tc.name, func(t *testing.T) {
r := strings.NewReader(tc.cgroups)
mountinfo := strings.NewReader(tc.mountinfo)
got, err := loadPathsHelper(r, mountinfo, true)
if len(tc.err) == 0 {
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
} else if !strings.Contains(err.Error(), tc.err) {
t.Fatalf("Wrong error message, want: *%s*, got: %v", tc.err, err)
}
for key, vWant := range tc.want {
vGot, ok := got[key]
if !ok {
t.Errorf("Missing controller %q", key)
}
if vWant != vGot {
t.Errorf("Wrong controller %q value, want: %q, got: %q", key, vWant, vGot)
}
delete(got, key)
}
for k, v := range got {
t.Errorf("Unexpected controller %q: %q", k, v)
}
})
}
}
func TestNumToStr(t *testing.T) {
cases := map[int64]string{
0: "",
-1: "max",
10: "10",
}
for i, expected := range cases {
got := numToStr(i)
if got != expected {
t.Errorf("expected numToStr(%d) to be %q, got %q", i, expected, got)
}
}
}
func TestConvertBlkIOToIOWeightValue(t *testing.T) {
cases := map[uint16]uint64{
0: 0,
10: 1,
1000: 10000,
}
for i, expected := range cases {
got := convertBlkIOToIOWeightValue(i)
if got != expected {
t.Errorf("expected ConvertBlkIOToIOWeightValue(%d) to be %d, got %d", i, expected, got)
}
}
}
func TestConvertCPUSharesToCgroupV2Value(t *testing.T) {
cases := map[uint64]uint64{
0: 0,
2: 1,
262144: 10000,
}
for i, expected := range cases {
got := convertCPUSharesToCgroupV2Value(i)
if got != expected {
t.Errorf("expected ConvertCPUSharesToCgroupV2Value(%d) to be %d, got %d", i, expected, got)
}
}
}
func TestConvertMemorySwapToCgroupV2Value(t *testing.T) {
cases := []struct {
memswap, memory int64
expected int64
expErr bool
}{
{
memswap: 0,
memory: 0,
expected: 0,
},
{
memswap: -1,
memory: 0,
expected: -1,
},
{
memswap: -1,
memory: -1,
expected: -1,
},
{
memswap: -2,
memory: 0,
expErr: true,
},
{
memswap: -1,
memory: 1000,
expected: -1,
},
{
memswap: 1000,
memory: 1000,
expected: 0,
},
{
memswap: 500,
memory: 200,
expected: 300,
},
{
memswap: 300,
memory: 400,
expErr: true,
},
{
memswap: 300,
memory: 0,
expErr: true,
},
{
memswap: 300,
memory: -300,
expErr: true,
},
{
memswap: 300,
memory: -1,
expErr: true,
},
}
for _, c := range cases {
swap, err := convertMemorySwapToCgroupV2Value(c.memswap, c.memory)
if c.expErr {
if err == nil {
t.Errorf("memswap: %d, memory %d, expected error, got %d, nil", c.memswap, c.memory, swap)
}
// no more checks
continue
}
if err != nil {
t.Errorf("memswap: %d, memory %d, expected success, got error %s", c.memswap, c.memory, err)
}
if swap != c.expected {
t.Errorf("memswap: %d, memory %d, expected %d, got %d", c.memswap, c.memory, c.expected, swap)
}
}
}
func TestParseCPUQuota(t *testing.T) {
cases := []struct {
quota string
expected float64
expErr bool
}{
{
quota: "max 100000\n",
expected: -1,
},
{
quota: "10000 100000",
expected: 0.1,
},
{
quota: "20000 100000\n",
expected: 0.2,
},
{
quota: "-1",
expected: -1,
expErr: true,
},
}
for _, c := range cases {
res, err := parseCPUQuota(c.quota)
if c.expErr {
if err == nil {
t.Errorf("quota: %q, expected error, got %.2f, nil", c.quota, res)
}
continue
}
if err != nil {
t.Errorf("quota: %q, expected success, got error %s", c.quota, err)
}
if res != c.expected {
t.Errorf("quota: %q, expected %.2f, got error %.2f", c.quota, c.expected, res)
}
}
}
runsc/cmd/install.go
+12
View File
@@ -32,6 +32,7 @@ type Install struct {
ConfigFile string
Runtime string
Experimental bool
CgroupDriver string
}
// Name implements subcommands.Command.Name.
@@ -55,6 +56,7 @@ func (i *Install) SetFlags(fs *flag.FlagSet) {
fs.StringVar(&i.ConfigFile, "config_file", "/etc/docker/daemon.json", "path to Docker daemon config file")
fs.StringVar(&i.Runtime, "runtime", "runsc", "runtime name")
fs.BoolVar(&i.Experimental, "experimental", false, "enable experimental features")
fs.StringVar(&i.CgroupDriver, "cgroupdriver", "", "docker cgroup driver")
}
// Execute implements subcommands.Command.Execute.
@@ -95,6 +97,16 @@ func (i *Install) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
c["experimental"] = true
}
if i.CgroupDriver != "" {
v, ok := c["exec-opts"]
if ok {
opts := v.([]interface{})
c["exec-opts"] = append(opts, fmt.Sprintf("native.cgroupdriver=%s", i.CgroupDriver))
} else {
c["exec-opts"] = []string{fmt.Sprintf("native.cgroupdriver=%s", i.CgroupDriver)}
}
}
// Write out the runtime.
if err := writeConfig(c, i.ConfigFile); err != nil {
log.Fatalf("Error writing config file %q: %v", i.ConfigFile, err)
runsc/container/container.go
-7
View File
@@ -1277,13 +1277,6 @@ func (c *Container) setupCgroupForSubcontainer(conf *config.Config, spec *specs.
// error is suppressed and a nil cgroups instance is returned to indicate that
// no cgroups was configured.
func cgroupInstall(conf *config.Config, cg cgroup.Cgroup, res *specs.LinuxResources) (cgroup.Cgroup, error) {
// TODO(gvisor.dev/issue/3481): Remove when cgroups v2 is supported.
if cgroup.IsOnlyV2() {
if conf.Rootless {
return nil, nil
}
return nil, fmt.Errorf("cgroups V2 is not yet supported. Enable cgroups V1 and retry")
}
if err := cg.Install(res); err != nil {
switch {
case errors.Is(err, unix.EACCES) && conf.Rootless:
runsc/sandbox/sandbox.go
+1 -1
View File
@@ -777,7 +777,7 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
quota, err := s.CgroupJSON.Cgroup.CPUQuota()
if err != nil {
return fmt.Errorf("getting cpu qouta from cgroups: %v", err)
return fmt.Errorf("getting cpu quota from cgroups: %v", err)
}
if n := int(math.Ceil(quota)); n > 0 {
if n < minCPUs {
test/root/cgroup_test.go
+167 -3
View File
@@ -86,8 +86,15 @@ func TestMemCgroup(t *testing.T) {
// or after looping below (so the application can start).
time.Sleep(100 * time.Millisecond)
var path string
// Read the cgroup memory limit.
if cgroup.IsOnlyV2() {
path = filepath.Join("/sys/fs/cgroup/docker", gid, "memory.max")
} else {
path = filepath.Join("/sys/fs/cgroup/memory/docker", gid, "memory.limit_in_bytes")
}
// Read the cgroup memory limit.
path := filepath.Join("/sys/fs/cgroup/memory/docker", gid, "memory.limit_in_bytes")
outRaw, err := ioutil.ReadFile(path)
if err != nil {
// It's possible that the container does not exist yet.
@@ -103,8 +110,14 @@ func TestMemCgroup(t *testing.T) {
continue
}
if cgroup.IsOnlyV2() {
// v2 does not have max_usage_in_bytes equivalent, so memory.current is the
// next best thing that we can use
path = filepath.Join("/sys/fs/cgroup/docker", gid, "memory.current")
} else {
path = filepath.Join("/sys/fs/cgroup/memory/docker", gid, "memory.max_usage_in_bytes")
}
// Read the cgroup memory usage.
path = filepath.Join("/sys/fs/cgroup/memory/docker", gid, "memory.max_usage_in_bytes")
outRaw, err = ioutil.ReadFile(path)
if err != nil {
t.Fatalf("error reading usage: %v", err)
@@ -126,7 +139,10 @@ func TestMemCgroup(t *testing.T) {
}
// TestCgroup sets cgroup options and checks that cgroup was properly configured.
func TestCgroup(t *testing.T) {
func TestCgroupV1(t *testing.T) {
if cgroup.IsOnlyV2() {
t.Skip("skipping cgroupv1 attribute testing in cgroupv2 setup")
}
ctx := context.Background()
d := dockerutil.MakeContainer(ctx, t)
defer d.CleanUp(ctx)
@@ -308,6 +324,154 @@ func TestCgroup(t *testing.T) {
}
}
// TestCgroupV2 sets cgroup options and checks that cgroup was properly configured with
// cgroupv2 setup
func TestCgroupV2(t *testing.T) {
if !cgroup.IsOnlyV2() {
t.Skip("skipping cgroupv2 attribute testing in cgroupv1 setup")
}
ctx := context.Background()
d := dockerutil.MakeContainer(ctx, t)
defer d.CleanUp(ctx)
// This is not a comprehensive list of attributes.
//
// Note that we are specifically missing cpusets, which fail if specified.
// In any case, it's unclear if cpusets can be reliably tested here: these
// are often run on a single core virtual machine, and there is only a single
// CPU available in our current set, and every container's set.
attrs := []struct {
field string
value int64
file string
want string
skipIfNotFound bool
}{
{
field: "cpu-shares",
value: 3333,
file: "cpu.weight",
want: "128",
},
{
field: "cpu-period",
value: 2000,
file: "cpu.max",
want: "max 2000",
},
{
field: "memory",
value: 1 << 30,
file: "memory.max",
want: "1073741824",
},
{
field: "memory-reservation",
value: 500 << 20,
file: "memory.low",
want: "524288000",
},
{
field: "memory-swap",
value: 1 << 31,
file: "memory.swap.max",
// memory.swap.max is only the swap value, unlike cgroupv1
want: fmt.Sprintf("%d", 1<<31-1<<30),
skipIfNotFound: true, // swap may be disabled on the machine.
},
{
field: "blkio-weight",
value: 750,
file: "io.bfq.weight",
want: fmt.Sprintf("default %d", 750),
skipIfNotFound: true, // blkio groups may not be available.
},
{
field: "pids-limit",
value: 1000,
file: "pids.max",
want: "1000",
},
}
// Make configs.
conf, hostconf, _ := d.ConfigsFrom(dockerutil.RunOpts{
Image: "basic/alpine",
}, "sleep", "10000")
// Add Cgroup arguments to configs.
for _, attr := range attrs {
switch attr.field {
case "cpu-shares":
hostconf.Resources.CPUShares = attr.value
case "cpu-period":
hostconf.Resources.CPUPeriod = attr.value
case "cpu-quota":
hostconf.Resources.CPUQuota = attr.value
case "kernel-memory":
hostconf.Resources.KernelMemory = attr.value
case "memory":
hostconf.Resources.Memory = attr.value
case "memory-reservation":
hostconf.Resources.MemoryReservation = attr.value
case "memory-swap":
hostconf.Resources.MemorySwap = attr.value
case "memory-swappiness":
val := attr.value
hostconf.Resources.MemorySwappiness = &val
case "blkio-weight":
// detect existence of io.bfq.weight as this is not always loaded
_, err := ioutil.ReadFile(filepath.Join("/sys/fs/cgroup/docker", attr.file))
if err == nil || !attr.skipIfNotFound {
hostconf.Resources.BlkioWeight = uint16(attr.value)
}
case "pids-limit":
val := attr.value
hostconf.Resources.PidsLimit = &val
}
}
// Create container.
if err := d.CreateFrom(ctx, "basic/alpine", conf, hostconf, nil); err != nil {
t.Fatalf("create failed with: %v", err)
}
// Start container.
if err := d.Start(ctx); err != nil {
t.Fatalf("start failed with: %v", err)
}
// Lookup the relevant cgroup ID.
gid := d.ID()
t.Logf("cgroup ID: %s", gid)
// Check list of attributes defined above.
for _, attr := range attrs {
path := filepath.Join("/sys/fs/cgroup/docker", gid, attr.file)
out, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) && attr.skipIfNotFound {
t.Logf("skipped %s", attr.file)
continue
}
t.Fatalf("failed to read %q: %v", path, err)
}
if got := strings.TrimSpace(string(out)); got != attr.want {
t.Errorf("field: %q, cgroup attribute %s, got: %q, want: %q", attr.field, attr.file, got, attr.want)
}
}
// Check that sandbox is inside cgroup.
pid, err := d.SandboxPid(ctx)
if err != nil {
t.Fatalf("SandboxPid: %v", err)
}
path := filepath.Join("/sys/fs/cgroup/docker", gid, "cgroup.procs")
if err := verifyPid(pid, path); err != nil {
t.Errorf("cgroup control processes: %v", err)
}
}
// TestCgroupParent sets the "CgroupParent" option and checks that the child and
// parent's cgroups are created correctly relative to each other.
func TestCgroupParent(t *testing.T) {
tools/install_containerd.sh
+6
View File
@@ -26,6 +26,12 @@ if [[ "${CONTAINERD_MAJOR}" -eq 1 ]] && [[ "${CONTAINERD_MINOR}" -le 4 ]]; then
export GO111MODULE=off
fi
# containerd < 1.4 doesn't work with cgroupv2 setup, so we check for that here
if [[ "$(stat -f -c %T /sys/fs/cgroup 2>/dev/null)" == "cgroup2fs" && "${CONTAINERD_MAJOR}" -eq 1 && "${CONTAINERD_MINOR}" -lt 4 ]]; then
echo "containerd < 1.4 does not work with cgroup2"
exit 1
fi
# Helper for Go packages below.
install_helper() {
declare -r PACKAGE="${1}"