diff --git a/Makefile b/Makefile index e4d93bf73..cceee7275 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,8 @@ iptables-tests: load-iptables $(RUNTIME_BIN) @#$(call test,--test_env=RUNTIME=runc //test/iptables:iptables_test) @$(call install_runtime,$(RUNTIME),--net-raw) @$(call test_runtime,$(RUNTIME),--test_env=TEST_NET_RAW=true //test/iptables:iptables_test) + @$(call install_runtime,$(RUNTIME)-nftables,--net-raw --reproduce-nftables) + @$(call test_runtime,$(RUNTIME)-nftables, --test_output=all //test/iptables:nftables_test --test_arg=$(RUNTIME)-nftables) .PHONY: iptables-tests packetdrill-tests: load-packetdrill $(RUNTIME_BIN) diff --git a/images/iptables/Dockerfile b/images/iptables/Dockerfile index 8fce5a0b2..f7df35a69 100644 --- a/images/iptables/Dockerfile +++ b/images/iptables/Dockerfile @@ -1,2 +1,5 @@ FROM ubuntu:mantic + +# The iptables package installs both iptables-legacy and iptables-nft, with +# iptables symlinked to the latter. RUN apt update && apt install -y iptables diff --git a/runsc/config/config.go b/runsc/config/config.go index 9059df7f0..a70e89740 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -334,6 +334,10 @@ type Config struct { // ReproduceNAT, when true, tells runsc to scrape the host network // namespace's NAT iptables and reproduce it inside the sandbox. ReproduceNAT bool `flag:"reproduce-nat"` + + // ReproduceNftables attempts to scrape nftables routing rules if + // present, and reproduce them in the sandbox. + ReproduceNftables bool `flag:"reproduce-nftables"` } func (c *Config) validate() error { diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 6b512f666..eb516a550 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -121,6 +121,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("buffer-pooling", true, "enable allocation of buffers from a shared pool instead of the heap.") flagSet.Bool("EXPERIMENTAL-afxdp", false, "EXPERIMENTAL. Use an AF_XDP socket to receive packets.") flagSet.Bool("reproduce-nat", false, "Scrape the host netns NAT table and reproduce it in the sandbox.") + flagSet.Bool("reproduce-nftables", false, "Attempt to scrape and reproduce nftable rules inside the sandbox. Overrides reproduce-nat when true.") // Flags that control sandbox runtime behavior: accelerator related. flagSet.Bool("nvproxy", false, "EXPERIMENTAL: enable support for Nvidia GPUs") diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index 1189771c0..236ee65fe 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -320,12 +321,19 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con } // Pass the host's NAT table if requested. - if conf.ReproduceNAT { - args.NATBlob = true - f, err := writeNATBlob() + if conf.ReproduceNftables || conf.ReproduceNAT { + var f *os.File + if conf.ReproduceNftables { + log.Infof("reproing nftables") + f, err = checkNftables() + } else if conf.ReproduceNAT { + log.Infof("reproing legacy tables") + f, err = writeNATBlob() + } if err != nil { return fmt.Errorf("failed to write NAT blob: %v", err) } + args.NATBlob = true args.FilePayload.Files = append(args.FilePayload.Files, f) } @@ -597,3 +605,95 @@ func removeAddress(source netlink.Link, ipAndMask string) error { } return netlink.AddrDel(source, addr) } + +// The below is a work around to generate iptables-legacy rules on machines +// that use iptables-nftables. The logic goes something like this: +// +// start +// | +// v no +// are legacy tables empty? -----> scrape rules -----> done <----+ +// | ^ | +// | yes | | +// v yes | | +// are nft tables empty? -------------------------------+ | +// | | +// | no | +// v | +// pipe iptables-nft-save -t nat to iptables-legacy-restore | +// scrape rules | +// delete iptables-legacy rules | +// | | +// +---------------------------------------------------+ +// +// If we fail at some point (e.g. to find a binary), we just try to scrape the +// legacy rules. + +const emptyNatRules = `-P PREROUTING ACCEPT +-P INPUT ACCEPT +-P OUTPUT ACCEPT +-P POSTROUTING ACCEPT +` + +func checkNftables() (*os.File, error) { + // Use iptables (not iptables-save) to test table emptiness because it + // gives predictable results: no counters and no comments. + + // Is the legacy table empty? + if out, err := exec.Command("iptables-legacy", "-t", "nat", "-S").Output(); err != nil || string(out) != emptyNatRules { + return writeNATBlob() + } + + // Is the nftables table empty? + if out, err := exec.Command("iptables-nft", "-t", "nat", "-S").Output(); err != nil || string(out) == emptyNatRules { + return nil, fmt.Errorf("no rules to scrape: %v", err) + } + + // Get the current (empty) legacy rules. + currLegacy, err := exec.Command("iptables-legacy-save", "-t", "nat").Output() + if err != nil { + return nil, fmt.Errorf("failed to save existing rules with error (%v) and output: %s", err, currLegacy) + } + + // Restore empty legacy rules. + defer func() { + cmd := exec.Command("iptables-legacy-restore") + stdin, err := cmd.StdinPipe() + if err != nil { + log.Warningf("failed to get stdin pipe: %v", err) + return + } + + go func() { + defer stdin.Close() + stdin.Write(currLegacy) + }() + + if out, err := cmd.CombinedOutput(); err != nil { + log.Warningf("failed to restore iptables error (%v) with output: %s", err, out) + } + }() + + // Pipe the output of iptables-nft-save to iptables-legacy-restore. + nftOut, err := exec.Command("iptables-nft-save", "-t", "nat").Output() + if err != nil { + return nil, fmt.Errorf("failed to run iptables-nft-save: %v", err) + } + + cmd := exec.Command("iptables-legacy-restore") + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stdin pipe: %v", err) + } + + go func() { + defer stdin.Close() + stdin.Write(nftOut) + }() + + if out, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("failed to restore iptables error (%v) with output: %s", err, out) + } + + return writeNATBlob() +} diff --git a/test/iptables/BUILD b/test/iptables/BUILD index 460fedd28..5f6bf600b 100644 --- a/test/iptables/BUILD +++ b/test/iptables/BUILD @@ -42,3 +42,13 @@ go_test( "//pkg/test/testutil", ], ) + +sh_test( + name = "nftables_test", + srcs = ["nftables_test.sh"], + tags = [ + "local", + "manual", + ], + visibility = ["//:sandbox"], +) diff --git a/test/iptables/nftables_test.sh b/test/iptables/nftables_test.sh new file mode 100755 index 000000000..b50b6ad9e --- /dev/null +++ b/test/iptables/nftables_test.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2023 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +# expected_regex is generated by running `iptables -t nat -S` inside a runc +# Docker container connected to a custom network. Custom networks cause Docker +# to install DNS routing rules. +expected_regex='-P PREROUTING ACCEPT +-P INPUT ACCEPT +-P OUTPUT ACCEPT +-P POSTROUTING ACCEPT +-N DOCKER_OUTPUT +-N DOCKER_POSTROUTING +-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT +-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING +-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:[0-9]+ +-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:[0-9]+ +-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport [0-9]+ -j SNAT --to-source :53 +-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport [0-9]+ -j SNAT --to-source :53' + +# The runtime name is the first and only argument. +runtime="$1" + +# The image passed to docker run uses iptables-nft by default, so the above +# rules can't be simply scraped and passed to gVisor. We test that those rules +# are correctly translated to iptables-legacy rules. +net_name="nftables-test-net-$(shuf -i 0-99999999 -n 1)" +docker network create "$net_name" +trap "docker network rm \"$net_name\"" EXIT +got=$(docker run --network="$net_name" --rm --runtime "$runtime" --privileged gvisor.dev/images/iptables iptables-legacy -t nat -S) +if ! [[ "$got" =~ $expected_regex ]]; then + echo "Got incorrect rules: got on the left, want on the right" + diff <(echo "$got") <(echo "$expected_regex") + exit 1 +fi