From b050c045d16d222a977f982be8de2c1a1052228a Mon Sep 17 00:00:00 2001 From: Jayden Nyamiaka Date: Thu, 12 Sep 2024 14:53:44 -0700 Subject: [PATCH] Implement Last operation (construction, evaluation, tests, NO interpretation). Last operation is thread-safe. Includes minor state/structure changes that enable time tracking: - Adds clock & start time to NFTables state (tcpip Standard Clock as default). - Passes rule as argument to operation evaluation for access to NFTables state. PiperOrigin-RevId: 674021450 --- pkg/tcpip/nftables/BUILD | 3 + pkg/tcpip/nftables/nftables.go | 69 ++++++++++++++---- pkg/tcpip/nftables/nftables_test.go | 105 ++++++++++++++++++++++++---- 3 files changed, 150 insertions(+), 27 deletions(-) diff --git a/pkg/tcpip/nftables/BUILD b/pkg/tcpip/nftables/BUILD index 3bc198613..8fb035da1 100644 --- a/pkg/tcpip/nftables/BUILD +++ b/pkg/tcpip/nftables/BUILD @@ -13,6 +13,7 @@ go_library( ], deps = [ "//pkg/abi/linux", + "//pkg/tcpip", "//pkg/tcpip/checksum", "//pkg/tcpip/header", "//pkg/tcpip/stack", @@ -29,7 +30,9 @@ go_test( deps = [ "//pkg/abi/linux", "//pkg/buffer", + "//pkg/sync", "//pkg/tcpip", + "//pkg/tcpip/faketime", "//pkg/tcpip/header", "//pkg/tcpip/stack", ], diff --git a/pkg/tcpip/nftables/nftables.go b/pkg/tcpip/nftables/nftables.go index af20ca6bc..63971a264 100644 --- a/pkg/tcpip/nftables/nftables.go +++ b/pkg/tcpip/nftables/nftables.go @@ -46,8 +46,10 @@ import ( "fmt" "slices" "sync/atomic" + "time" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/checksum" "gvisor.dev/gvisor/pkg/tcpip/header" "gvisor.dev/gvisor/pkg/tcpip/stack" @@ -208,7 +210,9 @@ func validateHook(hook Hook, family AddressFamily) error { // NFTables represents the nftables state for all address families. // Note: unlike iptables, nftables doesn't start with any initialized tables. type NFTables struct { - filters [NumAFs]*addressFamilyFilter + filters [NumAFs]*addressFamilyFilter // Filters for each address family. + clock tcpip.Clock // Clock for timing evaluations. + startTime time.Time // Time NFTables object was created. } // addressFamilyFilter represents the nftables state for a specific address @@ -217,6 +221,9 @@ type addressFamilyFilter struct { // family is the address family of the filter. family AddressFamily + // nftState is the NFTables object the filter belongs to. + nftState *NFTables + // tables is a map of tables for each address family. tables map[string]*Table @@ -570,8 +577,9 @@ type Rule struct { type operation interface { // evaluate evaluates the operation on the given packet and register set, - // changing the register set and possibly the packet in place. - evaluate(regs *registerSet, pkt *stack.PacketBuffer) + // changing the register set and possibly the packet in place. We pass the + // assigned rule to allow the operation to access parts of the NFTables state. + evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) } // Ensures all operations implement the Operation interface at compile time. @@ -583,6 +591,7 @@ var ( _ operation = (*payloadSet)(nil) _ operation = (*bitwise)(nil) _ operation = (*counter)(nil) + _ operation = (*last)(nil) ) // immediate is an operation that sets the data in a register. @@ -600,7 +609,7 @@ func newImmediate(dreg uint8, data registerData) (*immediate, error) { } // evaluate for Immediate sets the data in the destination register. -func (op immediate) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op immediate) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { op.data.storeData(regs, op.dreg) } @@ -667,7 +676,7 @@ func newComparison(sreg uint8, op int, data []byte) (*comparison, error) { // evaluate for Comparison compares the data in the source register to the given // data and breaks from the rule if the comparison is false. -func (op comparison) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op comparison) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { // Gets the data to compare to. data := op.data.data @@ -768,7 +777,7 @@ func newRanged(sreg uint8, op int, low, high []byte) (*ranged, error) { // evaluate for Ranged checks whether the source register data is within the // specified inclusive range and breaks from the rule if comparison is false. -func (op ranged) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op ranged) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { // Gets the upper and lower bounds as bytesData. low, high := op.low.data, op.high.data @@ -876,7 +885,7 @@ func newPayloadLoad(base payloadBase, offset, blen, dreg uint8) (*payloadLoad, e // evaluate for PayloadLoad loads data from the packet payload into the // destination register. -func (op payloadLoad) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op payloadLoad) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { // Gets the packet payload. payload := getPayloadBuffer(pkt, op.base) @@ -956,7 +965,7 @@ func newPayloadSet(base payloadBase, offset, blen, sreg, csumType, csumOffset, c // evaluate for PayloadSet sets data in the packet payload to the value in the // source register. -func (op payloadSet) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op payloadSet) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { // Gets the packet payload. payload := getPayloadBuffer(pkt, op.base) @@ -1197,7 +1206,7 @@ func evaluateBitwiseRshift(sregBuf, dregBuf []byte, shift uint32) { // evaluate for bitwise performs the bitwise operation on the source register // data and stores the result in the destination register. -func (op bitwise) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op bitwise) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { // Gets the specified buffers of the source and destination registers. sregBuf := getRegisterBuffer(regs, op.sreg)[:op.blen] dregBuf := getRegisterBuffer(regs, op.dreg)[:op.blen] @@ -1233,11 +1242,37 @@ func newCounter(startBytes, startPackets int64) *counter { } // evaluate for counter increments the counter for the packet and bytes. -func (op *counter) evaluate(regs *registerSet, pkt *stack.PacketBuffer) { +func (op *counter) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { op.bytes.Add(int64(pkt.Size())) op.packets.Add(1) } +// last is an operation that records the last time the operation was evaluated +// for the purpose of tracking the last time the rule has matched a packet. +// Note: no explicit constructor bc no fields need to be set (use &last{}). +type last struct { + // Must be thread-safe because data stored here is updated for each evaluation + // and evaluations can happen in parallel for processing multiple packets. + + // timestampMS is the time of last evaluation as a millisecond unix time. + // Milliseconds chosen as units because closest in magnitude to jiffies. + timestampMS atomic.Int64 + + // set is whether the operation has been evaluated at least once. + set atomic.Bool + + // Note: The last operation has not been observed in the nft binary debug + // output, so it has no interpretation, though it is fully implemented. +} + +// evaluate for last records the last time the operation was evaluated and flags +// if this was the first time the operation was evaluated. +func (op *last) evaluate(regs *registerSet, pkt *stack.PacketBuffer, rule *Rule) { + clock := rule.chain.table.afFilter.nftState.clock + op.timestampMS.Store(clock.Now().UnixMilli()) + op.set.CompareAndSwap(false, true) +} + // // Register and Register-Related Implementations. // Note: Registers are represented by type uint8 for the register number. @@ -1620,7 +1655,7 @@ func (c *Chain) evaluate(regs *registerSet, pkt *stack.PacketBuffer) error { // netfilter terminal verdict. func (r *Rule) evaluate(regs *registerSet, pkt *stack.PacketBuffer) error { for _, op := range r.ops { - op.evaluate(regs, pkt) + op.evaluate(regs, pkt, r) if regs.Verdict().Code != VC(linux.NFT_CONTINUE) { break } @@ -1634,10 +1669,13 @@ func (r *Rule) evaluate(regs *registerSet, pkt *stack.PacketBuffer) error { // chains, and rules for convenience. // -// NewNFTables creates a new NFTables object. -// Note: nothing needs to be initialized in the struct before use. -func NewNFTables() *NFTables { - return &NFTables{} +// NewNFTables creates a new NFTables state object using the given clock for +// timing operations. +func NewNFTables(clock tcpip.Clock) *NFTables { + if clock == nil { + panic("nftables state must be initialized with a non-nil clock") + } + return &NFTables{clock: clock, startTime: clock.Now()} } // Flush clears entire ruleset and all data for all address families. @@ -1701,6 +1739,7 @@ func (nf *NFTables) AddTable(family AddressFamily, name string, comment string, if nf.filters[family] == nil { nf.filters[family] = &addressFamilyFilter{ family: family, + nftState: nf, tables: make(map[string]*Table), hfStacks: make(map[Hook]*hookFunctionStack), } diff --git a/pkg/tcpip/nftables/nftables_test.go b/pkg/tcpip/nftables/nftables_test.go index ef81ae2c7..417b3ba47 100644 --- a/pkg/tcpip/nftables/nftables_test.go +++ b/pkg/tcpip/nftables/nftables_test.go @@ -20,10 +20,13 @@ import ( "reflect" "slices" "testing" + "time" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/faketime" "gvisor.dev/gvisor/pkg/tcpip/header" "gvisor.dev/gvisor/pkg/tcpip/stack" ) @@ -320,7 +323,7 @@ func makeIPv6TCPPacket(reserved int, ipv6Fields *header.IPv6Fields, tcpFields *h func TestUnsupportedAddressFamily(t *testing.T) { // Makes arbitrary packet for comparison (to check for no changes). cmpPkt := makeArbitraryPacket(arbitraryReservedHeaderBytes) - nf := NewNFTables() + nf := newNFTablesStd() for _, unsupportedFamily := range []AddressFamily{AddressFamily(NumAFs), AddressFamily(-1)} { // Note: the Prerouting hook is arbitrary (any hook would work). pkt := makeArbitraryPacket(arbitraryReservedHeaderBytes) @@ -341,7 +344,7 @@ func TestAcceptAllForSupportedHooks(t *testing.T) { cmpPkt := makeArbitraryPacket(arbitraryReservedHeaderBytes) for _, family := range []AddressFamily{IP, IP6, Inet, Arp, Bridge, Netdev} { t.Run(family.String()+" address family", func(t *testing.T) { - nf := NewNFTables() + nf := newNFTablesStd() for _, hook := range []Hook{Prerouting, Input, Forward, Output, Postrouting, Ingress, Egress} { pkt := makeArbitraryPacket(arbitraryReservedHeaderBytes) v, err := nf.EvaluateHook(family, hook, pkt) @@ -510,7 +513,7 @@ func TestEvaluateImmediateVerdict(t *testing.T) { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a base chain (for 2 rules) and another // target chain (for 1 rule). - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -574,7 +577,7 @@ func TestEvaluateImmediateBytesData(t *testing.T) { tname := fmt.Sprintf("immediately load %d bytes into %d-byte registers", blen, registerSize) t.Run(tname, func(t *testing.T) { // Sets up an NFTables object with a base chain with policy accept. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -1060,7 +1063,7 @@ func TestEvaluateComparison(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a single table, chain, and rule. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -1297,7 +1300,7 @@ func TestEvaluateRanged(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a single table, chain, and rule. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -1531,7 +1534,7 @@ func TestEvaluatePayloadLoad(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a single table, chain, and rule. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2035,7 +2038,7 @@ func TestEvaluatePayloadSet(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a single table, chain, and rule. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2265,7 +2268,7 @@ func TestEvaluateBitwise(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object with a single table, chain, and rule. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2331,7 +2334,7 @@ func TestEvaluateCounter(t *testing.T) { uncountedIPv4Pkt(), countedIPv4Pkt(), uncountedIPv4Pkt(), uncountedIPv4Pkt(), uncountedIPv4Pkt(), countedIPv4Pkt()} t.Run("counter increment tests", func(t *testing.T) { // Sets up an NFTables object with a base chain with policy accept. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2383,6 +2386,79 @@ func TestEvaluateCounter(t *testing.T) { }) } +// TestEvaluateLast tests that the Last operation correctly records the last +// time the operation was evaluated. +func TestEvaluateLast(t *testing.T) { + // Creates last operation and number of elapses (in milliseconds) for testing. + last := &last{} + elapses := []int64{0, 1000, 500, 1000, 100, 200, 300, 20000, 50, 700} + totalElapsed := make([]int64, len(elapses)) + copy(totalElapsed, elapses) + + t.Run("last timing tests", func(t *testing.T) { + // Makes an arbitrary packet to be used in the test. + pkt := makeArbitraryPacket(arbitraryReservedHeaderBytes) + + // Sets up an NFTables object with a base chain and fake manual clock. + fakeClock := faketime.NewManualClock() + nf := NewNFTables(fakeClock) + tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) + if err != nil { + t.Fatalf("unexpected error for AddTable: %v", err) + } + bc, err := tab.AddChain("base_chain", nil, "test chain", false) + if err != nil { + t.Fatalf("unexpected error for AddChain: %v", err) + } + bc.SetBaseChainInfo(arbitraryInfoPolicyAccept) + + // Registers a single rule with the last operation. + rule := &Rule{} + rule.addOperation(last) + if err := bc.RegisterRule(rule, -1); err != nil { + t.Fatalf("unexpected error for RegisterRule: %v", err) + } + + // Sets up a wait group to wait for all AfterFunc goroutines to complete. + var wg sync.WaitGroup + wg.Add(len(elapses)) + defer wg.Wait() + + // Calls EvaluateHook for each elapse and checks that the last operation + // recorded the correct timestamp and has the set flag set. + startStamp := nf.startTime.UnixMilli() + clock := nf.clock + for i := range elapses { + // Uses totalElapsed slice to avoid race conditions. + if i != 0 { + totalElapsed[i] += totalElapsed[i-1] + } + clock.AfterFunc(time.Duration(totalElapsed[i])*time.Millisecond, func() { + // Decrements wait group counter at end to signal func has completed. + defer wg.Done() + + // Evaluates the packet (which should update last's timestamp). + _, err := nf.EvaluateHook(arbitraryFamily, arbitraryHook, pkt) + if err != nil { + t.Fatalf("unexpected error for EvaluateHook for packet %d: %v", i, err) + } + + // Checks the set flag and the timestamp via total elapsed time. + if !last.set.Load() { + t.Fatalf("last operation not set for packet %d", i) + } + if dTotal := last.timestampMS.Load() - startStamp; dTotal != totalElapsed[i] { + t.Fatalf("last operation recorded %d milliseconds since start for packet %d, expected %d", dTotal, i, totalElapsed[i]) + } + }) + } + // Manually advances the clock to trigger the AfterFunc goroutines. + for _, elapse := range elapses { + fakeClock.Advance(time.Duration(elapse) * time.Millisecond) + } + }) +} + // TestLoopCheckOnRegisterAndUnregister tests the loop checking and accompanying // logic on registering and unregistering rules. func TestLoopCheckOnRegisterAndUnregister(t *testing.T) { @@ -2796,7 +2872,7 @@ func TestLoopCheckOnRegisterAndUnregister(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up an NFTables object based on test struct. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2907,7 +2983,7 @@ func TestMaxNestedJumps(t *testing.T) { } { t.Run(test.tname, func(t *testing.T) { // Sets up chains of nested jumps or gotos. - nf := NewNFTables() + nf := newNFTablesStd() tab, err := nf.AddTable(arbitraryFamily, "test", "test table", false) if err != nil { t.Fatalf("unexpected error for AddTable: %v", err) @@ -2979,6 +3055,11 @@ func packetResultString(initial, final *stack.PacketBuffer) string { return "modified" } +// newNFTablesStd creates a new NFTables object w/ a standard clock for testing. +func newNFTablesStd() *NFTables { + return NewNFTables(tcpip.NewStdClock()) +} + // mustCreateImmediate wraps the newImmediate function for brevity. func mustCreateImmediate(t *testing.T, dreg uint8, data registerData) *immediate { imm, err := newImmediate(dreg, data)