diff --git a/pkg/tcpip/network/internal/multicast/BUILD b/pkg/tcpip/network/internal/multicast/BUILD index e274d4f37..682091a94 100644 --- a/pkg/tcpip/network/internal/multicast/BUILD +++ b/pkg/tcpip/network/internal/multicast/BUILD @@ -9,7 +9,6 @@ go_library( ], visibility = ["//visibility:public"], deps = [ - "//pkg/atomicbitops", "//pkg/tcpip", "//pkg/tcpip/stack", ], @@ -21,7 +20,6 @@ go_test( srcs = ["route_table_test.go"], library = ":multicast", deps = [ - "//pkg/atomicbitops", "//pkg/refs", "//pkg/refsvfs2", "//pkg/tcpip", @@ -42,6 +40,7 @@ go_test( ":multicast", "//pkg/refs", "//pkg/refsvfs2", + "//pkg/tcpip", "//pkg/tcpip/buffer", "//pkg/tcpip/faketime", "//pkg/tcpip/stack", diff --git a/pkg/tcpip/network/internal/multicast/example_test.go b/pkg/tcpip/network/internal/multicast/example_test.go index 6b05b41ce..997ae2e9c 100644 --- a/pkg/tcpip/network/internal/multicast/example_test.go +++ b/pkg/tcpip/network/internal/multicast/example_test.go @@ -18,9 +18,11 @@ import ( "fmt" "os" "testing" + "time" "gvisor.dev/gvisor/pkg/refs" "gvisor.dev/gvisor/pkg/refsvfs2" + "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/buffer" "gvisor.dev/gvisor/pkg/tcpip/faketime" "gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast" @@ -28,17 +30,27 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/testutil" ) +const ( + defaultMinTTL = 10 + inputNICID tcpip.NICID = 1 + outgoingNICID tcpip.NICID = 2 +) + // Example shows how to interact with a multicast RouteTable. func Example() { address := testutil.MustParse4("192.168.1.1") + defaultOutgoingInterfaces := []multicast.OutgoingInterface{{ID: outgoingNICID, MinTTL: defaultMinTTL}} routeKey := multicast.RouteKey{UnicastSource: address, MulticastDestination: address} pkt := newPacketBuffer("hello") defer pkt.DecRef() + clock := faketime.NewManualClock() + clock.Advance(10 * time.Second) + // Create a route table from a specified config. table := multicast.RouteTable{} - config := multicast.DefaultConfig(faketime.NewManualClock()) + config := multicast.DefaultConfig(clock) if err := table.Init(config); err != nil { panic(err) @@ -72,12 +84,48 @@ func Example() { deliverPktLocally(pkt) } + // To transition a pending route to the installed state, call: + route := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) + pendingRoute, ok := table.AddInstalledRoute(routeKey, route) + + if !ok { + return + } + + // If there was a pending route, then the caller is responsible for + // flushing any pending packets. + for !pendingRoute.IsEmpty() { + pkt, err := pendingRoute.Dequeue() + if err != nil { + panic(fmt.Sprintf("pendingRoute.Dequeue() = (_, %s)", err)) + } + forwardPkt(pkt, route) + } + + // To obtain the last used time of the route, call: + timestamp, found := table.GetLastUsedTimestamp(routeKey) + + if !found { + panic(fmt.Sprintf("table.GetLastUsedTimestamp(%#v) = (_, false)", routeKey)) + } + + fmt.Printf("Last used timestamp: %d", timestamp.Nanoseconds()) + + // Finally, to remove an installed route, call: + if removed := table.RemoveInstalledRoute(routeKey); !removed { + panic(fmt.Sprintf("table.RemoveInstalledRoute(%#v) = false", routeKey)) + } + // Output: // emitMissingRouteEvent // deliverPktLocally + // forwardPkt + // Last used timestamp: 10000000000 } -func forwardPkt(*stack.PacketBuffer, *multicast.InstalledRoute) {} +func forwardPkt(*stack.PacketBuffer, *multicast.InstalledRoute) { + fmt.Println("forwardPkt") +} func emitMissingRouteEvent(multicast.RouteKey) { fmt.Println("emitMissingRouteEvent") diff --git a/pkg/tcpip/network/internal/multicast/route_table.go b/pkg/tcpip/network/internal/multicast/route_table.go index 96e104c41..fa3b5d077 100644 --- a/pkg/tcpip/network/internal/multicast/route_table.go +++ b/pkg/tcpip/network/internal/multicast/route_table.go @@ -19,9 +19,7 @@ import ( "errors" "fmt" "sync" - "time" - "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/stack" ) @@ -55,7 +53,7 @@ type RouteTable struct { pendingMu sync.RWMutex // +checklocks:pendingMu - pendingRoutes map[RouteKey]pendingRoute + pendingRoutes map[RouteKey]PendingRoute config Config } @@ -69,7 +67,7 @@ var ( // Config, but is required. ErrMissingClock = errors.New("clock must not be nil") - // ErrAlreadyInitialized indicate that RouteTable.Init was already invoked. + // ErrAlreadyInitialized indicates that RouteTable.Init was already invoked. ErrAlreadyInitialized = errors.New("table is already initialized") ) @@ -86,8 +84,10 @@ type RouteKey struct { type InstalledRoute struct { expectedInputInterface tcpip.NICID outgoingInterfaces []OutgoingInterface - // +checkatomic - lastUsedTimestamp atomicbitops.Int64 + + lastUsedTimestampMu sync.RWMutex + // +checklocks:lastUsedTimestampMu + lastUsedTimestamp tcpip.MonotonicTime } // ExpectedInputInterface returns the expected input interface for the route. @@ -100,17 +100,27 @@ func (r *InstalledRoute) OutgoingInterfaces() []OutgoingInterface { return r.outgoingInterfaces } -// LastUsedTimestamp returns a Unix based timestamp in microseconds that -// corresponds to the last time the route was used or updated. -func (r *InstalledRoute) LastUsedTimestamp() int64 { - return r.lastUsedTimestamp.Load() +// LastUsedTimestamp returns a monotonic timestamp that corresponds to the last +// time the route was used or updated. +func (r *InstalledRoute) LastUsedTimestamp() tcpip.MonotonicTime { + r.lastUsedTimestampMu.RLock() + defer r.lastUsedTimestampMu.RUnlock() + + return r.lastUsedTimestamp } // SetLastUsedTimestamp sets the time that the route was last used. // -// Callers should invoke this anytime the route is used to forward a packet. -func (r *InstalledRoute) SetLastUsedTimestamp(time time.Time) { - r.lastUsedTimestamp.Store(time.UnixMicro()) +// The timestamp is only updated if it occurs after the currently set +// timestamp. Callers should invoke this anytime the route is used to forward a +// packet. +func (r *InstalledRoute) SetLastUsedTimestamp(monotonicTime tcpip.MonotonicTime) { + r.lastUsedTimestampMu.Lock() + defer r.lastUsedTimestampMu.Unlock() + + if monotonicTime.After(r.lastUsedTimestamp) { + r.lastUsedTimestamp = monotonicTime + } } // OutgoingInterface represents an interface that packets should be forwarded @@ -124,23 +134,26 @@ type OutgoingInterface struct { MinTTL uint8 } -// pendingRoute represents a route that is in the "pending" state. +// PendingRoute represents a route that is in the "pending" state. // // A route is in the pending state if an installed route does not yet exist // for the entry. For such routes, packets are added to an expiring queue until // a route is installed. -type pendingRoute struct { +type PendingRoute struct { packets []*stack.PacketBuffer } -func newPendingRoute(maxSize uint8) pendingRoute { - return pendingRoute{packets: make([]*stack.PacketBuffer, 0, maxSize)} +func newPendingRoute(maxSize uint8) PendingRoute { + return PendingRoute{packets: make([]*stack.PacketBuffer, 0, maxSize)} } // Dequeue removes the first element in the queue and returns it. // // If the queue is empty, then an error will be returned. -func (p *pendingRoute) Dequeue() (*stack.PacketBuffer, error) { +// +// TODO(https://gvisor.dev/issue/7338): Remove this and instead just return the +// list of packets from AddInstalledRoute. +func (p *PendingRoute) Dequeue() (*stack.PacketBuffer, error) { if len(p.packets) == 0 { return nil, errors.New("dequeue called on queue empty") } @@ -152,7 +165,7 @@ func (p *pendingRoute) Dequeue() (*stack.PacketBuffer, error) { // IsEmpty returns true if the queue contains no more elements. Otherwise, // returns false. -func (p *pendingRoute) IsEmpty() bool { +func (p *PendingRoute) IsEmpty() bool { return len(p.packets) == 0 } @@ -205,7 +218,7 @@ func (r *RouteTable) Init(config Config) error { r.config = config r.installedRoutes = make(map[RouteKey]*InstalledRoute) - r.pendingRoutes = make(map[RouteKey]pendingRoute) + r.pendingRoutes = make(map[RouteKey]PendingRoute) return nil } @@ -214,7 +227,7 @@ func (r *RouteTable) NewInstalledRoute(inputInterface tcpip.NICID, outgoingInter return &InstalledRoute{ expectedInputInterface: inputInterface, outgoingInterfaces: outgoingInterfaces, - lastUsedTimestamp: atomicbitops.FromInt64(r.config.Clock.Now().UnixMicro()), + lastUsedTimestamp: r.config.Clock.NowMonotonic(), } } @@ -297,7 +310,7 @@ func (r *RouteTable) GetRouteOrInsertPending(key RouteKey, pkt *stack.PacketBuff } // +checklocks:r.pendingMu -func (r *RouteTable) getOrCreatePendingRouteRLocked(key RouteKey) (pendingRoute, PendingRouteState) { +func (r *RouteTable) getOrCreatePendingRouteRLocked(key RouteKey) (PendingRoute, PendingRouteState) { if pendingRoute, ok := r.pendingRoutes[key]; ok { return pendingRoute, PendingRouteStateAppended } @@ -305,3 +318,54 @@ func (r *RouteTable) getOrCreatePendingRouteRLocked(key RouteKey) (pendingRoute, pendingRoute := newPendingRoute(r.config.MaxPendingQueueSize) return pendingRoute, PendingRouteStateInstalled } + +// AddInstalledRoute adds the provided route to the table. +// +// Returns true if the route was previously in the pending state. Otherwise, +// returns false. +// +// If the route was previously pending, then the caller is responsible for +// flushing the returned pending route packet queue. Conversely, if the route +// was not pending, then any existing installed route will be overwritten. +func (r *RouteTable) AddInstalledRoute(key RouteKey, route *InstalledRoute) (PendingRoute, bool) { + r.installedMu.Lock() + defer r.installedMu.Unlock() + r.installedRoutes[key] = route + + r.pendingMu.Lock() + defer r.pendingMu.Unlock() + + pendingRoute, ok := r.pendingRoutes[key] + delete(r.pendingRoutes, key) + return pendingRoute, ok +} + +// RemoveInstalledRoute deletes the installed route that matches the provided +// key. +// +// Returns true if a route was removed. Otherwise returns false. +func (r *RouteTable) RemoveInstalledRoute(key RouteKey) bool { + r.installedMu.Lock() + defer r.installedMu.Unlock() + + if _, ok := r.installedRoutes[key]; ok { + delete(r.installedRoutes, key) + return true + } + + return false +} + +// GetLastUsedTimestamp returns a monotonic timestamp that represents the last +// time the route that matches the provided key was used or updated. +// +// Returns true if a matching route was found. Otherwise returns false. +func (r *RouteTable) GetLastUsedTimestamp(key RouteKey) (tcpip.MonotonicTime, bool) { + r.installedMu.RLock() + defer r.installedMu.RUnlock() + + if route, ok := r.installedRoutes[key]; ok { + return route.LastUsedTimestamp(), true + } + return tcpip.MonotonicTime{}, false +} diff --git a/pkg/tcpip/network/internal/multicast/route_table_test.go b/pkg/tcpip/network/internal/multicast/route_table_test.go index 9bca1177a..d4a93a4b1 100644 --- a/pkg/tcpip/network/internal/multicast/route_table_test.go +++ b/pkg/tcpip/network/internal/multicast/route_table_test.go @@ -21,7 +21,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/refs" "gvisor.dev/gvisor/pkg/refsvfs2" "gvisor.dev/gvisor/pkg/tcpip" @@ -35,6 +34,7 @@ const ( defaultMinTTL = 10 inputNICID tcpip.NICID = 1 outgoingNICID tcpip.NICID = 2 + defaultNICID tcpip.NICID = 3 ) var ( @@ -76,6 +76,18 @@ func defaultConfig(opts ...configOption) Config { return *c } +func installedRouteComparer(a *InstalledRoute, b *InstalledRoute) bool { + if !cmp.Equal(a.OutgoingInterfaces(), b.OutgoingInterfaces()) { + return false + } + + if a.ExpectedInputInterface() != b.ExpectedInputInterface() { + return false + } + + return a.LastUsedTimestamp() == b.LastUsedTimestamp() +} + func TestInit(t *testing.T) { tests := []struct { name string @@ -130,19 +142,9 @@ func TestNewInstalledRoute(t *testing.T) { } route := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) - expectedRoute := &InstalledRoute{expectedInputInterface: inputNICID, outgoingInterfaces: defaultOutgoingInterfaces, lastUsedTimestamp: atomicbitops.FromInt64(clock.Now().UnixMicro())} + expectedRoute := &InstalledRoute{expectedInputInterface: inputNICID, outgoingInterfaces: defaultOutgoingInterfaces, lastUsedTimestamp: clock.NowMonotonic()} - if diff := cmp.Diff(expectedRoute, route, cmp.Comparer(func(a *InstalledRoute, b *InstalledRoute) bool { - if !cmp.Equal(a.OutgoingInterfaces(), b.OutgoingInterfaces()) { - return false - } - - if a.ExpectedInputInterface() != b.ExpectedInputInterface() { - return false - } - - return a.LastUsedTimestamp() == b.LastUsedTimestamp() - })); diff != "" { + if diff := cmp.Diff(expectedRoute, route, cmp.Comparer(installedRouteComparer)); diff != "" { t.Errorf("installed route mismatch (-want +got):\n%s", diff) } } @@ -158,7 +160,7 @@ func TestPendingRouteStates(t *testing.T) { defer pkt.DecRef() // Queue two pending packets for the same route. The PendingRouteState should // transition from PendingRouteStateInstalled to PendingRouteStateAppended. - for _, wantPendingRouteState := range [...]PendingRouteState{PendingRouteStateInstalled, PendingRouteStateAppended} { + for _, wantPendingRouteState := range []PendingRouteState{PendingRouteStateInstalled, PendingRouteStateAppended} { routeResult, err := table.GetRouteOrInsertPending(defaultRouteKey, pkt) if err != nil { @@ -178,6 +180,194 @@ func TestPendingRouteStates(t *testing.T) { } } +func TestAddInstalledRouteWithPending(t *testing.T) { + table := RouteTable{} + config := defaultConfig() + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + wantPkt := newPacketBuffer("hello") + defer wantPkt.DecRef() + // Queue a pending packet. This packet should later be returned in a + // PendingRoute when table.AddInstalledRoute is invoked. + _, err := table.GetRouteOrInsertPending(defaultRouteKey, wantPkt) + + if err != nil { + t.Fatalf("table.GetRouteOrInsertPending(%#v, %#v): %v", defaultRouteKey, wantPkt, err) + } + + route := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) + + pendingRoute, wasPending := table.AddInstalledRoute(defaultRouteKey, route) + if !wasPending { + t.Fatalf("got table.AddInstalledRoute(%#v, %#v) = (nil, false), want = (_, true)", defaultRouteKey, route) + } + + // Verify that packets are properly dequeued from the PendingRoute. + pkt, err := pendingRoute.Dequeue() + + if err != nil { + t.Fatalf("got pendingRoute.Dequeue() = (_, %v), want = (_, nil)", err) + } + + if !cmp.Equal(wantPkt.Views(), pkt.Views()) { + t.Errorf("got pendingRoute.Dequeue() = (%v, nil), want = (%v, nil)", pkt.Views(), wantPkt.Views()) + } + + if !pendingRoute.IsEmpty() { + t.Errorf("got pendingRoute.IsEmpty() = false, want = true") + } + + // Verify that the pending route is deleted (not returned on subsequent + // calls to AddInstalledRoute). + pendingRoute, wasPending = table.AddInstalledRoute(defaultRouteKey, route) + if wasPending { + t.Errorf("got table.AddInstalledRoute(%#v, %#v) = (%#v, true), want (_, false)", defaultRouteKey, route, pendingRoute) + } +} + +func TestAddInstalledRouteWithNoPending(t *testing.T) { + table := RouteTable{} + config := defaultConfig() + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + firstRoute := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) + secondRoute := table.NewInstalledRoute(defaultNICID, defaultOutgoingInterfaces) + + pkt := newPacketBuffer("hello") + defer pkt.DecRef() + for _, route := range [...]*InstalledRoute{firstRoute, secondRoute} { + if pendingRoute, wasPending := table.AddInstalledRoute(defaultRouteKey, route); wasPending { + t.Errorf("got table.AddInstalledRoute(%#v, %#v) = (%#v, true), want = (_, false)", defaultRouteKey, route, pendingRoute) + } + + // AddInstalledRoute is invoked for the same routeKey two times. Verify + // that the fetched InstalledRoute reflects the most recent invocation of + // AddInstalledRoute. + routeResult, err := table.GetRouteOrInsertPending(defaultRouteKey, pkt) + + if err != nil { + t.Fatalf("table.GetRouteOrInsertPending(%#v, %#v): %v", defaultRouteKey, pkt, err) + } + + if routeResult.PendingRouteState != PendingRouteStateNone { + t.Errorf("got routeResult.PendingRouteState = %s, want = PendingRouteStateNone", routeResult.PendingRouteState) + } + + if diff := cmp.Diff(route, routeResult.InstalledRoute, cmp.Comparer(installedRouteComparer)); diff != "" { + t.Errorf("route.InstalledRoute mismatch (-want +got):\n%s", diff) + } + } +} + +func TestRemoveInstalledRoute(t *testing.T) { + table := RouteTable{} + config := defaultConfig() + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + route := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) + + table.AddInstalledRoute(defaultRouteKey, route) + + if removed := table.RemoveInstalledRoute(defaultRouteKey); !removed { + t.Errorf("got table.RemoveInstalledRoute(%#v) = false, want = true", defaultRouteKey) + } + + pkt := newPacketBuffer("hello") + defer pkt.DecRef() + + result, err := table.GetRouteOrInsertPending(defaultRouteKey, pkt) + + if err != nil { + t.Fatalf("table.GetRouteOrInsertPending(%#v, %#v): %v", defaultRouteKey, pkt, err) + } + + if result.InstalledRoute != nil { + t.Errorf("got result.InstalledRoute = %v, want = nil", result.InstalledRoute) + } +} + +func TestRemoveInstalledRouteWithNoMatchingRoute(t *testing.T) { + table := RouteTable{} + config := defaultConfig() + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + if removed := table.RemoveInstalledRoute(defaultRouteKey); removed { + t.Errorf("got table.RemoveInstalledRoute(%#v) = true, want = false", defaultRouteKey) + } +} + +func TestGetLastUsedTimestampWithNoMatchingRoute(t *testing.T) { + table := RouteTable{} + config := defaultConfig() + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + if _, found := table.GetLastUsedTimestamp(defaultRouteKey); found { + t.Errorf("got table.GetLastUsedTimetsamp(%#v) = (_, true), want = (_, false)", defaultRouteKey) + } +} + +func TestSetLastUsedTimestamp(t *testing.T) { + clock := faketime.NewManualClock() + clock.Advance(10 * time.Second) + + currentTime := clock.NowMonotonic() + validLastUsedTime := currentTime.Add(10 * time.Second) + + tests := []struct { + name string + lastUsedTime tcpip.MonotonicTime + wantLastUsedTime tcpip.MonotonicTime + }{ + { + name: "valid timestamp", + lastUsedTime: validLastUsedTime, + wantLastUsedTime: validLastUsedTime, + }, + { + name: "timestamp before", + lastUsedTime: currentTime.Add(-5 * time.Second), + wantLastUsedTime: currentTime, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + table := RouteTable{} + config := defaultConfig(withClock(clock)) + if err := table.Init(config); err != nil { + t.Fatalf("table.Init(%#v): %s", config, err) + } + + route := table.NewInstalledRoute(inputNICID, defaultOutgoingInterfaces) + + table.AddInstalledRoute(defaultRouteKey, route) + + route.SetLastUsedTimestamp(test.lastUsedTime) + + // Verify that the updated timestamp is actually reflected in the RouteTable. + timestamp, found := table.GetLastUsedTimestamp(defaultRouteKey) + + if !found { + t.Fatalf("got table.GetLastUsedTimestamp(%#v) = (_, false_), want = (_, true)", defaultRouteKey) + } + + if got, want := timestamp.Nanoseconds(), test.wantLastUsedTime.Nanoseconds(); got != want { + t.Errorf("got table.GetLastUsedTimestamp(%#v) = (%v, _), want (%v, _)", defaultRouteKey, got, want) + } + }) + } +} + func TestMain(m *testing.M) { refs.SetLeakMode(refs.LeaksPanic) code := m.Run() diff --git a/pkg/tcpip/tcpip.go b/pkg/tcpip/tcpip.go index be97aab46..11dae0c18 100644 --- a/pkg/tcpip/tcpip.go +++ b/pkg/tcpip/tcpip.go @@ -71,6 +71,11 @@ type MonotonicTime struct { nanoseconds int64 } +// Nanoseconds returns the monotonic time in nanoseconds. +func (mt MonotonicTime) Nanoseconds() int64 { + return mt.nanoseconds +} + // Before reports whether the monotonic clock reading mt is before u. func (mt MonotonicTime) Before(u MonotonicTime) bool { return mt.nanoseconds < u.nanoseconds