Support plugin network stack

This commit supports a third-party network stack as a plugin stack for
gVisor.

The overall plugin package structure is the following:

- pkg/sentry/socket/plugin:
  Interfaces for initializing plugin network stack. It will be used
  in network setting up during sandbox creating.

- pkg/sentry/socket/plugin/stack:
  Glue layer for plugin stack's socket and stack ops with sentry. It
  will also register plugin stack operations if imported.

- pkg/sentry/socket/plugin/cgo:
  Interfaces defined in C for plugin network stack to support.

To build target runsc-plugin-stack, which imports
pkg/sentry/socket/plugin/stack package and enables CGO:

bazel build --config=plugin-tldk runsc:runsc-plugin-stack

(i.e. --config=plugin-tldk indicates that using TLDK as plugin stack)

By using runsc-plugin-stack binary and setting "--network=plugin" in
runtimeArgs, user can use third-party network stack instead of
netstack embedded in gVisor to get better network performance.

Redis benchmark with following setups:
1. KVM platform
2. 4 physical cores for target pod
3. target pod as redis server

Runc:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 115207.38 requests per second, p50=0.215 msec
GET: 92336.11 requests per second, p50=0.279 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 113895.21 requests per second, p50=0.247 msec
GET: 96899.23 requests per second, p50=0.271 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 126582.27 requests per second, p50=0.199 msec
GET: 95969.28 requests per second, p50=0.271 msec

Runsc with plugin stack:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 123915.74 requests per second, p50=0.343 msec
GET: 115473.45 requests per second, p50=0.335 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 120918.98 requests per second, p50=0.351 msec
GET: 117647.05 requests per second, p50=0.351 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 119904.08 requests per second, p50=0.367 msec
GET: 112739.57 requests per second, p50=0.375 msec

Runsc with netstack:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 59952.04 requests per second, p50=0.759 msec
GET: 61162.08 requests per second, p50=0.631 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 52219.32 requests per second, p50=0.719 msec
GET: 58719.91 requests per second, p50=0.663 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 59952.04 requests per second, p50=0.751 msec
GET: 60827.25 requests per second, p50=0.751 msec

Updates https://github.com/google/gvisor/issues/9266

Co-developed-by: Tianyu Zhou <wentong.zty@antgroup.com>
Signed-off-by: Anqi Shen <amy.saq@antgroup.com>

pkg/sentry/socket/plugin/stack/notifier.go

@@ -0,0 +1,165 @@
+// Copyright 2023 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stack
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/sentry/socket/plugin"
+	"gvisor.dev/gvisor/pkg/sentry/socket/plugin/cgo"
+	"gvisor.dev/gvisor/pkg/waiter"
+)
+
+// Notifier holds all the state necessary to issue notifications when
+// IO events occur on the observed FDs in plugin stack.
+type Notifier struct {
+	// the epoll FD used to register for io notifications.
+	epFD int32
+
+	// mu protects eventMap.
+	mu sync.Mutex
+
+	// eventMap maps file descriptors to their notification queues
+	// and waiting status.
+	eventMap map[uint32]*plugin.EventInfo
+}
+
+const (
+	MaxEpollEvents = 128
+	SleepInMsecond = 100
+)
+
+// NewNotifier initialize the event notifier for plugin stack.
+// It will allocate a eventMap with fd as key and corresponding eventInfo
+// as value and start a goroutine waiting the arrival of events.
+func NewNotifier() *Notifier {
+	ioInit := make(chan int32)
+
+	n := &Notifier{
+		eventMap: make(map[uint32]*plugin.EventInfo),
+	}
+
+	go n.waitAndNotify(ioInit)
+
+	epFD := <-ioInit
+	if epFD < 0 {
+		return nil
+	}
+	return n
+}
+
+// AddFD implements plugin.PluginNotifier.AddFD.
+func (n *Notifier) AddFD(fd uint32, eventInfo *plugin.EventInfo) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	// Panic if we're already notifying on this FD.
+	if _, ok := n.eventMap[fd]; ok {
+		panic(fmt.Sprintf("File descriptor %d added twice", fd))
+	}
+
+	// We have nothing to wait for at the moment. Just add it to the map.
+	n.eventMap[fd] = eventInfo
+}
+
+// RemoveFD implements plugin.PluginNotifier.RemoveFD.
+func (n *Notifier) RemoveFD(fd uint32) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	delete(n.eventMap, fd)
+}
+
+// UpdateFD implements plugin.PluginNotifier.UpdateFD.
+func (n *Notifier) UpdateFD(fd uint32) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	if eventInfo, ok := n.eventMap[fd]; ok {
+		n.waitFD(fd, eventInfo)
+	}
+}
+
+// waitAndNotify loops waiting for io event notifications from the epoll
+// object. Once notifications arrive, they are dispatched to the
+// registered queue.
+func (n *Notifier) waitAndNotify(ioInit chan int32) error {
+	// plugin stack leverages TLS varaibles, so bind this goroutine with
+	// one specific OS thread
+	runtime.LockOSThread()
+
+	// If current thread is not the main thread, change the thread name.
+	if syscall.Getpid() != syscall.Gettid() {
+		threadName := []byte("io-thread\x00")
+		if err := unix.Prctl(unix.PR_SET_NAME, uintptr(cgo.GetPtr(threadName)), 0, 0, 0); err != nil {
+			return err
+		}
+	}
+
+	n.epFD = int32(cgo.EpollCreate())
+
+	ioInit <- n.epFD
+
+	var events [MaxEpollEvents]syscall.EpollEvent
+	for {
+		num := cgo.EpollWait(n.epFD, events[:], MaxEpollEvents, SleepInMsecond)
+		if num <= 0 {
+			continue
+		}
+
+		n.mu.Lock()
+		for i := 0; i < num; i++ {
+			h := uint32(events[i].Fd)
+			eventInfo, ok := n.eventMap[h]
+			if !ok {
+				continue
+			}
+
+			ev := waiter.EventMask(events[i].Events)
+			eventInfo.Ready |= ev & (eventInfo.Mask | waiter.EventErr | waiter.EventHUp)
+			// When an error occurred, invoke all events
+			if ev&(waiter.EventErr|waiter.EventHUp) != 0 {
+				ev |= waiter.EventIn | waiter.EventOut
+			}
+			eventInfo.Wq.Notify(ev)
+		}
+		n.mu.Unlock()
+	}
+}
+
+func (n *Notifier) waitFD(fd uint32, eventInfo *plugin.EventInfo) {
+	mask := eventInfo.Wq.Events()
+
+	eventInfo.Mask = mask
+	if !eventInfo.Waiting && mask == 0 {
+		return
+	}
+
+	switch {
+	case !eventInfo.Waiting && mask != 0:
+		cgo.EpollCtl(n.epFD, syscall.EPOLL_CTL_ADD, fd, uint32(mask))
+		eventInfo.Waiting = true
+	case eventInfo.Waiting && mask == 0:
+		cgo.EpollCtl(n.epFD, syscall.EPOLL_CTL_DEL, fd, uint32(mask))
+		eventInfo.Ready = 0
+		eventInfo.Waiting = false
+	case eventInfo.Waiting && mask != 0:
+		cgo.EpollCtl(n.epFD, syscall.EPOLL_CTL_MOD, fd, uint32(mask))
+		eventInfo.Ready &= mask
+	}
+}