systrap: use seccomp notifications to communicate with syscall threads

The new synchronous mode of seccomp-unotify (v6.6-rc1~205^2~6) reduces overhead
of context switches.

PiperOrigin-RevId: 622924980

pkg/abi/linux/seccomp.go

@@ -25,9 +25,20 @@ const (
 	SECCOMP_RET_ACTION      = 0x7fff0000
 	SECCOMP_RET_DATA        = 0x0000ffff
 
-	SECCOMP_SET_MODE_FILTER   = 1
-	SECCOMP_FILTER_FLAG_TSYNC = 1
-	SECCOMP_GET_ACTION_AVAIL  = 2
+	SECCOMP_SET_MODE_FILTER  = 1
+	SECCOMP_GET_ACTION_AVAIL = 2
+	SECCOMP_GET_NOTIF_SIZES  = 3
+
+	SECCOMP_FILTER_FLAG_TSYNC        = 1
+	SECCOMP_FILTER_FLAG_NEW_LISTENER = 1 << 3
+
+	SECCOMP_USER_NOTIF_FLAG_CONTINUE = 1
+
+	SECCOMP_IOCTL_NOTIF_RECV      = 0xc0502100
+	SECCOMP_IOCTL_NOTIF_SEND      = 0xc0182101
+	SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104
+
+	SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP = 1
 )
 
 // BPFAction is an action for a BPF filter.
@@ -40,6 +51,7 @@ const (
 	SECCOMP_RET_TRAP         BPFAction = 0x00030000
 	SECCOMP_RET_ERRNO        BPFAction = 0x00050000
 	SECCOMP_RET_TRACE        BPFAction = 0x7ff00000
+	SECCOMP_RET_USER_NOTIF   BPFAction = 0x7fc00000
 	SECCOMP_RET_ALLOW        BPFAction = 0x7fff0000
 )
 
@@ -65,6 +77,8 @@ func (a BPFAction) String() string {
 		return fmt.Sprintf("trace (data=%#x)", data)
 	case SECCOMP_RET_ALLOW:
 		return "allow"
+	case SECCOMP_RET_USER_NOTIF:
+		return "unotify"
 	}
 	return fmt.Sprintf("invalid action: %#x", a)
 }
@@ -113,6 +127,35 @@ type SeccompData struct {
 	Args [6]uint64
 }
 
+// SeccompNotifResp is equivalent to struct seccomp_notif_resp.
+//
+// +marshal
+type SeccompNotifResp struct {
+	ID    uint64
+	Val   int64
+	Error int32
+	Flags uint32
+}
+
+// SeccompNotifSizes is equivalent to struct seccomp_notif_sizes.
+//
+// +marshal
+type SeccompNotifSizes struct {
+	Notif      uint16
+	Notif_resp uint16
+	Data       uint16
+}
+
+// SeccompNotif is equivalent to struct seccomp_notif.
+//
+// +marshal
+type SeccompNotif struct {
+	ID    uint64
+	Pid   int32
+	Flags uint32
+	Data  SeccompData
+}
+
 // String returns a human-friendly representation of this `SeccompData`.
 func (sd SeccompData) String() string {
 	return fmt.Sprintf(

pkg/sentry/platform/systrap/filters.go

@@ -94,6 +94,21 @@ func (systrapSeccomp) SyscallFilters(vars precompiledseccomp.Values) seccomp.Sys
 		},
 		unix.SYS_TGKILL: seccomp.MatchAll{},
 		unix.SYS_WAIT4:  seccomp.MatchAll{},
+		unix.SYS_IOCTL: seccomp.Or{
+			seccomp.PerArg{
+				seccomp.NonNegativeFD{},
+				seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_RECV),
+			},
+			seccomp.PerArg{
+				seccomp.NonNegativeFD{},
+				seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_SEND),
+			},
+			seccomp.PerArg{
+				seccomp.NonNegativeFD{},
+				seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_SET_FLAGS),
+				seccomp.EqualTo(linux.SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP),
+			},
+		},
 		unix.SYS_WAITID: seccomp.PerArg{
 			seccomp.EqualTo(unix.P_PID),
 			seccomp.AnyValue{},

pkg/sentry/platform/systrap/stub_amd64.s

@@ -22,6 +22,7 @@
 #define SIGKILL		 9   // +checkconst unix SIGKILL
 #define SIGSTOP		 19  // +checkconst unix SIGSTOP
 #define SYS_PRCTL	 157 // +checkconst unix SYS_PRCTL
+#define SYS_EXIT_GROUP   231 // +checkconst unix SYS_EXIT_GROUP
 #define PR_SET_PDEATHSIG 1   // +checkconst unix PR_SET_PDEATHSIG
 
 #define SYS_FUTEX	 202 // +checkconst unix SYS_FUTEX
@@ -30,6 +31,7 @@
 
 #define NEW_STUB	 1 // +checkconst . _NEW_STUB
 #define RUN_SYSCALL_LOOP 5 // +checkconst . _RUN_SYSCALL_LOOP
+#define RUN_SECCOMP_LOOP 6 // +checkconst . _RUN_SECCOMP_LOOP
 
 // syscallSentryMessage offsets.
 #define SENTRY_MESSAGE_STATE 0  // +checkoffset . syscallSentryMessage.state
@@ -104,6 +106,9 @@ begin:
 	CMPQ BX, $RUN_SYSCALL_LOOP
 	JE syscall_loop
 
+	CMPQ BX, $RUN_SECCOMP_LOOP
+	JE seccomp_loop
+
 	// Notify the Sentry that syscall exited.
 done:
 	INT $3
@@ -195,6 +200,36 @@ wake_up_sentry:
 
 	INCL R13
 	JMP syscall_loop
+seccomp_loop:
+	// SYS_EXIT_GROUP triggers seccomp notifications.
+	MOVQ $SYS_EXIT_GROUP, AX
+	SYSCALL
+
+	// ret = syscall(sysno, args...)
+	MOVQ SENTRY_MESSAGE_SYSNO(R12), AX
+	MOVQ SENTRY_MESSAGE_ARG0(R12), DI
+	MOVQ SENTRY_MESSAGE_ARG1(R12), SI
+	MOVQ SENTRY_MESSAGE_ARG2(R12), DX
+	MOVQ SENTRY_MESSAGE_ARG3(R12), R10
+	MOVQ SENTRY_MESSAGE_ARG4(R12), R8
+	MOVQ SENTRY_MESSAGE_ARG5(R12), R9
+	SYSCALL
+
+	// stubMessage->ret = ret
+	MOVQ AX, (STUB_MESSAGE_OFFSET + STUB_MESSAGE_RET)(R12)
+
+	// for {
+	//   if futex(sentryMessage->state, FUTEX_WAKE, 1) == 1 {
+	//     break;
+	//   }
+	// }
+	MOVQ R12, DI
+	MOVQ $FUTEX_WAKE, SI
+	MOVQ $1, DX
+	MOVQ $0, R10
+	MOVQ $0, R8
+	MOVQ $0, R9
+	JMP seccomp_loop
 
 // func addrOfInitStubProcess() uintptr
 TEXT ·addrOfInitStubProcess(SB), $0-8

pkg/sentry/platform/systrap/stub_arm64.s

@@ -22,6 +22,7 @@
 #define SIGKILL		 9   // +checkconst unix SIGKILL
 #define SIGSTOP		 19  // +checkconst unix SIGSTOP
 #define SYS_PRCTL	 167 // +checkconst unix SYS_PRCTL
+#define SYS_EXIT_GROUP   94  // +checkconst unix SYS_EXIT_GROUP
 #define PR_SET_PDEATHSIG 1   // +checkconst unix PR_SET_PDEATHSIG
 
 #define SYS_FUTEX	 98 // +checkconst unix SYS_FUTEX
@@ -30,6 +31,7 @@
 
 #define NEW_STUB	 1 // +checkconst . _NEW_STUB
 #define RUN_SYSCALL_LOOP 5 // +checkconst . _RUN_SYSCALL_LOOP
+#define RUN_SECCOMP_LOOP 6 // +checkconst . _RUN_SECCOMP_LOOP
 
 // syscallSentryMessage offsets.
 #define SENTRY_MESSAGE_STATE 0  // +checkoffset . syscallSentryMessage.state
@@ -98,6 +100,8 @@ begin:
 	CMP $RUN_SYSCALL_LOOP, R9
 	BEQ syscall_loop
 
+	CMP $RUN_SECCOMP_LOOP, R9
+	BEQ seccomp_loop
 done:
 	// Notify the Sentry that syscall exited.
 	BRK $3
@@ -189,6 +193,23 @@ wake_up_sentry:
 
 	ADDW $1, R13, R13
 	JMP syscall_loop
+seccomp_loop:
+	// SYS_EXIT_GROUP triggers seccomp notifications.
+	MOVD $SYS_EXIT_GROUP, R8
+	SVC
+
+	MOVD SENTRY_MESSAGE_SYSNO(R12), R8
+	MOVD SENTRY_MESSAGE_ARG0(R12), R0
+	MOVD SENTRY_MESSAGE_ARG1(R12), R1
+	MOVD SENTRY_MESSAGE_ARG2(R12), R2
+	MOVD SENTRY_MESSAGE_ARG3(R12), R3
+	MOVD SENTRY_MESSAGE_ARG4(R12), R4
+	MOVD SENTRY_MESSAGE_ARG5(R12), R5
+	SVC
+
+	// stubMessage->ret = ret
+	MOVD R0, (STUB_MESSAGE_OFFSET + STUB_MESSAGE_RET)(R12)
+	JMP seccomp_loop
 
 // func addrOfInitStubProcess() uintptr
 TEXT ·addrOfInitStubProcess(SB), $0-8

pkg/sentry/platform/systrap/stub_defs.go

@@ -25,3 +25,4 @@ const _NEW_STUB = 1
 
 // _NEW_STUB is the value of the BX register when the syscall loop is executed.
 const _RUN_SYSCALL_LOOP = 5
+const _RUN_SECCOMP_LOOP = 6

pkg/sentry/platform/systrap/stub_unsafe.go

@@ -55,12 +55,21 @@ func unsafeSlice(addr uintptr, length int) (slice []byte) {
 // seccomp system call to apply these filters.
 //
 //go:nosplit
-func prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen uintptr) {
+func prepareSeccompRules(stubSysmsgStart,
+	stubSysmsgRules, stubSysmsgRulesLen,
+	stubSyscallRules, stubSyscallRulesLen uintptr) {
 	instrs := sysmsgThreadRules(stubSysmsgStart)
-	progLen := len(instrs) * int(unsafe.Sizeof(bpf.Instruction{}))
-	progPtr := stubSysmsgRules + unsafe.Sizeof(linux.SockFprog{})
+	copySeccompRulesToStub(instrs, stubSysmsgRules, stubSysmsgRulesLen)
 
-	if progLen+int(unsafe.Sizeof(linux.SockFprog{})) > int(stubSysmsgRulesLen) {
+	instrs = sysmsgSyscallNotifyRules()
+	copySeccompRulesToStub(instrs, stubSyscallRules, stubSyscallRulesLen)
+}
+
+func copySeccompRulesToStub(instrs []bpf.Instruction, stubAddr, size uintptr) {
+	progLen := len(instrs) * int(unsafe.Sizeof(bpf.Instruction{}))
+	progPtr := stubAddr + unsafe.Sizeof(linux.SockFprog{})
+
+	if progLen+int(unsafe.Sizeof(linux.SockFprog{})) > int(size) {
 		panic("not enough space for sysmsg seccomp rules")
 	}
 
@@ -75,15 +84,14 @@ func prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen ui
 	// stubSysmsgRules and progPtr are addresses from a stub mapping which
 	// is mapped once and never moved, so it is safe to use unsafe.Pointer
 	// this way for them.
-	sockProg := (*linux.SockFprog)(unsafe.Pointer(stubSysmsgRules))
+	sockProg := (*linux.SockFprog)(unsafe.Pointer(stubAddr))
 	sockProg.Len = uint16(len(instrs))
 	sockProg.Filter = (*linux.BPFInstruction)(unsafe.Pointer(progPtr))
-
 	// Make the seccomp rules stub read-only.
 	if _, _, errno := unix.RawSyscall(
 		unix.SYS_MPROTECT,
-		stubSysmsgRules,
-		stubSysmsgRulesLen,
+		stubAddr,
+		size,
 		unix.PROT_READ); errno != 0 {
 		panic("mprotect failed: " + errno.Error())
 	}
@@ -127,8 +135,11 @@ func stubInit() {
 	mapLen, _ = hostarch.PageRoundUp(mapLen + uintptr(stubSysmsgLen))
 
 	stubSysmsgRules = mapLen
-	stubSysmsgRulesLen = hostarch.PageSize * 4
+	stubSysmsgRulesLen = hostarch.PageSize * 2
 	mapLen += stubSysmsgRulesLen
+	stubSyscallRules = mapLen
+	stubSyscallRulesLen = hostarch.PageSize
+	mapLen += stubSyscallRulesLen
 
 	stubROMapEnd = mapLen
 	// Add a guard page.
@@ -211,6 +222,7 @@ func stubInit() {
 		stubSysmsgStack += sysmsg.PerThreadMemSize - offset
 	}
 	stubSysmsgRules += stubStart
+	stubSyscallRules += stubStart
 
 	targetSlice = unsafeSlice(stubSysmsgStart, stubSysmsgLen)
 	copy(targetSlice, sysmsg.SighandlerBlob)
@@ -229,7 +241,9 @@ func stubInit() {
 	p = (*uint64)(unsafe.Pointer(stubSysmsgStart + uintptr(sysmsg.Sighandler_blob_offset____export_spinning_queue_addr)))
 	*p = uint64(stubSpinningThreadQueueAddr)
 
-	prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen)
+	prepareSeccompRules(stubSysmsgStart,
+		stubSysmsgRules, stubSysmsgRulesLen,
+		stubSyscallRules, stubSyscallRulesLen)
 
 	// Make the stub executable.
 	if _, _, errno := unix.RawSyscall(

pkg/sentry/platform/systrap/subprocess.go

@@ -86,6 +86,8 @@ type thread struct {
 	//
 	// These are used for the register set for system calls.
 	initRegs arch.Registers
+
+	logPrefix atomic.Pointer[string]
 }
 
 // requestThread is used to request a new sysmsg thread. A thread identifier will
@@ -175,7 +177,7 @@ type subprocess struct {
 	dead atomicbitops.Bool
 }
 
-func (s *subprocess) initSyscallThread(ptraceThread *thread) error {
+func (s *subprocess) initSyscallThread(ptraceThread *thread, seccompNotify bool) error {
 	s.syscallThreadMu.Lock()
 	defer s.syscallThreadMu.Unlock()
 
@@ -190,7 +192,7 @@ func (s *subprocess) initSyscallThread(ptraceThread *thread) error {
 		thread:  ptraceThread,
 	}
 
-	if err := t.init(); err != nil {
+	if err := t.init(seccompNotify); err != nil {
 		panic(fmt.Sprintf("failed to create a syscall thread"))
 	}
 	s.syscallThread = &t
@@ -293,7 +295,13 @@ func (s *subprocess) handlePtraceSyscallRequest(req any) {
 // This will either be a newly created subprocess, or one from the global pool.
 // The create function will be called in the latter case, which is guaranteed
 // to happen with the runtime thread locked.
-func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFile) (*subprocess, error) {
+//
+// seccompNotify indicates a ways of comunications with syscall threads.
+// If it is false, futex-s are used. Otherwise, seccomp-unotify is used.
+// seccomp-unotify can't be used for the source pool process, because it is a
+// parent of all other stub processes, but only one filter can be installed
+// with SECCOMP_FILTER_FLAG_NEW_LISTENER.
+func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFile, seccompNotify bool) (*subprocess, error) {
 	if sp := globalPool.fetchAvailable(); sp != nil {
 		sp.subprocessRefs.InitRefs()
 		sp.usertrap = usertrap.New()
@@ -326,7 +334,7 @@ func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFil
 	}
 	sp.sysmsgInitRegs = ptraceThread.initRegs
 
-	if err := sp.initSyscallThread(ptraceThread); err != nil {
+	if err := sp.initSyscallThread(ptraceThread, seccompNotify); err != nil {
 		return nil, err
 	}
 
@@ -346,12 +354,15 @@ func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFil
 	sp.mapSharedRegions()
 	sp.mapPrivateRegions()
 
-	// Create the initial sysmsg thread.
-	atomic.AddUint32(&sp.contextQueue.numThreadsToWakeup, 1)
-	if err := sp.createSysmsgThread(); err != nil {
-		return nil, err
+	// The main stub doesn't need sysmsg threads.
+	if seccompNotify {
+		// Create the initial sysmsg thread.
+		atomic.AddUint32(&sp.contextQueue.numThreadsToWakeup, 1)
+		if err := sp.createSysmsgThread(); err != nil {
+			return nil, err
+		}
+		sp.numSysmsgThreads++
 	}
-	sp.numSysmsgThreads++
 
 	return sp, nil
 }
@@ -467,6 +478,10 @@ func (s *subprocess) Release() {
 func (s *subprocess) release() {
 	if s.alive() {
 		globalPool.markAvailable(s)
+		return
+	}
+	if s.syscallThread != nil && s.syscallThread.seccompNotify != nil {
+		s.syscallThread.seccompNotify.Close()
 	}
 }
 
@@ -522,9 +537,28 @@ const (
 	killed
 )
 
+func (t *thread) loadLogPrefix() *string {
+	p := t.logPrefix.Load()
+	if p == nil {
+		prefix := fmt.Sprintf("[% 4d:% 4d] ", t.tgid, t.tid)
+		t.logPrefix.Store(&prefix)
+		p = &prefix
+	}
+	return p
+}
+
+// Debugf logs with the debugging severity.
 func (t *thread) Debugf(format string, v ...any) {
-	prefix := fmt.Sprintf("%8d:", t.tid)
-	log.DebugfAtDepth(1, prefix+format, v...)
+	if log.IsLogging(log.Debug) {
+		log.DebugfAtDepth(1, *t.loadLogPrefix()+format, v...)
+	}
+}
+
+// Warningf logs with the warning severity.
+func (t *thread) Warningf(format string, v ...any) {
+	if log.IsLogging(log.Warning) {
+		log.WarningfAtDepth(1, *t.loadLogPrefix()+format, v...)
+	}
 }
 
 func (t *thread) dumpAndPanic(message string) {
@@ -611,7 +645,12 @@ func (t *thread) wait(outcome waitOutcome) unix.Signal {
 	}
 }
 
-// destroy kills the thread.
+// kill kills the thread;
+func (t *thread) kill() {
+	unix.Tgkill(int(t.tgid), int(t.tid), unix.Signal(unix.SIGKILL))
+}
+
+// destroy kills and waits on the thread.
 //
 // Note that this should not be used in the general case; the death of threads
 // will typically cause the death of the parent. This is a utility method for

pkg/sentry/platform/systrap/subprocess_linux.go

@@ -120,11 +120,19 @@ func attachedThread(flags uintptr, defaultAction linux.BPFAction) (*thread, erro
 					seccomp.AnyValue{},
 					seccomp.EqualTo(unix.SIGSTOP),
 				},
-				unix.SYS_GETTID: seccomp.MatchAll{},
-				seccomp.SYS_SECCOMP: seccomp.PerArg{
-					seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER),
-					seccomp.EqualTo(0),
-					seccomp.AnyValue{},
+				unix.SYS_GETTID:     seccomp.MatchAll{},
+				unix.SYS_EXIT_GROUP: seccomp.MatchAll{},
+				seccomp.SYS_SECCOMP: seccomp.Or{
+					seccomp.PerArg{
+						seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER),
+						seccomp.EqualTo(0),
+						seccomp.AnyValue{},
+					},
+					seccomp.PerArg{
+						seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER),
+						seccomp.EqualTo(linux.SECCOMP_FILTER_FLAG_NEW_LISTENER),
+						seccomp.AnyValue{},
+					},
 				},
 			}),
 			Action: linux.SECCOMP_RET_ALLOW,

pkg/sentry/platform/systrap/syscall_thread.go

@@ -16,10 +16,13 @@ package systrap
 
 import (
 	"fmt"
+	"os"
 	"sync/atomic"
 
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/seccomp"
 	"gvisor.dev/gvisor/pkg/sentry/arch"
 	"gvisor.dev/gvisor/pkg/sentry/memmap"
 	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
@@ -70,9 +73,12 @@ type syscallThread struct {
 	// stubMessage is the second page of the shared message that can be
 	// modified by the stub thread.
 	stubMessage *syscallStubMessage
+
+	seccompNotify     *os.File
+	seccompNotifyResp linux.SeccompNotifResp
 }
 
-func (t *syscallThread) init() error {
+func (t *syscallThread) init(seccompNotify bool) error {
 	// Allocate a new shared memory message.
 	opts := pgalloc.AllocOpts{
 		Kind: usage.System,
@@ -91,6 +97,10 @@ func (t *syscallThread) init() error {
 		return err
 	}
 
+	if seccompNotify {
+		t.seccompNotify = t.installSeccompNotify()
+	}
+
 	// Map the stack into the sentry.
 	sentryAddr, _, errno := unix.RawSyscall6(
 		unix.SYS_MMAP,
@@ -132,6 +142,21 @@ func (t *syscallThread) destroy() {
 	t.subproc.sysmsgStackPool.Put(t.thread.sysmsgStackID)
 }
 
+func (t *syscallThread) installSeccompNotify() *os.File {
+	fd, err := t.thread.syscallIgnoreInterrupt(&t.thread.initRegs, seccomp.SYS_SECCOMP,
+		arch.SyscallArgument{Value: uintptr(linux.SECCOMP_SET_MODE_FILTER)},
+		arch.SyscallArgument{Value: uintptr(linux.SECCOMP_FILTER_FLAG_NEW_LISTENER)},
+		arch.SyscallArgument{Value: stubSyscallRules})
+	if err != nil {
+		panic(fmt.Sprintf("seccomp failed: %v", err))
+	}
+	_, _, errno := unix.RawSyscall(unix.SYS_IOCTL, fd, linux.SECCOMP_IOCTL_NOTIF_SET_FLAGS, linux.SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP)
+	if errno != 0 {
+		t.thread.Debugf("failed to set SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP")
+	}
+	return os.NewFile(fd, "seccomp_notify")
+}
+
 // mapMessageIntoStub maps the syscall message into the stub process address space.
 func (t *syscallThread) mapMessageIntoStub() error {
 	// Map sentryMessage as read-only.
@@ -184,15 +209,28 @@ func (t *syscallThread) syscall(sysno uintptr, args ...arch.SyscallArgument) (ui
 		}
 	}
 
-	// Notify the syscall thread about a new syscall request.
-	atomic.AddUint32(&sentryMsg.state, 1)
-	futexWakeUint32(&sentryMsg.state)
+	if t.seccompNotify != nil {
+		if errno := t.kickSeccompNotify(); errno != 0 {
+			t.thread.kill()
+			t.thread.Warningf("failed sending request to syscall thread: %s", errno)
+			return 0, errDeadSubprocess
+		}
+		if err := t.waitForSeccompNotify(); err != nil {
+			t.thread.Warningf("failed waiting for seccomp notify: %s", err)
+			return 0, errDeadSubprocess
+		}
+	} else {
 
-	// Wait for reply.
-	//
-	// futex waits for sentryMsg.state that isn't changed, so it will
-	// returns only only when the other side will call FUTEX_WAKE.
-	futexWaitWake(&sentryMsg.state, atomic.LoadUint32(&sentryMsg.state))
+		// Notify the syscall thread about a new syscall request.
+		atomic.AddUint32(&sentryMsg.state, 1)
+		futexWakeUint32(&sentryMsg.state)
+
+		// Wait for reply.
+		//
+		// futex waits for sentryMsg.state that isn't changed, so it will
+		// returns only only when the other side will call FUTEX_WAKE.
+		futexWaitWake(&sentryMsg.state, atomic.LoadUint32(&sentryMsg.state))
+	}
 
 	errno := -uintptr(stubMsg.ret)
 	if errno > 0 && errno < maxErrno {

pkg/sentry/platform/systrap/syscall_thread_amd64.go

@@ -36,7 +36,11 @@ func (t *syscallThread) detach() {
 	regs.Rsp = 0
 	regs.R12 = uint64(t.stubAddr)
 	regs.R13 = uint64(t.sentryMessage.state + 1)
-	regs.Rbx = _RUN_SYSCALL_LOOP
+	if t.seccompNotify != nil {
+		regs.Rbx = _RUN_SECCOMP_LOOP
+	} else {
+		regs.Rbx = _RUN_SYSCALL_LOOP
+	}
 	// Skip the syscall instruction.
 	regs.Rip += arch.SyscallWidth
 	if err := p.setRegs(&regs); err != nil {
@@ -47,4 +51,10 @@ func (t *syscallThread) detach() {
 		panic(fmt.Sprintf("tkill failed: %v", e))
 	}
 	runtime.UnlockOSThread()
+
+	if t.seccompNotify != nil {
+		if err := t.waitForSeccompNotify(); err != nil {
+			panic(fmt.Sprintf("%s", err))
+		}
+	}
 }

pkg/sentry/platform/systrap/syscall_thread_arm64.go

@@ -36,7 +36,11 @@ func (t *syscallThread) detach() {
 	regs.Sp = 0
 	regs.Regs[12] = uint64(t.stubAddr)
 	regs.Regs[13] = uint64(t.sentryMessage.state + 1)
-	regs.Regs[9] = _RUN_SYSCALL_LOOP
+	if t.seccompNotify != nil {
+		regs.Regs[9] = _RUN_SECCOMP_LOOP
+	} else {
+		regs.Regs[9] = _RUN_SYSCALL_LOOP
+	}
 	// Skip the syscall instruction.
 	regs.Pc += arch.SyscallWidth
 	if err := p.setRegs(&regs); err != nil {
@@ -47,4 +51,10 @@ func (t *syscallThread) detach() {
 		panic(fmt.Sprintf("tkill failed: %v", e))
 	}
 	runtime.UnlockOSThread()
+
+	if t.seccompNotify != nil {
+		if err := t.waitForSeccompNotify(); err != nil {
+			panic(fmt.Sprintf("%s", err))
+		}
+	}
 }

pkg/sentry/platform/systrap/syscall_thread_unsafe.go

@@ -89,3 +89,29 @@ func futexWaitWake(futexAddr *uint32, futexValue uint32) error {
 
 	return nil
 }
+
+func (t *syscallThread) kickSeccompNotify() unix.Errno {
+	_, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(t.seccompNotify.Fd()),
+		uintptr(linux.SECCOMP_IOCTL_NOTIF_SEND),
+		uintptr(unsafe.Pointer(&t.seccompNotifyResp)))
+	return errno
+}
+
+func (t *syscallThread) waitForSeccompNotify() error {
+	for {
+		req := linux.SeccompNotif{}
+		_, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(t.seccompNotify.Fd()),
+			uintptr(linux.SECCOMP_IOCTL_NOTIF_RECV),
+			uintptr(unsafe.Pointer(&req)))
+		if errno == 0 {
+			t.seccompNotifyResp.ID = req.ID
+			break
+		}
+		if errno == unix.EINTR && t.subproc.alive() {
+			continue
+		}
+		t.thread.kill()
+		return fmt.Errorf("failed getting response from syscall thread : %w", errno)
+	}
+	return nil
+}

pkg/sentry/platform/systrap/sysmsg_thread.go

@@ -97,6 +97,25 @@ func (p *sysmsgThread) Debugf(format string, v ...any) {
 	p.thread.Debugf(format+postfix, v...)
 }
 
+func sysmsgSyscallNotifyRules() []bpf.Instruction {
+	rules := []seccomp.RuleSet{
+		seccomp.RuleSet{
+			Rules: seccomp.MakeSyscallRules(map[uintptr]seccomp.SyscallRule{
+				unix.SYS_EXIT_GROUP: seccomp.MatchAll{},
+			}),
+			Action: linux.SECCOMP_RET_USER_NOTIF,
+		},
+	}
+	instrs, _, err := seccomp.BuildProgram(rules, seccomp.ProgramOptions{
+		DefaultAction: linux.SECCOMP_RET_ALLOW,
+		BadArchAction: linux.SECCOMP_RET_ALLOW,
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to build rules for sysmsg threads: %v", err))
+	}
+	return instrs
+}
+
 func sysmsgThreadRules(stubStart uintptr) []bpf.Instruction {
 	rules := []seccomp.RuleSet{}
 	rules = appendSysThreadArchSeccompRules(rules)

pkg/sentry/platform/systrap/systrap.go

@@ -86,8 +86,10 @@ var (
 	stubContextRegion    uintptr
 	stubContextRegionLen uintptr
 	// The memory blob with precompiled seccomp rules.
-	stubSysmsgRules    uintptr
-	stubSysmsgRulesLen uintptr
+	stubSysmsgRules     uintptr
+	stubSysmsgRulesLen  uintptr
+	stubSyscallRules    uintptr
+	stubSyscallRulesLen uintptr
 
 	stubSpinningThreadQueueAddr uintptr
 	stubSpinningThreadQueueSize uintptr
@@ -329,7 +331,7 @@ func New() (*Systrap, error) {
 
 		// Create the source process for the global pool. This must be
 		// done before initializing any other processes.
-		source, err := newSubprocess(createStub, mf)
+		source, err := newSubprocess(createStub, mf, false)
 		if err != nil {
 			// Should never happen.
 			panic("unable to initialize systrap source: " + err.Error())
@@ -374,7 +376,7 @@ func (*Systrap) MaxUserAddress() hostarch.Addr {
 
 // NewAddressSpace returns a new subprocess.
 func (p *Systrap) NewAddressSpace(any) (platform.AddressSpace, <-chan struct{}, error) {
-	as, err := newSubprocess(globalPool.source.createStub, p.memoryFile)
+	as, err := newSubprocess(globalPool.source.createStub, p.memoryFile, true)
 	return as, nil, err
 }