diff --git a/pkg/abi/linux/seccomp.go b/pkg/abi/linux/seccomp.go index 5e98dd8c0..a8cc45fdb 100644 --- a/pkg/abi/linux/seccomp.go +++ b/pkg/abi/linux/seccomp.go @@ -25,9 +25,20 @@ const ( SECCOMP_RET_ACTION = 0x7fff0000 SECCOMP_RET_DATA = 0x0000ffff - SECCOMP_SET_MODE_FILTER = 1 - SECCOMP_FILTER_FLAG_TSYNC = 1 - SECCOMP_GET_ACTION_AVAIL = 2 + SECCOMP_SET_MODE_FILTER = 1 + SECCOMP_GET_ACTION_AVAIL = 2 + SECCOMP_GET_NOTIF_SIZES = 3 + + SECCOMP_FILTER_FLAG_TSYNC = 1 + SECCOMP_FILTER_FLAG_NEW_LISTENER = 1 << 3 + + SECCOMP_USER_NOTIF_FLAG_CONTINUE = 1 + + SECCOMP_IOCTL_NOTIF_RECV = 0xc0502100 + SECCOMP_IOCTL_NOTIF_SEND = 0xc0182101 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 + + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP = 1 ) // BPFAction is an action for a BPF filter. @@ -40,6 +51,7 @@ const ( SECCOMP_RET_TRAP BPFAction = 0x00030000 SECCOMP_RET_ERRNO BPFAction = 0x00050000 SECCOMP_RET_TRACE BPFAction = 0x7ff00000 + SECCOMP_RET_USER_NOTIF BPFAction = 0x7fc00000 SECCOMP_RET_ALLOW BPFAction = 0x7fff0000 ) @@ -65,6 +77,8 @@ func (a BPFAction) String() string { return fmt.Sprintf("trace (data=%#x)", data) case SECCOMP_RET_ALLOW: return "allow" + case SECCOMP_RET_USER_NOTIF: + return "unotify" } return fmt.Sprintf("invalid action: %#x", a) } @@ -113,6 +127,35 @@ type SeccompData struct { Args [6]uint64 } +// SeccompNotifResp is equivalent to struct seccomp_notif_resp. +// +// +marshal +type SeccompNotifResp struct { + ID uint64 + Val int64 + Error int32 + Flags uint32 +} + +// SeccompNotifSizes is equivalent to struct seccomp_notif_sizes. +// +// +marshal +type SeccompNotifSizes struct { + Notif uint16 + Notif_resp uint16 + Data uint16 +} + +// SeccompNotif is equivalent to struct seccomp_notif. +// +// +marshal +type SeccompNotif struct { + ID uint64 + Pid int32 + Flags uint32 + Data SeccompData +} + // String returns a human-friendly representation of this `SeccompData`. func (sd SeccompData) String() string { return fmt.Sprintf( diff --git a/pkg/sentry/platform/systrap/filters.go b/pkg/sentry/platform/systrap/filters.go index 7dc2cd5a1..932969a17 100644 --- a/pkg/sentry/platform/systrap/filters.go +++ b/pkg/sentry/platform/systrap/filters.go @@ -94,6 +94,21 @@ func (systrapSeccomp) SyscallFilters(vars precompiledseccomp.Values) seccomp.Sys }, unix.SYS_TGKILL: seccomp.MatchAll{}, unix.SYS_WAIT4: seccomp.MatchAll{}, + unix.SYS_IOCTL: seccomp.Or{ + seccomp.PerArg{ + seccomp.NonNegativeFD{}, + seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_RECV), + }, + seccomp.PerArg{ + seccomp.NonNegativeFD{}, + seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_SEND), + }, + seccomp.PerArg{ + seccomp.NonNegativeFD{}, + seccomp.EqualTo(linux.SECCOMP_IOCTL_NOTIF_SET_FLAGS), + seccomp.EqualTo(linux.SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP), + }, + }, unix.SYS_WAITID: seccomp.PerArg{ seccomp.EqualTo(unix.P_PID), seccomp.AnyValue{}, diff --git a/pkg/sentry/platform/systrap/stub_amd64.s b/pkg/sentry/platform/systrap/stub_amd64.s index f0bee6762..fe78ca15a 100644 --- a/pkg/sentry/platform/systrap/stub_amd64.s +++ b/pkg/sentry/platform/systrap/stub_amd64.s @@ -22,6 +22,7 @@ #define SIGKILL 9 // +checkconst unix SIGKILL #define SIGSTOP 19 // +checkconst unix SIGSTOP #define SYS_PRCTL 157 // +checkconst unix SYS_PRCTL +#define SYS_EXIT_GROUP 231 // +checkconst unix SYS_EXIT_GROUP #define PR_SET_PDEATHSIG 1 // +checkconst unix PR_SET_PDEATHSIG #define SYS_FUTEX 202 // +checkconst unix SYS_FUTEX @@ -30,6 +31,7 @@ #define NEW_STUB 1 // +checkconst . _NEW_STUB #define RUN_SYSCALL_LOOP 5 // +checkconst . _RUN_SYSCALL_LOOP +#define RUN_SECCOMP_LOOP 6 // +checkconst . _RUN_SECCOMP_LOOP // syscallSentryMessage offsets. #define SENTRY_MESSAGE_STATE 0 // +checkoffset . syscallSentryMessage.state @@ -104,6 +106,9 @@ begin: CMPQ BX, $RUN_SYSCALL_LOOP JE syscall_loop + CMPQ BX, $RUN_SECCOMP_LOOP + JE seccomp_loop + // Notify the Sentry that syscall exited. done: INT $3 @@ -195,6 +200,36 @@ wake_up_sentry: INCL R13 JMP syscall_loop +seccomp_loop: + // SYS_EXIT_GROUP triggers seccomp notifications. + MOVQ $SYS_EXIT_GROUP, AX + SYSCALL + + // ret = syscall(sysno, args...) + MOVQ SENTRY_MESSAGE_SYSNO(R12), AX + MOVQ SENTRY_MESSAGE_ARG0(R12), DI + MOVQ SENTRY_MESSAGE_ARG1(R12), SI + MOVQ SENTRY_MESSAGE_ARG2(R12), DX + MOVQ SENTRY_MESSAGE_ARG3(R12), R10 + MOVQ SENTRY_MESSAGE_ARG4(R12), R8 + MOVQ SENTRY_MESSAGE_ARG5(R12), R9 + SYSCALL + + // stubMessage->ret = ret + MOVQ AX, (STUB_MESSAGE_OFFSET + STUB_MESSAGE_RET)(R12) + + // for { + // if futex(sentryMessage->state, FUTEX_WAKE, 1) == 1 { + // break; + // } + // } + MOVQ R12, DI + MOVQ $FUTEX_WAKE, SI + MOVQ $1, DX + MOVQ $0, R10 + MOVQ $0, R8 + MOVQ $0, R9 + JMP seccomp_loop // func addrOfInitStubProcess() uintptr TEXT ·addrOfInitStubProcess(SB), $0-8 diff --git a/pkg/sentry/platform/systrap/stub_arm64.s b/pkg/sentry/platform/systrap/stub_arm64.s index 62c583d50..2bae8b789 100644 --- a/pkg/sentry/platform/systrap/stub_arm64.s +++ b/pkg/sentry/platform/systrap/stub_arm64.s @@ -22,6 +22,7 @@ #define SIGKILL 9 // +checkconst unix SIGKILL #define SIGSTOP 19 // +checkconst unix SIGSTOP #define SYS_PRCTL 167 // +checkconst unix SYS_PRCTL +#define SYS_EXIT_GROUP 94 // +checkconst unix SYS_EXIT_GROUP #define PR_SET_PDEATHSIG 1 // +checkconst unix PR_SET_PDEATHSIG #define SYS_FUTEX 98 // +checkconst unix SYS_FUTEX @@ -30,6 +31,7 @@ #define NEW_STUB 1 // +checkconst . _NEW_STUB #define RUN_SYSCALL_LOOP 5 // +checkconst . _RUN_SYSCALL_LOOP +#define RUN_SECCOMP_LOOP 6 // +checkconst . _RUN_SECCOMP_LOOP // syscallSentryMessage offsets. #define SENTRY_MESSAGE_STATE 0 // +checkoffset . syscallSentryMessage.state @@ -98,6 +100,8 @@ begin: CMP $RUN_SYSCALL_LOOP, R9 BEQ syscall_loop + CMP $RUN_SECCOMP_LOOP, R9 + BEQ seccomp_loop done: // Notify the Sentry that syscall exited. BRK $3 @@ -189,6 +193,23 @@ wake_up_sentry: ADDW $1, R13, R13 JMP syscall_loop +seccomp_loop: + // SYS_EXIT_GROUP triggers seccomp notifications. + MOVD $SYS_EXIT_GROUP, R8 + SVC + + MOVD SENTRY_MESSAGE_SYSNO(R12), R8 + MOVD SENTRY_MESSAGE_ARG0(R12), R0 + MOVD SENTRY_MESSAGE_ARG1(R12), R1 + MOVD SENTRY_MESSAGE_ARG2(R12), R2 + MOVD SENTRY_MESSAGE_ARG3(R12), R3 + MOVD SENTRY_MESSAGE_ARG4(R12), R4 + MOVD SENTRY_MESSAGE_ARG5(R12), R5 + SVC + + // stubMessage->ret = ret + MOVD R0, (STUB_MESSAGE_OFFSET + STUB_MESSAGE_RET)(R12) + JMP seccomp_loop // func addrOfInitStubProcess() uintptr TEXT ·addrOfInitStubProcess(SB), $0-8 diff --git a/pkg/sentry/platform/systrap/stub_defs.go b/pkg/sentry/platform/systrap/stub_defs.go index d596a86ea..396d1d1b0 100644 --- a/pkg/sentry/platform/systrap/stub_defs.go +++ b/pkg/sentry/platform/systrap/stub_defs.go @@ -25,3 +25,4 @@ const _NEW_STUB = 1 // _NEW_STUB is the value of the BX register when the syscall loop is executed. const _RUN_SYSCALL_LOOP = 5 +const _RUN_SECCOMP_LOOP = 6 diff --git a/pkg/sentry/platform/systrap/stub_unsafe.go b/pkg/sentry/platform/systrap/stub_unsafe.go index 1c5b89f4b..25a119aa7 100644 --- a/pkg/sentry/platform/systrap/stub_unsafe.go +++ b/pkg/sentry/platform/systrap/stub_unsafe.go @@ -55,12 +55,21 @@ func unsafeSlice(addr uintptr, length int) (slice []byte) { // seccomp system call to apply these filters. // //go:nosplit -func prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen uintptr) { +func prepareSeccompRules(stubSysmsgStart, + stubSysmsgRules, stubSysmsgRulesLen, + stubSyscallRules, stubSyscallRulesLen uintptr) { instrs := sysmsgThreadRules(stubSysmsgStart) - progLen := len(instrs) * int(unsafe.Sizeof(bpf.Instruction{})) - progPtr := stubSysmsgRules + unsafe.Sizeof(linux.SockFprog{}) + copySeccompRulesToStub(instrs, stubSysmsgRules, stubSysmsgRulesLen) - if progLen+int(unsafe.Sizeof(linux.SockFprog{})) > int(stubSysmsgRulesLen) { + instrs = sysmsgSyscallNotifyRules() + copySeccompRulesToStub(instrs, stubSyscallRules, stubSyscallRulesLen) +} + +func copySeccompRulesToStub(instrs []bpf.Instruction, stubAddr, size uintptr) { + progLen := len(instrs) * int(unsafe.Sizeof(bpf.Instruction{})) + progPtr := stubAddr + unsafe.Sizeof(linux.SockFprog{}) + + if progLen+int(unsafe.Sizeof(linux.SockFprog{})) > int(size) { panic("not enough space for sysmsg seccomp rules") } @@ -75,15 +84,14 @@ func prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen ui // stubSysmsgRules and progPtr are addresses from a stub mapping which // is mapped once and never moved, so it is safe to use unsafe.Pointer // this way for them. - sockProg := (*linux.SockFprog)(unsafe.Pointer(stubSysmsgRules)) + sockProg := (*linux.SockFprog)(unsafe.Pointer(stubAddr)) sockProg.Len = uint16(len(instrs)) sockProg.Filter = (*linux.BPFInstruction)(unsafe.Pointer(progPtr)) - // Make the seccomp rules stub read-only. if _, _, errno := unix.RawSyscall( unix.SYS_MPROTECT, - stubSysmsgRules, - stubSysmsgRulesLen, + stubAddr, + size, unix.PROT_READ); errno != 0 { panic("mprotect failed: " + errno.Error()) } @@ -127,8 +135,11 @@ func stubInit() { mapLen, _ = hostarch.PageRoundUp(mapLen + uintptr(stubSysmsgLen)) stubSysmsgRules = mapLen - stubSysmsgRulesLen = hostarch.PageSize * 4 + stubSysmsgRulesLen = hostarch.PageSize * 2 mapLen += stubSysmsgRulesLen + stubSyscallRules = mapLen + stubSyscallRulesLen = hostarch.PageSize + mapLen += stubSyscallRulesLen stubROMapEnd = mapLen // Add a guard page. @@ -211,6 +222,7 @@ func stubInit() { stubSysmsgStack += sysmsg.PerThreadMemSize - offset } stubSysmsgRules += stubStart + stubSyscallRules += stubStart targetSlice = unsafeSlice(stubSysmsgStart, stubSysmsgLen) copy(targetSlice, sysmsg.SighandlerBlob) @@ -229,7 +241,9 @@ func stubInit() { p = (*uint64)(unsafe.Pointer(stubSysmsgStart + uintptr(sysmsg.Sighandler_blob_offset____export_spinning_queue_addr))) *p = uint64(stubSpinningThreadQueueAddr) - prepareSeccompRules(stubSysmsgStart, stubSysmsgRules, stubSysmsgRulesLen) + prepareSeccompRules(stubSysmsgStart, + stubSysmsgRules, stubSysmsgRulesLen, + stubSyscallRules, stubSyscallRulesLen) // Make the stub executable. if _, _, errno := unix.RawSyscall( diff --git a/pkg/sentry/platform/systrap/subprocess.go b/pkg/sentry/platform/systrap/subprocess.go index 900514d80..d978c69f1 100644 --- a/pkg/sentry/platform/systrap/subprocess.go +++ b/pkg/sentry/platform/systrap/subprocess.go @@ -86,6 +86,8 @@ type thread struct { // // These are used for the register set for system calls. initRegs arch.Registers + + logPrefix atomic.Pointer[string] } // requestThread is used to request a new sysmsg thread. A thread identifier will @@ -175,7 +177,7 @@ type subprocess struct { dead atomicbitops.Bool } -func (s *subprocess) initSyscallThread(ptraceThread *thread) error { +func (s *subprocess) initSyscallThread(ptraceThread *thread, seccompNotify bool) error { s.syscallThreadMu.Lock() defer s.syscallThreadMu.Unlock() @@ -190,7 +192,7 @@ func (s *subprocess) initSyscallThread(ptraceThread *thread) error { thread: ptraceThread, } - if err := t.init(); err != nil { + if err := t.init(seccompNotify); err != nil { panic(fmt.Sprintf("failed to create a syscall thread")) } s.syscallThread = &t @@ -293,7 +295,13 @@ func (s *subprocess) handlePtraceSyscallRequest(req any) { // This will either be a newly created subprocess, or one from the global pool. // The create function will be called in the latter case, which is guaranteed // to happen with the runtime thread locked. -func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFile) (*subprocess, error) { +// +// seccompNotify indicates a ways of comunications with syscall threads. +// If it is false, futex-s are used. Otherwise, seccomp-unotify is used. +// seccomp-unotify can't be used for the source pool process, because it is a +// parent of all other stub processes, but only one filter can be installed +// with SECCOMP_FILTER_FLAG_NEW_LISTENER. +func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFile, seccompNotify bool) (*subprocess, error) { if sp := globalPool.fetchAvailable(); sp != nil { sp.subprocessRefs.InitRefs() sp.usertrap = usertrap.New() @@ -326,7 +334,7 @@ func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFil } sp.sysmsgInitRegs = ptraceThread.initRegs - if err := sp.initSyscallThread(ptraceThread); err != nil { + if err := sp.initSyscallThread(ptraceThread, seccompNotify); err != nil { return nil, err } @@ -346,12 +354,15 @@ func newSubprocess(create func() (*thread, error), memoryFile *pgalloc.MemoryFil sp.mapSharedRegions() sp.mapPrivateRegions() - // Create the initial sysmsg thread. - atomic.AddUint32(&sp.contextQueue.numThreadsToWakeup, 1) - if err := sp.createSysmsgThread(); err != nil { - return nil, err + // The main stub doesn't need sysmsg threads. + if seccompNotify { + // Create the initial sysmsg thread. + atomic.AddUint32(&sp.contextQueue.numThreadsToWakeup, 1) + if err := sp.createSysmsgThread(); err != nil { + return nil, err + } + sp.numSysmsgThreads++ } - sp.numSysmsgThreads++ return sp, nil } @@ -467,6 +478,10 @@ func (s *subprocess) Release() { func (s *subprocess) release() { if s.alive() { globalPool.markAvailable(s) + return + } + if s.syscallThread != nil && s.syscallThread.seccompNotify != nil { + s.syscallThread.seccompNotify.Close() } } @@ -522,9 +537,28 @@ const ( killed ) +func (t *thread) loadLogPrefix() *string { + p := t.logPrefix.Load() + if p == nil { + prefix := fmt.Sprintf("[% 4d:% 4d] ", t.tgid, t.tid) + t.logPrefix.Store(&prefix) + p = &prefix + } + return p +} + +// Debugf logs with the debugging severity. func (t *thread) Debugf(format string, v ...any) { - prefix := fmt.Sprintf("%8d:", t.tid) - log.DebugfAtDepth(1, prefix+format, v...) + if log.IsLogging(log.Debug) { + log.DebugfAtDepth(1, *t.loadLogPrefix()+format, v...) + } +} + +// Warningf logs with the warning severity. +func (t *thread) Warningf(format string, v ...any) { + if log.IsLogging(log.Warning) { + log.WarningfAtDepth(1, *t.loadLogPrefix()+format, v...) + } } func (t *thread) dumpAndPanic(message string) { @@ -611,7 +645,12 @@ func (t *thread) wait(outcome waitOutcome) unix.Signal { } } -// destroy kills the thread. +// kill kills the thread; +func (t *thread) kill() { + unix.Tgkill(int(t.tgid), int(t.tid), unix.Signal(unix.SIGKILL)) +} + +// destroy kills and waits on the thread. // // Note that this should not be used in the general case; the death of threads // will typically cause the death of the parent. This is a utility method for diff --git a/pkg/sentry/platform/systrap/subprocess_linux.go b/pkg/sentry/platform/systrap/subprocess_linux.go index 4e551823c..844f5043a 100644 --- a/pkg/sentry/platform/systrap/subprocess_linux.go +++ b/pkg/sentry/platform/systrap/subprocess_linux.go @@ -120,11 +120,19 @@ func attachedThread(flags uintptr, defaultAction linux.BPFAction) (*thread, erro seccomp.AnyValue{}, seccomp.EqualTo(unix.SIGSTOP), }, - unix.SYS_GETTID: seccomp.MatchAll{}, - seccomp.SYS_SECCOMP: seccomp.PerArg{ - seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER), - seccomp.EqualTo(0), - seccomp.AnyValue{}, + unix.SYS_GETTID: seccomp.MatchAll{}, + unix.SYS_EXIT_GROUP: seccomp.MatchAll{}, + seccomp.SYS_SECCOMP: seccomp.Or{ + seccomp.PerArg{ + seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER), + seccomp.EqualTo(0), + seccomp.AnyValue{}, + }, + seccomp.PerArg{ + seccomp.EqualTo(linux.SECCOMP_SET_MODE_FILTER), + seccomp.EqualTo(linux.SECCOMP_FILTER_FLAG_NEW_LISTENER), + seccomp.AnyValue{}, + }, }, }), Action: linux.SECCOMP_RET_ALLOW, diff --git a/pkg/sentry/platform/systrap/syscall_thread.go b/pkg/sentry/platform/systrap/syscall_thread.go index 2dfdd96b7..d4fce702b 100644 --- a/pkg/sentry/platform/systrap/syscall_thread.go +++ b/pkg/sentry/platform/systrap/syscall_thread.go @@ -16,10 +16,13 @@ package systrap import ( "fmt" + "os" "sync/atomic" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/seccomp" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/pgalloc" @@ -70,9 +73,12 @@ type syscallThread struct { // stubMessage is the second page of the shared message that can be // modified by the stub thread. stubMessage *syscallStubMessage + + seccompNotify *os.File + seccompNotifyResp linux.SeccompNotifResp } -func (t *syscallThread) init() error { +func (t *syscallThread) init(seccompNotify bool) error { // Allocate a new shared memory message. opts := pgalloc.AllocOpts{ Kind: usage.System, @@ -91,6 +97,10 @@ func (t *syscallThread) init() error { return err } + if seccompNotify { + t.seccompNotify = t.installSeccompNotify() + } + // Map the stack into the sentry. sentryAddr, _, errno := unix.RawSyscall6( unix.SYS_MMAP, @@ -132,6 +142,21 @@ func (t *syscallThread) destroy() { t.subproc.sysmsgStackPool.Put(t.thread.sysmsgStackID) } +func (t *syscallThread) installSeccompNotify() *os.File { + fd, err := t.thread.syscallIgnoreInterrupt(&t.thread.initRegs, seccomp.SYS_SECCOMP, + arch.SyscallArgument{Value: uintptr(linux.SECCOMP_SET_MODE_FILTER)}, + arch.SyscallArgument{Value: uintptr(linux.SECCOMP_FILTER_FLAG_NEW_LISTENER)}, + arch.SyscallArgument{Value: stubSyscallRules}) + if err != nil { + panic(fmt.Sprintf("seccomp failed: %v", err)) + } + _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, fd, linux.SECCOMP_IOCTL_NOTIF_SET_FLAGS, linux.SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + if errno != 0 { + t.thread.Debugf("failed to set SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP") + } + return os.NewFile(fd, "seccomp_notify") +} + // mapMessageIntoStub maps the syscall message into the stub process address space. func (t *syscallThread) mapMessageIntoStub() error { // Map sentryMessage as read-only. @@ -184,15 +209,28 @@ func (t *syscallThread) syscall(sysno uintptr, args ...arch.SyscallArgument) (ui } } - // Notify the syscall thread about a new syscall request. - atomic.AddUint32(&sentryMsg.state, 1) - futexWakeUint32(&sentryMsg.state) + if t.seccompNotify != nil { + if errno := t.kickSeccompNotify(); errno != 0 { + t.thread.kill() + t.thread.Warningf("failed sending request to syscall thread: %s", errno) + return 0, errDeadSubprocess + } + if err := t.waitForSeccompNotify(); err != nil { + t.thread.Warningf("failed waiting for seccomp notify: %s", err) + return 0, errDeadSubprocess + } + } else { - // Wait for reply. - // - // futex waits for sentryMsg.state that isn't changed, so it will - // returns only only when the other side will call FUTEX_WAKE. - futexWaitWake(&sentryMsg.state, atomic.LoadUint32(&sentryMsg.state)) + // Notify the syscall thread about a new syscall request. + atomic.AddUint32(&sentryMsg.state, 1) + futexWakeUint32(&sentryMsg.state) + + // Wait for reply. + // + // futex waits for sentryMsg.state that isn't changed, so it will + // returns only only when the other side will call FUTEX_WAKE. + futexWaitWake(&sentryMsg.state, atomic.LoadUint32(&sentryMsg.state)) + } errno := -uintptr(stubMsg.ret) if errno > 0 && errno < maxErrno { diff --git a/pkg/sentry/platform/systrap/syscall_thread_amd64.go b/pkg/sentry/platform/systrap/syscall_thread_amd64.go index fe81dd399..179c366ef 100644 --- a/pkg/sentry/platform/systrap/syscall_thread_amd64.go +++ b/pkg/sentry/platform/systrap/syscall_thread_amd64.go @@ -36,7 +36,11 @@ func (t *syscallThread) detach() { regs.Rsp = 0 regs.R12 = uint64(t.stubAddr) regs.R13 = uint64(t.sentryMessage.state + 1) - regs.Rbx = _RUN_SYSCALL_LOOP + if t.seccompNotify != nil { + regs.Rbx = _RUN_SECCOMP_LOOP + } else { + regs.Rbx = _RUN_SYSCALL_LOOP + } // Skip the syscall instruction. regs.Rip += arch.SyscallWidth if err := p.setRegs(®s); err != nil { @@ -47,4 +51,10 @@ func (t *syscallThread) detach() { panic(fmt.Sprintf("tkill failed: %v", e)) } runtime.UnlockOSThread() + + if t.seccompNotify != nil { + if err := t.waitForSeccompNotify(); err != nil { + panic(fmt.Sprintf("%s", err)) + } + } } diff --git a/pkg/sentry/platform/systrap/syscall_thread_arm64.go b/pkg/sentry/platform/systrap/syscall_thread_arm64.go index c9b4201b4..055aaba84 100644 --- a/pkg/sentry/platform/systrap/syscall_thread_arm64.go +++ b/pkg/sentry/platform/systrap/syscall_thread_arm64.go @@ -36,7 +36,11 @@ func (t *syscallThread) detach() { regs.Sp = 0 regs.Regs[12] = uint64(t.stubAddr) regs.Regs[13] = uint64(t.sentryMessage.state + 1) - regs.Regs[9] = _RUN_SYSCALL_LOOP + if t.seccompNotify != nil { + regs.Regs[9] = _RUN_SECCOMP_LOOP + } else { + regs.Regs[9] = _RUN_SYSCALL_LOOP + } // Skip the syscall instruction. regs.Pc += arch.SyscallWidth if err := p.setRegs(®s); err != nil { @@ -47,4 +51,10 @@ func (t *syscallThread) detach() { panic(fmt.Sprintf("tkill failed: %v", e)) } runtime.UnlockOSThread() + + if t.seccompNotify != nil { + if err := t.waitForSeccompNotify(); err != nil { + panic(fmt.Sprintf("%s", err)) + } + } } diff --git a/pkg/sentry/platform/systrap/syscall_thread_unsafe.go b/pkg/sentry/platform/systrap/syscall_thread_unsafe.go index 73f7744da..898cdb8ad 100644 --- a/pkg/sentry/platform/systrap/syscall_thread_unsafe.go +++ b/pkg/sentry/platform/systrap/syscall_thread_unsafe.go @@ -89,3 +89,29 @@ func futexWaitWake(futexAddr *uint32, futexValue uint32) error { return nil } + +func (t *syscallThread) kickSeccompNotify() unix.Errno { + _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(t.seccompNotify.Fd()), + uintptr(linux.SECCOMP_IOCTL_NOTIF_SEND), + uintptr(unsafe.Pointer(&t.seccompNotifyResp))) + return errno +} + +func (t *syscallThread) waitForSeccompNotify() error { + for { + req := linux.SeccompNotif{} + _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(t.seccompNotify.Fd()), + uintptr(linux.SECCOMP_IOCTL_NOTIF_RECV), + uintptr(unsafe.Pointer(&req))) + if errno == 0 { + t.seccompNotifyResp.ID = req.ID + break + } + if errno == unix.EINTR && t.subproc.alive() { + continue + } + t.thread.kill() + return fmt.Errorf("failed getting response from syscall thread : %w", errno) + } + return nil +} diff --git a/pkg/sentry/platform/systrap/sysmsg_thread.go b/pkg/sentry/platform/systrap/sysmsg_thread.go index 8ea66f2f4..56587d260 100644 --- a/pkg/sentry/platform/systrap/sysmsg_thread.go +++ b/pkg/sentry/platform/systrap/sysmsg_thread.go @@ -97,6 +97,25 @@ func (p *sysmsgThread) Debugf(format string, v ...any) { p.thread.Debugf(format+postfix, v...) } +func sysmsgSyscallNotifyRules() []bpf.Instruction { + rules := []seccomp.RuleSet{ + seccomp.RuleSet{ + Rules: seccomp.MakeSyscallRules(map[uintptr]seccomp.SyscallRule{ + unix.SYS_EXIT_GROUP: seccomp.MatchAll{}, + }), + Action: linux.SECCOMP_RET_USER_NOTIF, + }, + } + instrs, _, err := seccomp.BuildProgram(rules, seccomp.ProgramOptions{ + DefaultAction: linux.SECCOMP_RET_ALLOW, + BadArchAction: linux.SECCOMP_RET_ALLOW, + }) + if err != nil { + panic(fmt.Sprintf("failed to build rules for sysmsg threads: %v", err)) + } + return instrs +} + func sysmsgThreadRules(stubStart uintptr) []bpf.Instruction { rules := []seccomp.RuleSet{} rules = appendSysThreadArchSeccompRules(rules) diff --git a/pkg/sentry/platform/systrap/systrap.go b/pkg/sentry/platform/systrap/systrap.go index 40e08732e..97bdbfda0 100644 --- a/pkg/sentry/platform/systrap/systrap.go +++ b/pkg/sentry/platform/systrap/systrap.go @@ -86,8 +86,10 @@ var ( stubContextRegion uintptr stubContextRegionLen uintptr // The memory blob with precompiled seccomp rules. - stubSysmsgRules uintptr - stubSysmsgRulesLen uintptr + stubSysmsgRules uintptr + stubSysmsgRulesLen uintptr + stubSyscallRules uintptr + stubSyscallRulesLen uintptr stubSpinningThreadQueueAddr uintptr stubSpinningThreadQueueSize uintptr @@ -329,7 +331,7 @@ func New() (*Systrap, error) { // Create the source process for the global pool. This must be // done before initializing any other processes. - source, err := newSubprocess(createStub, mf) + source, err := newSubprocess(createStub, mf, false) if err != nil { // Should never happen. panic("unable to initialize systrap source: " + err.Error()) @@ -374,7 +376,7 @@ func (*Systrap) MaxUserAddress() hostarch.Addr { // NewAddressSpace returns a new subprocess. func (p *Systrap) NewAddressSpace(any) (platform.AddressSpace, <-chan struct{}, error) { - as, err := newSubprocess(globalPool.source.createStub, p.memoryFile) + as, err := newSubprocess(globalPool.source.createStub, p.memoryFile, true) return as, nil, err }