diff --git a/pkg/sentry/kernel/BUILD b/pkg/sentry/kernel/BUILD index a9495ed31..b40bd63c5 100644 --- a/pkg/sentry/kernel/BUILD +++ b/pkg/sentry/kernel/BUILD @@ -40,13 +40,6 @@ go_template_instance( }, ) -declare_mutex( - name = "cpu_clock_mutex", - out = "cpu_clock_mutex.go", - package = "kernel", - prefix = "cpuClock", -) - declare_mutex( name = "user_counters_mutex", out = "user_counters_mutex.go", @@ -143,17 +136,6 @@ go_template_instance( }, ) -go_template_instance( - name = "seqatomic_taskgoroutineschedinfo", - out = "seqatomic_taskgoroutineschedinfo_unsafe.go", - package = "kernel", - suffix = "TaskGoroutineSchedInfo", - template = "//pkg/sync/seqatomic:generic_seqatomic", - types = { - "Value": "TaskGoroutineSchedInfo", - }, -) - go_template_instance( name = "session_list", out = "session_list.go", @@ -240,7 +222,6 @@ go_library( "cgroup_mounts_mutex.go", "cgroup_mutex.go", "context.go", - "cpu_clock_mutex.go", "fd_table.go", "fd_table_mutex.go", "fd_table_refs.go", @@ -267,7 +248,6 @@ go_library( "running_tasks_mutex.go", "seccheck.go", "seccomp.go", - "seqatomic_taskgoroutineschedinfo_unsafe.go", "session_list.go", "session_refs.go", "sessions.go", diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index d96d5aaf6..b4ba2ca0c 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -21,7 +21,7 @@ // Kernel.extMu // TTY.mu // ThreadGroup.timerMu -// ktime.Timer.mu (for IntervalTimer) and Kernel.cpuClockMu +// Locks acquired by ktime.Timer methods // TaskSet.mu // SignalHandlers.mu // Task.mu @@ -197,30 +197,11 @@ type Kernel struct { // Invariant: runningTasksCond.L == &runningTasksMu. runningTasksCond sync.Cond `state:"nosave"` - // cpuClock is incremented every linux.ClockTick by a goroutine running - // kernel.runCPUClockTicker() while runningTasks != 0. - // - // cpuClock is used to measure task CPU usage, since sampling monotonicClock - // twice on every syscall turns out to be unreasonably expensive. This is - // similar to how Linux does task CPU accounting on x86 - // (CONFIG_IRQ_TIME_ACCOUNTING), although Linux also uses scheduler timing - // information to improve resolution - // (kernel/sched/cputime.c:cputime_adjust()), which we can't do since - // "preeemptive" scheduling is managed by the Go runtime, which doesn't - // provide this information. - // - // cpuClock is mutable, and is accessed using atomic memory operations. - cpuClock atomicbitops.Uint64 - // cpuClockTickTimer drives increments of cpuClock. cpuClockTickTimer *time.Timer `state:"nosave"` - // cpuClockMu is used to make increments of cpuClock, and updates of timers - // based on cpuClock, atomic. - cpuClockMu cpuClockMutex `state:"nosave"` - // cpuClockTickerRunning is true if the goroutine that increments cpuClock is - // running and false if it is blocked in runningTasksCond.Wait() or if it + // running, and false if it is blocked in runningTasksCond.Wait() or if it // never started. // // cpuClockTickerRunning is protected by runningTasksMu. @@ -236,6 +217,13 @@ type Kernel struct { // Invariant: cpuClockTickerStopCond.L == &runningTasksMu. cpuClockTickerStopCond sync.Cond `state:"nosave"` + // cpuClock is a coarse monotonic clock that is advanced by the CPU clock + // ticker and thus approximates wall time when tasks are running (but is + // strictly slower due to CPU clock ticker goroutine wakeup latency). This + // does not use ktime.SyntheticClock since this clock currently does not + // need to support timers. + cpuClock atomicbitops.Int64 + // uniqueID is used to generate unique identifiers. // // uniqueID is mutable, and is accessed using atomic memory operations. @@ -1643,11 +1631,6 @@ func (k *Kernel) MonotonicClock() ktime.SampledClock { return k.timekeeper.monotonicClock } -// CPUClockNow returns the current value of k.cpuClock. -func (k *Kernel) CPUClockNow() uint64 { - return k.cpuClock.Load() -} - // Syslog returns the syslog. func (k *Kernel) Syslog() *syslog { return &k.syslog diff --git a/pkg/sentry/kernel/kernel_state.go b/pkg/sentry/kernel/kernel_state.go index a47d164e9..0bbd1ba9c 100644 --- a/pkg/sentry/kernel/kernel_state.go +++ b/pkg/sentry/kernel/kernel_state.go @@ -36,3 +36,63 @@ func (k *Kernel) loadDanglingEndpoints(_ context.Context, es []tcpip.Endpoint) { tcpip.AddDanglingEndpoint(e) } } + +// saveVforkParent is invoked by stateify. +func (t *Task) saveVforkParent() *Task { + return t.vforkParent.Load() +} + +// loadVforkParent is invoked by stateify. +func (t *Task) loadVforkParent(_ context.Context, vforkParent *Task) { + t.vforkParent.Store(vforkParent) +} + +// savePtraceTracer is invoked by stateify. +func (t *Task) savePtraceTracer() *Task { + return t.ptraceTracer.Load() +} + +// loadPtraceTracer is invoked by stateify. +func (t *Task) loadPtraceTracer(_ context.Context, tracer *Task) { + t.ptraceTracer.Store(tracer) +} + +// saveSeccomp is invoked by stateify. +func (t *Task) saveSeccomp() *taskSeccomp { + return t.seccomp.Load() +} + +// loadSeccomp is invoked by stateify. +func (t *Task) loadSeccomp(_ context.Context, seccompData *taskSeccomp) { + t.seccomp.Store(seccompData) +} + +// saveAppCPUClockLast is invoked by stateify. +func (tg *ThreadGroup) saveAppCPUClockLast() *Task { + return tg.appCPUClockLast.Load() +} + +// loadAppCPUClockLast is invoked by stateify. +func (tg *ThreadGroup) loadAppCPUClockLast(_ context.Context, task *Task) { + tg.appCPUClockLast.Store(task) +} + +// saveAppSysCPUClockLast is invoked by stateify. +func (tg *ThreadGroup) saveAppSysCPUClockLast() *Task { + return tg.appSysCPUClockLast.Load() +} + +// loadAppSysCPUClockLast is invoked by stateify. +func (tg *ThreadGroup) loadAppSysCPUClockLast(_ context.Context, task *Task) { + tg.appSysCPUClockLast.Store(task) +} + +// saveOldRSeqCritical is invoked by stateify. +func (tg *ThreadGroup) saveOldRSeqCritical() *OldRSeqCriticalRegion { + return tg.oldRSeqCritical.Load() +} + +// loadOldRSeqCritical is invoked by stateify. +func (tg *ThreadGroup) loadOldRSeqCritical(_ context.Context, r *OldRSeqCriticalRegion) { + tg.oldRSeqCritical.Store(r) +} diff --git a/pkg/sentry/kernel/task.go b/pkg/sentry/kernel/task.go index 04ecf8588..0c65b1828 100644 --- a/pkg/sentry/kernel/task.go +++ b/pkg/sentry/kernel/task.go @@ -106,12 +106,30 @@ type Task struct { // interruptChan is always notified after restore (see Task.run). interruptChan chan struct{} `state:"nosave"` - // gosched contains the current scheduling state of the task goroutine. + // gostateSeq allows Task.TaskGoroutineStateTime() to read gostate and + // gostateTime atomically. // - // gosched is protected by goschedSeq. gosched is owned by the task - // goroutine. - goschedSeq sync.SeqCount `state:"nosave"` - gosched TaskGoroutineSchedInfo + // gostateSeq is owned by the task goroutine. + gostateSeq sync.SeqCount `state:"nosave"` + + // gostate is the current scheduling state of the task goroutine. + // + // gostate is owned by the task goroutine. + gostate atomicbitops.Uint32 + + // gostateTime was the value of Kernel.cpuClock when gostate was last + // updated or refreshed. + // + // gostateTime is owned by the task goroutine. + gostateTime atomicbitops.Int64 + + // appCPUClock approximates the amount of time the task goroutine has spent + // in TaskGoroutineRunningApp. + appCPUClock ktime.SyntheticClock + + // appSysCPUClock approximates the amount of time the task goroutine has + // spent in TaskGoroutineRunningApp or TaskGoroutineRunningSys. + appSysCPUClock ktime.SyntheticClock // yieldCount is the number of times the task goroutine has called // Task.InterruptibleSleepStart, Task.UninterruptibleSleepStart, or @@ -633,30 +651,6 @@ var ( }) ) -func (t *Task) saveVforkParent() *Task { - return t.vforkParent.Load() -} - -func (t *Task) loadVforkParent(_ gocontext.Context, vforkParent *Task) { - t.vforkParent.Store(vforkParent) -} - -func (t *Task) savePtraceTracer() *Task { - return t.ptraceTracer.Load() -} - -func (t *Task) loadPtraceTracer(_ gocontext.Context, tracer *Task) { - t.ptraceTracer.Store(tracer) -} - -func (t *Task) saveSeccomp() *taskSeccomp { - return t.seccomp.Load() -} - -func (t *Task) loadSeccomp(_ gocontext.Context, seccompData *taskSeccomp) { - t.seccomp.Store(seccompData) -} - // afterLoad is invoked by stateify. func (t *Task) afterLoad(gocontext.Context) { t.updateInfoLocked() @@ -664,7 +658,7 @@ func (t *Task) afterLoad(gocontext.Context) { ts.populateCache(t) } t.interruptChan = make(chan struct{}, 1) - t.gosched.State = TaskGoroutineNonexistent + t.gostate.Store(uint32(TaskGoroutineNonexistent)) if t.stop != nil { t.stopCount = atomicbitops.FromInt32(1) } diff --git a/pkg/sentry/kernel/task_acct.go b/pkg/sentry/kernel/task_acct.go index 01db48358..f37701514 100644 --- a/pkg/sentry/kernel/task_acct.go +++ b/pkg/sentry/kernel/task_acct.go @@ -17,6 +17,10 @@ package kernel // Accounting, limits, timers. import ( + "math" + "sync/atomic" + "time" + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/sentry/ktime" @@ -28,24 +32,18 @@ import ( // // Preconditions: The caller must be running on the task goroutine. func (t *Task) Getitimer(id int32) (linux.ItimerVal, error) { - var tm ktime.Time - var s ktime.Setting + var timer ktime.Timer switch id { case linux.ITIMER_REAL: - tm, s = t.tg.itimerRealTimer.Get() + timer = t.tg.itimerRealTimer case linux.ITIMER_VIRTUAL: - tm = t.tg.UserCPUClock().Now() - t.tg.signalHandlers.mu.Lock() - s, _ = t.tg.itimerVirtSetting.At(tm) - t.tg.signalHandlers.mu.Unlock() + timer = &t.tg.itimerVirtTimer case linux.ITIMER_PROF: - tm = t.tg.CPUClock().Now() - t.tg.signalHandlers.mu.Lock() - s, _ = t.tg.itimerProfSetting.At(tm) - t.tg.signalHandlers.mu.Unlock() + timer = &t.tg.itimerProfTimer default: return linux.ItimerVal{}, linuxerr.EINVAL } + tm, s := timer.Get() val, iv := ktime.SpecFromSetting(tm, s) return linux.ItimerVal{ Value: linux.DurationToTimeval(val), @@ -57,46 +55,30 @@ func (t *Task) Getitimer(id int32) (linux.ItimerVal, error) { // // Preconditions: The caller must be running on the task goroutine. func (t *Task) Setitimer(id int32, newitv linux.ItimerVal) (linux.ItimerVal, error) { - var tm ktime.Time - var olds ktime.Setting + var ( + timer ktime.Timer + last *atomic.Pointer[Task] + ) switch id { case linux.ITIMER_REAL: - news, err := ktime.SettingFromSpec(newitv.Value.ToDuration(), newitv.Interval.ToDuration(), t.tg.itimerRealTimer.Clock()) - if err != nil { - return linux.ItimerVal{}, err - } - tm, olds = t.tg.itimerRealTimer.Set(news, nil) + timer = t.tg.itimerRealTimer case linux.ITIMER_VIRTUAL: - c := t.tg.UserCPUClock() - t.k.cpuClockMu.Lock() - defer t.k.cpuClockMu.Unlock() - tm = c.Now() - news, err := ktime.SettingFromSpecAt(newitv.Value.ToDuration(), newitv.Interval.ToDuration(), tm) - if err != nil { - return linux.ItimerVal{}, err - } - t.tg.signalHandlers.mu.Lock() - olds = t.tg.itimerVirtSetting - t.tg.itimerVirtSetting = news - t.tg.updateCPUTimersEnabledLocked() - t.tg.signalHandlers.mu.Unlock() + timer = &t.tg.itimerVirtTimer + last = &t.tg.appCPUClockLast case linux.ITIMER_PROF: - c := t.tg.CPUClock() - t.k.cpuClockMu.Lock() - defer t.k.cpuClockMu.Unlock() - tm = c.Now() - news, err := ktime.SettingFromSpecAt(newitv.Value.ToDuration(), newitv.Interval.ToDuration(), tm) - if err != nil { - return linux.ItimerVal{}, err - } - t.tg.signalHandlers.mu.Lock() - olds = t.tg.itimerProfSetting - t.tg.itimerProfSetting = news - t.tg.updateCPUTimersEnabledLocked() - t.tg.signalHandlers.mu.Unlock() + timer = &t.tg.itimerProfTimer + last = &t.tg.appSysCPUClockLast default: return linux.ItimerVal{}, linuxerr.EINVAL } + news, err := ktime.SettingFromSpec(newitv.Value.ToDuration(), newitv.Interval.ToDuration(), timer.Clock()) + if err != nil { + return linux.ItimerVal{}, err + } + if last != nil { + last.Store(t) + } + tm, olds := timer.Set(news, nil) oldval, oldiv := ktime.SpecFromSetting(tm, olds) return linux.ItimerVal{ Value: linux.DurationToTimeval(oldval), @@ -104,6 +86,77 @@ func (t *Task) Setitimer(id int32, newitv linux.ItimerVal) (linux.ItimerVal, err }, nil } +// NotifyRlimitCPUUpdated is called by setrlimit. +// +// Preconditions: The caller must be running on the task goroutine. +func (t *Task) NotifyRlimitCPUUpdated() { + // Lock t.tg.timerMu to synchronize updates to these timers between tasks + // in t.tg. + t.tg.timerMu.Lock() + defer t.tg.timerMu.Unlock() + rlimitCPU := t.tg.limits.Get(limits.CPU) + t.tg.appSysCPUClockLast.Store(t) + t.tg.rlimitCPUSoftTimer.Set(ktime.Setting{ + Enabled: rlimitCPU.Cur != limits.Infinity, + Next: ktime.FromSeconds(int64(min(rlimitCPU.Cur, math.MaxInt64))), + Period: time.Second, + }, nil) + t.tg.rlimitCPUHardTimer.Set(ktime.Setting{ + Enabled: rlimitCPU.Max != limits.Infinity, + Next: ktime.FromSeconds(int64(min(rlimitCPU.Max, math.MaxInt64))), + }, nil) +} + +// +stateify savable +type itimerRealListener struct { + tg *ThreadGroup +} + +// NotifyTimer implements ktime.Listener.NotifyTimer. +func (l *itimerRealListener) NotifyTimer(exp uint64) { + l.tg.SendSignal(SignalInfoPriv(linux.SIGALRM)) +} + +// +stateify savable +type itimerVirtListener struct { + tg *ThreadGroup +} + +// NotifyTimer implements ktime.Listener.NotifyTimer. +func (l *itimerVirtListener) NotifyTimer(exp uint64) { + l.tg.appCPUClockLast.Load().SendGroupSignal(SignalInfoPriv(linux.SIGVTALRM)) +} + +// +stateify savable +type itimerProfListener struct { + tg *ThreadGroup +} + +// NotifyTimer implements ktime.Listener.NotifyTimer. +func (l *itimerProfListener) NotifyTimer(exp uint64) { + l.tg.appSysCPUClockLast.Load().SendGroupSignal(SignalInfoPriv(linux.SIGPROF)) +} + +// +stateify savable +type rlimitCPUSoftListener struct { + tg *ThreadGroup +} + +// NotifyTimer implements ktime.Listener.NotifyTimer. +func (l *rlimitCPUSoftListener) NotifyTimer(exp uint64) { + l.tg.appSysCPUClockLast.Load().SendGroupSignal(SignalInfoPriv(linux.SIGXCPU)) +} + +// +stateify savable +type rlimitCPUHardListener struct { + tg *ThreadGroup +} + +// NotifyTimer implements ktime.Listener.NotifyTimer. +func (l *rlimitCPUHardListener) NotifyTimer(exp uint64) { + l.tg.appSysCPUClockLast.Load().SendGroupSignal(SignalInfoPriv(linux.SIGKILL)) +} + // IOUsage returns the io usage of the thread. func (t *Task) IOUsage() *usage.IO { return t.ioUsage diff --git a/pkg/sentry/kernel/task_exit.go b/pkg/sentry/kernel/task_exit.go index 6d0c3ba5f..0e8ac8067 100644 --- a/pkg/sentry/kernel/task_exit.go +++ b/pkg/sentry/kernel/task_exit.go @@ -741,7 +741,6 @@ func (t *Task) exitNotifyLocked(fromPtraceDetach bool) { t.tg.tasks.Remove(t) t.tg.tasksCount-- tc := t.tg.tasksCount - t.tg.exitedCPUStats.Accumulate(t.CPUStats()) t.tg.signalHandlers.mu.Unlock() t.tg.ioUsage.Accumulate(t.ioUsage) if tc == 1 && t != t.tg.leader { @@ -1126,14 +1125,7 @@ func (t *Task) waitCollectZombieLocked(target *Task, opts *WaitOptions, asPtrace if target.parent != nil && target.parent.tg == t.tg && target.exitParentNotified { target.exitParentAcked = true if target == target.tg.leader { - // target.tg.exitedCPUStats doesn't include target.CPUStats() yet, - // and won't until after target.exitNotifyLocked() (maybe). Include - // target.CPUStats() explicitly. This is consistent with Linux, - // which accounts an exited task's cputime to its thread group in - // kernel/exit.c:release_task() => __exit_signal(), and uses - // thread_group_cputime_adjusted() in wait_task_zombie(). - t.tg.childCPUStats.Accumulate(target.CPUStats()) - t.tg.childCPUStats.Accumulate(target.tg.exitedCPUStats) + t.tg.childCPUStats.Accumulate(target.tg.CPUStats()) t.tg.childCPUStats.Accumulate(target.tg.childCPUStats) // Update t's child max resident set size. The size will be the maximum // of this thread's size and all its childrens' sizes. diff --git a/pkg/sentry/kernel/task_run.go b/pkg/sentry/kernel/task_run.go index 0e585ce74..6f1c5adfd 100644 --- a/pkg/sentry/kernel/task_run.go +++ b/pkg/sentry/kernel/task_run.go @@ -391,5 +391,6 @@ func (tg *ThreadGroup) WaitExited() { // Yield yields the processor for the calling task. func (t *Task) Yield() { t.yieldCount.Add(1) + t.tg.yieldCount.Add(1) runtime.Gosched() } diff --git a/pkg/sentry/kernel/task_sched.go b/pkg/sentry/kernel/task_sched.go index d2c3f3a37..8f3de982d 100644 --- a/pkg/sentry/kernel/task_sched.go +++ b/pkg/sentry/kernel/task_sched.go @@ -18,7 +18,7 @@ package kernel import ( "fmt" - "math/rand" + "math/rand/v2" "time" "gvisor.dev/gvisor/pkg/abi/linux" @@ -26,13 +26,12 @@ import ( "gvisor.dev/gvisor/pkg/sentry/hostcpu" "gvisor.dev/gvisor/pkg/sentry/kernel/sched" "gvisor.dev/gvisor/pkg/sentry/ktime" - "gvisor.dev/gvisor/pkg/sentry/limits" "gvisor.dev/gvisor/pkg/sentry/usage" ) // TaskGoroutineState is a coarse representation of the current execution // status of a kernel.Task goroutine. -type TaskGoroutineState int +type TaskGoroutineState uint32 const ( // TaskGoroutineNonexistent indicates that the task goroutine has either @@ -65,67 +64,34 @@ const ( TaskGoroutineStopped ) -// TaskGoroutineSchedInfo contains task goroutine scheduling state which must -// be read and updated atomically. -// -// +stateify savable -type TaskGoroutineSchedInfo struct { - // Timestamp was the value of Kernel.cpuClock when this - // TaskGoroutineSchedInfo was last updated. - Timestamp uint64 - - // State is the current state of the task goroutine. - State TaskGoroutineState - - // UserTicks is the amount of time the task goroutine has spent executing - // its associated Task's application code, in units of linux.ClockTick. - UserTicks uint64 - - // SysTicks is the amount of time the task goroutine has spent executing in - // the sentry, in units of linux.ClockTick. - SysTicks uint64 +// TaskGoroutineState returns the current state of the task goroutine. +func (t *Task) TaskGoroutineState() TaskGoroutineState { + return TaskGoroutineState(t.gostate.Load()) } -// userTicksAt returns the extrapolated value of ts.UserTicks after -// Kernel.CPUClockNow() indicates a time of now. -// -// Preconditions: now <= Kernel.CPUClockNow(). (Since Kernel.cpuClock is -// monotonic, this is satisfied if now is the result of a previous call to -// Kernel.CPUClockNow().) This requirement exists because otherwise a racing -// change to t.gosched can cause userTicksAt to adjust stats by too much, -// making the observed stats non-monotonic. -func (ts *TaskGoroutineSchedInfo) userTicksAt(now uint64) uint64 { - if ts.Timestamp < now && ts.State == TaskGoroutineRunningApp { - // Update stats to reflect execution since the last update. - return ts.UserTicks + (now - ts.Timestamp) +// TaskGoroutineStateTime returns the current state of the task goroutine, and +// the value of Kernel.CPUClockNow() when that state was last updated or +// refreshed. +func (t *Task) TaskGoroutineStateTime() (state TaskGoroutineState, time ktime.Time) { + for { + epoch := t.gostateSeq.BeginRead() + state = t.TaskGoroutineState() + time = ktime.FromNanoseconds(t.gostateTime.Load()) + if t.gostateSeq.ReadOk(epoch) { + return + } } - return ts.UserTicks -} - -// sysTicksAt returns the extrapolated value of ts.SysTicks after -// Kernel.CPUClockNow() indicates a time of now. -// -// Preconditions: As for userTicksAt. -func (ts *TaskGoroutineSchedInfo) sysTicksAt(now uint64) uint64 { - if ts.Timestamp < now && ts.State == TaskGoroutineRunningSys { - return ts.SysTicks + (now - ts.Timestamp) - } - return ts.SysTicks } // Preconditions: The caller must be running on the task goroutine. func (t *Task) accountTaskGoroutineEnter(state TaskGoroutineState) { - now := t.k.CPUClockNow() - if t.gosched.State != TaskGoroutineRunningSys { - panic(fmt.Sprintf("Task goroutine switching from state %v (expected %v) to %v", t.gosched.State, TaskGoroutineRunningSys, state)) + if oldState := t.TaskGoroutineState(); oldState != TaskGoroutineRunningSys { + panic(fmt.Sprintf("Task goroutine switching from state %v (expected %v) to %v", oldState, TaskGoroutineRunningSys, state)) } - t.goschedSeq.BeginWrite() - // This function is very hot; avoid defer. - t.gosched.SysTicks += now - t.gosched.Timestamp - t.gosched.Timestamp = now - t.gosched.State = state - t.goschedSeq.EndWrite() - + t.gostateSeq.BeginWrite() + t.gostate.Store(uint32(state)) + t.touchGostateTime() + t.gostateSeq.EndWrite() if state != TaskGoroutineRunningApp { // Task is blocking/stopping. t.k.decRunningTasks() @@ -133,7 +99,7 @@ func (t *Task) accountTaskGoroutineEnter(state TaskGoroutineState) { } // Preconditions: -// - The caller must be running on the task goroutine +// - The caller must be running on the task goroutine. // - The caller must be leaving a state indicated by a previous call to // t.accountTaskGoroutineEnter(state). func (t *Task) accountTaskGoroutineLeave(state TaskGoroutineState) { @@ -141,50 +107,68 @@ func (t *Task) accountTaskGoroutineLeave(state TaskGoroutineState) { // Task is unblocking/continuing. t.k.incRunningTasks() } - - now := t.k.CPUClockNow() - if t.gosched.State != state { - panic(fmt.Sprintf("Task goroutine switching from state %v (expected %v) to %v", t.gosched.State, state, TaskGoroutineRunningSys)) + if oldState := t.TaskGoroutineState(); oldState != state { + panic(fmt.Sprintf("Task goroutine switching from state %v (expected %v) to %v", oldState, state, TaskGoroutineRunningSys)) } - t.goschedSeq.BeginWrite() - // This function is very hot; avoid defer. - if state == TaskGoroutineRunningApp { - t.gosched.UserTicks += now - t.gosched.Timestamp - } - t.gosched.Timestamp = now - t.gosched.State = TaskGoroutineRunningSys - t.goschedSeq.EndWrite() + t.gostateSeq.BeginWrite() + t.gostate.Store(uint32(TaskGoroutineRunningSys)) + t.touchGostateTime() + t.gostateSeq.EndWrite() } // Preconditions: The caller must be running on the task goroutine. func (t *Task) accountTaskGoroutineRunning() { - now := t.k.CPUClockNow() - if t.gosched.State != TaskGoroutineRunningSys { - panic(fmt.Sprintf("Task goroutine in state %v (expected %v)", t.gosched.State, TaskGoroutineRunningSys)) + if oldState := t.TaskGoroutineState(); oldState != TaskGoroutineRunningSys { + panic(fmt.Sprintf("Task goroutine in state %v (expected %v)", oldState, TaskGoroutineRunningSys)) } - t.goschedSeq.BeginWrite() - t.gosched.SysTicks += now - t.gosched.Timestamp - t.gosched.Timestamp = now - t.goschedSeq.EndWrite() + t.touchGostateTime() } -// TaskGoroutineSchedInfo returns a copy of t's task goroutine scheduling info. -// Most clients should use t.CPUStats() instead. -func (t *Task) TaskGoroutineSchedInfo() TaskGoroutineSchedInfo { - return SeqAtomicLoadTaskGoroutineSchedInfo(&t.goschedSeq, &t.gosched) +// Preconditions: The caller must be running on the task goroutine. +func (t *Task) touchGostateTime() { + t.gostateTime.Store(t.k.cpuClock.Load()) +} + +// CPUClockNow returns the current value of the kernel CPU clock, which +// coarsely approximates wall time but is suspended when no tasks are running. +func (k *Kernel) CPUClockNow() ktime.Time { + return ktime.FromNanoseconds(k.cpuClock.Load()) +} + +// UserCPUClock returns a clock measuring the CPU time the task has spent +// executing application code. +func (t *Task) UserCPUClock() ktime.Clock { + return &t.appCPUClock +} + +// CPUClock returns a clock measuring the CPU time the task has spent executing +// application and "kernel" code. +func (t *Task) CPUClock() ktime.Clock { + return &t.appSysCPUClock +} + +// UserCPUClock returns a ktime.Clock that measures the time that a thread +// group has spent executing. +func (tg *ThreadGroup) UserCPUClock() ktime.Clock { + return &tg.appCPUClock +} + +// CPUClock returns a ktime.Clock that measures the time that a thread group +// has spent executing, including sentry time. +func (tg *ThreadGroup) CPUClock() ktime.Clock { + return &tg.appSysCPUClock } // CPUStats returns the CPU usage statistics of t. func (t *Task) CPUStats() usage.CPUStats { - return t.cpuStatsAt(t.k.CPUClockNow()) -} - -// Preconditions: As for TaskGoroutineSchedInfo.userTicksAt. -func (t *Task) cpuStatsAt(now uint64) usage.CPUStats { - tsched := t.TaskGoroutineSchedInfo() + // The CPU clock ticker advances t.appCPUClock before t.appSysCPUClock, so + // it's possible for the former to transiently exceed the latter. + appNS := t.appCPUClock.Now().Nanoseconds() + appSysNS := t.appSysCPUClock.Now().Nanoseconds() + sysNS := max(appSysNS-appNS, 0) return usage.CPUStats{ - UserTime: time.Duration(tsched.userTicksAt(now) * uint64(linux.ClockTick)), - SysTime: time.Duration(tsched.sysTicksAt(now) * uint64(linux.ClockTick)), + UserTime: time.Duration(appNS), + SysTime: time.Duration(sysNS), VoluntarySwitches: t.yieldCount.Load(), } } @@ -192,26 +176,16 @@ func (t *Task) cpuStatsAt(now uint64) usage.CPUStats { // CPUStats returns the combined CPU usage statistics of all past and present // threads in tg. func (tg *ThreadGroup) CPUStats() usage.CPUStats { - tg.pidns.owner.mu.RLock() - defer tg.pidns.owner.mu.RUnlock() - // Hack to get a pointer to the Kernel. - if tg.leader == nil { - // Per comment on tg.leader, this is only possible if nothing in the - // ThreadGroup has ever executed anyway. - return usage.CPUStats{} + // The CPU clock ticker advances tg.appCPUClock before tg.appSysCPUClock, + // so it's possible for the former to transiently exceed the latter. + appNS := tg.appCPUClock.Now().Nanoseconds() + appSysNS := tg.appSysCPUClock.Now().Nanoseconds() + sysNS := max(appSysNS-appNS, 0) + return usage.CPUStats{ + UserTime: time.Duration(appNS), + SysTime: time.Duration(sysNS), + VoluntarySwitches: tg.yieldCount.Load(), } - return tg.cpuStatsAtLocked(tg.leader.k.CPUClockNow()) -} - -// Preconditions: Same as TaskGoroutineSchedInfo.userTicksAt, plus: -// - Either the TaskSet mutex or the signal mutex must be locked. -func (tg *ThreadGroup) cpuStatsAtLocked(now uint64) usage.CPUStats { - stats := tg.exitedCPUStats - // Account for live tasks. - for t := tg.tasks.Front(); t != nil; t = t.Next() { - stats.Accumulate(t.cpuStatsAt(now)) - } - return stats } // JoinedChildCPUStats implements the semantics of RUSAGE_CHILDREN: "Return @@ -225,132 +199,15 @@ func (tg *ThreadGroup) JoinedChildCPUStats() usage.CPUStats { return tg.childCPUStats } -// taskClock is a ktime.Clock that measures the time that a task has spent -// executing. taskClock is primarily used to implement CLOCK_THREAD_CPUTIME_ID. -// -// +stateify savable -type taskClock struct { - t *Task - - // If includeSys is true, the taskClock includes both time spent executing - // application code as well as time spent in the sentry. Otherwise, the - // taskClock includes only time spent executing application code. - includeSys bool - - // Implements waiter.Waitable. TimeUntil wouldn't change its estimation - // based on either of the clock events, so there's no event to be - // notified for. - ktime.NoClockEvents `state:"nosave"` - - // Implements ktime.Clock.WallTimeUntil. - // - // As an upper bound, a task's clock cannot advance faster than CPU - // time. It would have to execute at a rate of more than 1 task-second - // per 1 CPU-second, which isn't possible. - ktime.WallRateClock `state:"nosave"` -} - -// UserCPUClock returns a clock measuring the CPU time the task has spent -// executing application code. -func (t *Task) UserCPUClock() ktime.Clock { - return &taskClock{t: t, includeSys: false} -} - -// CPUClock returns a clock measuring the CPU time the task has spent executing -// application and "kernel" code. -func (t *Task) CPUClock() ktime.Clock { - return &taskClock{t: t, includeSys: true} -} - -// Now implements ktime.Clock.Now. -func (tc *taskClock) Now() ktime.Time { - stats := tc.t.CPUStats() - if tc.includeSys { - return ktime.FromNanoseconds((stats.UserTime + stats.SysTime).Nanoseconds()) - } - return ktime.FromNanoseconds(stats.UserTime.Nanoseconds()) -} - -// NewTimer implements ktime.Clock.NewTimer. -func (tc *taskClock) NewTimer(l ktime.Listener) ktime.Timer { - return ktime.NewSampledTimer(tc, l) -} - -// tgClock is a ktime.Clock that measures the time a thread group has spent -// executing. tgClock is primarily used to implement CLOCK_PROCESS_CPUTIME_ID. -// -// +stateify savable -type tgClock struct { - tg *ThreadGroup - - // If includeSys is true, the tgClock includes both time spent executing - // application code as well as time spent in the sentry. Otherwise, the - // tgClock includes only time spent executing application code. - includeSys bool - - // Implements waiter.Waitable. - ktime.ClockEventsQueue `state:"nosave"` -} - -// Now implements ktime.Clock.Now. -func (tgc *tgClock) Now() ktime.Time { - stats := tgc.tg.CPUStats() - if tgc.includeSys { - return ktime.FromNanoseconds((stats.UserTime + stats.SysTime).Nanoseconds()) - } - return ktime.FromNanoseconds(stats.UserTime.Nanoseconds()) -} - -// NewTimer implements ktime.Clock.NewTimer. -func (tgc *tgClock) NewTimer(l ktime.Listener) ktime.Timer { - return ktime.NewSampledTimer(tgc, l) -} - -// WallTimeUntil implements ktime.Clock.WallTimeUntil. -func (tgc *tgClock) WallTimeUntil(t, now ktime.Time) time.Duration { - // Thread group CPU time should not exceed wall time * live tasks, since - // task goroutines exit after the transition to TaskExitZombie in - // runExitNotify. - tgc.tg.pidns.owner.mu.RLock() - n := tgc.tg.liveTasks - tgc.tg.pidns.owner.mu.RUnlock() - if n == 0 { - if t.Before(now) { - return 0 - } - // The timer tick raced with thread group exit, after which no more - // tasks can enter the thread group. So tgc.Now() will never advance - // again. Return a large delay; the timer should be stopped long before - // it comes again anyway. - return time.Hour - } - // This is a lower bound on the amount of time that can elapse before an - // associated timer expires, so returning this value tends to result in a - // sequence of closely-spaced ticks just before timer expiry. To avoid - // this, round up to the nearest ClockTick; CPU usage measurements are - // limited to this resolution anyway. - remaining := time.Duration(t.Sub(now).Nanoseconds()/int64(n)) * time.Nanosecond - return ((remaining + (linux.ClockTick - time.Nanosecond)) / linux.ClockTick) * linux.ClockTick -} - -// UserCPUClock returns a ktime.Clock that measures the time that a thread -// group has spent executing. -func (tg *ThreadGroup) UserCPUClock() ktime.Clock { - return &tgClock{tg: tg, includeSys: false} -} - -// CPUClock returns a ktime.Clock that measures the time that a thread group -// has spent executing, including sentry time. -func (tg *ThreadGroup) CPUClock() ktime.Clock { - return &tgClock{tg: tg, includeSys: true} -} - func (k *Kernel) runCPUClockTicker() { - rng := rand.New(rand.NewSource(rand.Int63())) - var tgs []*ThreadGroup + // Storage reused between iterations of the main loop: + var ( + allTasks []*Task + incTasks = make([]*Task, k.applicationCores) + ) for { - // Stop the CPU clock while nothing is running. + // Stop CPU clocks while nothing is running. if k.runningTasks.Load() == 0 { k.runningTasksMu.Lock() if k.runningTasks.Load() == 0 { @@ -373,157 +230,75 @@ func (k *Kernel) runCPUClockTicker() { continue } - // Advance the CPU clock, and timers based on the CPU clock, atomically - // under cpuClockMu. - k.cpuClockMu.Lock() - now := k.cpuClock.Add(1) + // Advance the "kernel CPU clock". + k.cpuClock.Add(linux.ClockTick.Nanoseconds()) - // Check thread group CPU timers. - tgs = k.tasks.Root.ThreadGroupsAppend(tgs) - for _, tg := range tgs { - if tg.cpuTimersEnabled.Load() == 0 { + // Advance CPU clocks. gVisor generally has no knowledge of when sentry + // or application code is actually running on a CPU (due to Go and/or + // host kernel scheduling, with significant variation between + // platforms), so CPU clocks are approximated. We do so by choosing up + // to applicationCores running tasks (randomly, using reservoir + // sampling) and accounting a full CPU clock tick to each of those + // tasks. The alternative would be to distribute CPU time evenly to all + // running tasks, but: + // + // - If the CPU time per task is between 0 and 1 (nanoseconds), then + // neither rounded value is desirable: 0 would cause all CPU clocks to + // cease advancing, while 1 would cause the total CPU time accrued by + // all tasks to exceed the number of claimed CPUs. + // + // - This would require us to mutate CPU clocks and check timers for + // all running tasks and their thread groups, rather than only up to + // applicationCores running tasks (and their thread groups). + allTasks = k.tasks.Root.TasksAppend(allTasks) + runningTasks := 0 + for _, t := range allTasks { + state := t.TaskGoroutineState() + if state != TaskGoroutineRunningApp && state != TaskGoroutineRunningSys { continue } - - sh := tg.signalLock() - - // Accumulate thread group CPU stats, and randomly select running tasks - // using reservoir sampling to receive CPU timer signals. - var virtReceiver *Task - nrVirtCandidates := 0 - var profReceiver *Task - nrProfCandidates := 0 - tgUserTime := tg.exitedCPUStats.UserTime - tgSysTime := tg.exitedCPUStats.SysTime - for t := tg.tasks.Front(); t != nil; t = t.Next() { - tsched := t.TaskGoroutineSchedInfo() - tgUserTime += time.Duration(tsched.userTicksAt(now) * uint64(linux.ClockTick)) - tgSysTime += time.Duration(tsched.sysTicksAt(now) * uint64(linux.ClockTick)) - switch tsched.State { - case TaskGoroutineRunningApp: - // Considered by ITIMER_VIRT, ITIMER_PROF, and RLIMIT_CPU - // timers. - nrVirtCandidates++ - if int(randInt31n(rng, int32(nrVirtCandidates))) == 0 { - virtReceiver = t - } - fallthrough - case TaskGoroutineRunningSys: - // Considered by ITIMER_PROF and RLIMIT_CPU timers. - nrProfCandidates++ - if int(randInt31n(rng, int32(nrProfCandidates))) == 0 { - profReceiver = t - } - } + if runningTasks < len(incTasks) { + incTasks[runningTasks] = t + runningTasks++ + continue } - tgVirtNow := ktime.FromNanoseconds(tgUserTime.Nanoseconds()) - tgProfNow := ktime.FromNanoseconds((tgUserTime + tgSysTime).Nanoseconds()) - - // All of the following are standard (not real-time) signals, which are - // automatically deduplicated, so we ignore the number of expirations. - // It should only be possible for these timers to advance if we found - // at least one running task. - if virtReceiver != nil { - // ITIMER_VIRTUAL - newItimerVirtSetting, exp := tg.itimerVirtSetting.At(tgVirtNow) - tg.itimerVirtSetting = newItimerVirtSetting - if exp != 0 { - virtReceiver.sendSignalLocked(SignalInfoPriv(linux.SIGVTALRM), true) - } + runningTasks++ + if i := rand.IntN(runningTasks); i < len(incTasks) { + incTasks[i] = t } - if profReceiver != nil { - // ITIMER_PROF - newItimerProfSetting, exp := tg.itimerProfSetting.At(tgProfNow) - tg.itimerProfSetting = newItimerProfSetting - if exp != 0 { - profReceiver.sendSignalLocked(SignalInfoPriv(linux.SIGPROF), true) - } - // RLIMIT_CPU soft limit - newRlimitCPUSoftSetting, exp := tg.rlimitCPUSoftSetting.At(tgProfNow) - tg.rlimitCPUSoftSetting = newRlimitCPUSoftSetting - if exp != 0 { - profReceiver.sendSignalLocked(SignalInfoPriv(linux.SIGXCPU), true) - } - // RLIMIT_CPU hard limit - rlimitCPUMax := tg.limits.Get(limits.CPU).Max - if rlimitCPUMax != limits.Infinity && !tgProfNow.Before(ktime.FromSeconds(int64(rlimitCPUMax))) { - profReceiver.sendSignalLocked(SignalInfoPriv(linux.SIGKILL), true) - } + } + numIncTasks := min(runningTasks, len(incTasks)) + // Shuffle incTasks to ensure that if multiple tasks are in the same + // thread group, then all are equally likely to be + // ThreadGroup.app[Sys]CPUClockLast when a ThreadGroup CPU timer fires. + rand.Shuffle(numIncTasks, func(i, j int) { + incTasks[i], incTasks[j] = incTasks[j], incTasks[i] + }) + for _, t := range incTasks[:numIncTasks] { + switch t.TaskGoroutineState() { + case TaskGoroutineRunningApp: + t.appCPUClock.Add(linux.ClockTick) + t.tg.appCPUClockLast.Store(t) + t.tg.appCPUClock.Add(linux.ClockTick) + fallthrough + case TaskGoroutineRunningSys: + t.appSysCPUClock.Add(linux.ClockTick) + t.tg.appSysCPUClockLast.Store(t) + t.tg.appSysCPUClock.Add(linux.ClockTick) } - - sh.mu.Unlock() } - k.cpuClockMu.Unlock() - - // Retain tgs between calls to Notify to reduce allocations. - for i := range tgs { - tgs[i] = nil - } - tgs = tgs[:0] - } -} - -// randInt31n returns a random integer in [0, n). -// -// randInt31n is equivalent to math/rand.Rand.int31n(), which is unexported. -// See that function for details. -func randInt31n(rng *rand.Rand, n int32) int32 { - v := rng.Uint32() - prod := uint64(v) * uint64(n) - low := uint32(prod) - if low < uint32(n) { - thresh := uint32(-n) % uint32(n) - for low < thresh { - v = rng.Uint32() - prod = uint64(v) * uint64(n) - low = uint32(prod) - } - } - return int32(prod >> 32) -} - -// NotifyRlimitCPUUpdated is called by setrlimit. -// -// Preconditions: The caller must be running on the task goroutine. -func (t *Task) NotifyRlimitCPUUpdated() { - t.k.cpuClockMu.Lock() - defer t.k.cpuClockMu.Unlock() - t.tg.pidns.owner.mu.RLock() - defer t.tg.pidns.owner.mu.RUnlock() - t.tg.signalHandlers.mu.Lock() - defer t.tg.signalHandlers.mu.Unlock() - rlimitCPU := t.tg.limits.Get(limits.CPU) - t.tg.rlimitCPUSoftSetting = ktime.Setting{ - Enabled: rlimitCPU.Cur != limits.Infinity, - Next: ktime.FromNanoseconds((time.Duration(rlimitCPU.Cur) * time.Second).Nanoseconds()), - Period: time.Second, - } - if rlimitCPU.Max != limits.Infinity { - // Check if tg is already over the hard limit. - tgcpu := t.tg.cpuStatsAtLocked(t.k.CPUClockNow()) - tgProfNow := ktime.FromNanoseconds((tgcpu.UserTime + tgcpu.SysTime).Nanoseconds()) - if !tgProfNow.Before(ktime.FromSeconds(int64(rlimitCPU.Max))) { - t.sendSignalLocked(SignalInfoPriv(linux.SIGKILL), true) - } - } - t.tg.updateCPUTimersEnabledLocked() -} - -// Preconditions: The signal mutex must be locked. -func (tg *ThreadGroup) updateCPUTimersEnabledLocked() { - rlimitCPU := tg.limits.Get(limits.CPU) - if tg.itimerVirtSetting.Enabled || tg.itimerProfSetting.Enabled || tg.rlimitCPUSoftSetting.Enabled || rlimitCPU.Max != limits.Infinity { - tg.cpuTimersEnabled.Store(1) - } else { - tg.cpuTimersEnabled.Store(0) + // Reset storage for the next iteration. + clear(allTasks) + allTasks = allTasks[:0] + clear(incTasks[:numIncTasks]) } } // StateStatus returns a string representation of the task's current state, // appropriate for /proc/[pid]/status. func (t *Task) StateStatus() string { - switch s := t.TaskGoroutineSchedInfo().State; s { + switch s := t.TaskGoroutineState(); s { case TaskGoroutineNonexistent, TaskGoroutineRunningSys: switch t.ExitState() { case TaskExitZombie: diff --git a/pkg/sentry/kernel/thread_group.go b/pkg/sentry/kernel/thread_group.go index 0f6f72d69..26566de04 100644 --- a/pkg/sentry/kernel/thread_group.go +++ b/pkg/sentry/kernel/thread_group.go @@ -15,7 +15,6 @@ package kernel import ( - goContext "context" "sync/atomic" "gvisor.dev/gvisor/pkg/abi/linux" @@ -170,31 +169,19 @@ type ThreadGroup struct { timerMu threadGroupTimerMutex `state:"nosave"` - // itimerRealTimer implements ITIMER_REAL for the thread group. - itimerRealTimer *ktime.SampledTimer + // ITIMER_* timers: + itimerRealTimer *ktime.SampledTimer + itimerRealListener itimerRealListener + itimerVirtTimer ktime.SyntheticTimer + itimerVirtListener itimerVirtListener + itimerProfTimer ktime.SyntheticTimer + itimerProfListener itimerProfListener - // itimerVirtSetting is the ITIMER_VIRTUAL setting for the thread group. - // - // itimerVirtSetting is protected by the signal mutex. - itimerVirtSetting ktime.Setting - - // itimerProfSetting is the ITIMER_PROF setting for the thread group. - // - // itimerProfSetting is protected by the signal mutex. - itimerProfSetting ktime.Setting - - // rlimitCPUSoftSetting is the setting for RLIMIT_CPU soft limit - // notifications for the thread group. - // - // rlimitCPUSoftSetting is protected by the signal mutex. - rlimitCPUSoftSetting ktime.Setting - - // cpuTimersEnabled is non-zero if itimerVirtSetting.Enabled is true, - // itimerProfSetting.Enabled is true, rlimitCPUSoftSetting.Enabled is true, - // or limits.Get(CPU) is finite. - // - // cpuTimersEnabled is protected by the signal mutex. - cpuTimersEnabled atomicbitops.Uint32 + // RLIMIT_CPU timers: + rlimitCPUSoftTimer ktime.SyntheticTimer + rlimitCPUSoftListener rlimitCPUSoftListener + rlimitCPUHardTimer ktime.SyntheticTimer + rlimitCPUHardListener rlimitCPUHardListener // timers is the thread group's POSIX interval timers. nextTimerID is the // TimerID at which allocation should begin searching for an unused ID. @@ -203,13 +190,25 @@ type ThreadGroup struct { timers map[linux.TimerID]*IntervalTimer nextTimerID linux.TimerID - // exitedCPUStats is the CPU usage for all exited tasks in the thread - // group. exitedCPUStats is protected by both the TaskSet mutex and the - // signal mutex. Mutating it requires that the TaskSet mutex is locked for - // writing *and* that the signal mutex is locked. Reading it requires - // locking the TaskSet mutex (for reading or writing) *or* locking the - // signal mutex. - exitedCPUStats usage.CPUStats + // appCPUClockLast is the last task to have incremented appCPUClock or set + // a timer dependent on appCPUClock. + appCPUClockLast atomic.Pointer[Task] `state:".(*Task)"` + + // appCPUClock is the sum of Task.appCPUClock for all past and present + // tasks in the thread group. + appCPUClock ktime.SyntheticClock + + // appSysCPUClockLast is the last task to have incremented appSysCPUClock + // or set a timer dependent on appSysCPUClock. + appSysCPUClockLast atomic.Pointer[Task] `state:".(*Task)"` + + // appSysCPUClock is the sum of Task.appSysCPUClock for all past and + // present tasks in the thread group. + appSysCPUClock ktime.SyntheticClock + + // yieldCount is the sum of Task.yieldCount for all past and present tasks + // in the thread group. + yieldCount atomicbitops.Uint64 // childCPUStats is the CPU usage of all joined descendants of this thread // group. childCPUStats is protected by the TaskSet mutex. @@ -292,25 +291,24 @@ func (k *Kernel) NewThreadGroup(pidns *PIDNamespace, sh *SignalHandlers, termina }, signalHandlers: sh, terminationSignal: terminationSignal, + timers: make(map[linux.TimerID]*IntervalTimer), ioUsage: &usage.IO{}, limits: limits, } - tg.itimerRealTimer = ktime.NewSampledTimer(k.timekeeper.monotonicClock, &itimerRealListener{tg: tg}) - tg.timers = make(map[linux.TimerID]*IntervalTimer) + tg.itimerRealTimer = ktime.NewSampledTimer(k.timekeeper.monotonicClock, &tg.itimerRealListener) + tg.itimerRealListener.tg = tg + tg.itimerVirtTimer.Init(&tg.appCPUClock, &tg.itimerVirtListener) + tg.itimerVirtListener.tg = tg + tg.itimerProfTimer.Init(&tg.appSysCPUClock, &tg.itimerProfListener) + tg.itimerProfListener.tg = tg + tg.rlimitCPUSoftTimer.Init(&tg.appSysCPUClock, &tg.rlimitCPUSoftListener) + tg.rlimitCPUSoftListener.tg = tg + tg.rlimitCPUHardTimer.Init(&tg.appSysCPUClock, &tg.rlimitCPUHardListener) + tg.rlimitCPUHardListener.tg = tg tg.oldRSeqCritical.Store(&OldRSeqCriticalRegion{}) return tg } -// saveOldRSeqCritical is invoked by stateify. -func (tg *ThreadGroup) saveOldRSeqCritical() *OldRSeqCriticalRegion { - return tg.oldRSeqCritical.Load() -} - -// loadOldRSeqCritical is invoked by stateify. -func (tg *ThreadGroup) loadOldRSeqCritical(_ goContext.Context, r *OldRSeqCriticalRegion) { - tg.oldRSeqCritical.Store(r) -} - // signalLock atomically locks tg.SignalHandlers().mu and returns the // SignalHandlers. func (tg *ThreadGroup) signalLock() *SignalHandlers { @@ -336,6 +334,10 @@ func (tg *ThreadGroup) Release(ctx context.Context) { // Timers must be destroyed without holding the TaskSet or signal mutexes // since timers send signals with Timer.mu locked. tg.itimerRealTimer.Destroy() + tg.itimerVirtTimer.Destroy() + tg.itimerProfTimer.Destroy() + tg.rlimitCPUSoftTimer.Destroy() + tg.rlimitCPUHardTimer.Destroy() var its []*IntervalTimer tg.signalHandlers.mu.Lock() for _, it := range tg.timers { @@ -649,15 +651,3 @@ func (tg *ThreadGroup) IsInitIn(pidns *PIDNamespace) bool { func (tg *ThreadGroup) isInitInLocked(pidns *PIDNamespace) bool { return pidns.tgids[tg] == initTID } - -// itimerRealListener implements ktime.Listener for ITIMER_REAL expirations. -// -// +stateify savable -type itimerRealListener struct { - tg *ThreadGroup -} - -// NotifyTimer implements ktime.TimerListener.NotifyTimer. -func (l *itimerRealListener) NotifyTimer(exp uint64) { - l.tg.SendSignal(SignalInfoPriv(linux.SIGALRM)) -} diff --git a/pkg/sentry/kernel/threads.go b/pkg/sentry/kernel/threads.go index 6c7d7f782..3e34e0eaa 100644 --- a/pkg/sentry/kernel/threads.go +++ b/pkg/sentry/kernel/threads.go @@ -299,13 +299,17 @@ func (ns *PIDNamespace) IDOfThreadGroup(tg *ThreadGroup) ThreadID { // Tasks returns a snapshot of the tasks in ns. func (ns *PIDNamespace) Tasks() []*Task { + return ns.TasksAppend(nil) +} + +// TasksAppend appends a snapshot of the tasks in ns to ts. +func (ns *PIDNamespace) TasksAppend(ts []*Task) []*Task { ns.owner.mu.RLock() defer ns.owner.mu.RUnlock() - tasks := make([]*Task, 0, len(ns.tasks)) for t := range ns.tids { - tasks = append(tasks, t) + ts = append(ts, t) } - return tasks + return ts } // NumTasks returns the number of tasks in ns. @@ -487,7 +491,7 @@ func (tg *ThreadGroup) ID() ThreadID { type taskNode struct { // tg is the thread group that this task belongs to. The tg pointer is // immutable. - tg *ThreadGroup `state:"wait"` + tg *ThreadGroup // taskEntry links into tg.tasks. Note that this means that // Task.Next/Prev/SetNext/SetPrev refer to sibling tasks in the same thread diff --git a/pkg/sentry/ktime/BUILD b/pkg/sentry/ktime/BUILD index 1a8b33913..d16dbe02a 100644 --- a/pkg/sentry/ktime/BUILD +++ b/pkg/sentry/ktime/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_library") +load("//tools:defs.bzl", "go_library", "go_test") load("//tools/go_generics:defs.bzl", "go_template_instance") package( @@ -17,6 +17,43 @@ go_template_instance( }, ) +go_template_instance( + name = "synthetic_timer_list", + out = "synthetic_timer_list.go", + package = "ktime", + prefix = "syntheticTimer", + template = "//pkg/ilist:generic_list", + types = { + "Element": "*SyntheticTimer", + "Linker": "*SyntheticTimer", + }, +) + +go_template_instance( + name = "synthetic_timer_set", + out = "synthetic_timer_set.go", + package = "ktime", + prefix = "syntheticTimer", + template = "//pkg/segment:generic_set", + types = { + "Key": "uint64", + "Range": "uint64Range", + "Value": "syntheticTimerQueue", + "Functions": "syntheticTimerSetFunctions", + }, +) + +go_template_instance( + name = "uint64_range", + out = "uint64_range.go", + package = "ktime", + prefix = "uint64", + template = "//pkg/segment:generic_range", + types = { + "T": "uint64", + }, +) + go_library( name = "ktime", srcs = [ @@ -24,11 +61,16 @@ go_library( "ktime.go", "sampled_timer.go", "seqatomic_sampled_clock_unsafe.go", + "synthetic_timer.go", + "synthetic_timer_list.go", + "synthetic_timer_set.go", + "uint64_range.go", "util.go", ], visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/abi/linux", + "//pkg/atomicbitops", "//pkg/context", "//pkg/errors/linuxerr", "//pkg/gohacks", @@ -36,3 +78,12 @@ go_library( "//pkg/waiter", ], ) + +go_test( + name = "ktime_test", + size = "small", + srcs = [ + "synthetic_timer_test.go", + ], + library = ":ktime", +) diff --git a/pkg/sentry/ktime/synthetic_timer.go b/pkg/sentry/ktime/synthetic_timer.go new file mode 100644 index 000000000..6fddb97a2 --- /dev/null +++ b/pkg/sentry/ktime/synthetic_timer.go @@ -0,0 +1,260 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ktime + +import ( + "fmt" + "math" + "time" + + "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/pkg/sync" +) + +// SyntheticTimer implements Timer for SyntheticClocks. +// +// +stateify savable +type SyntheticTimer struct { + // immutable + clock *SyntheticClock + listener Listener + + // setting is the timer's current setting. setting is protected by + // clock.mu. + setting Setting + + // syntheticTimerEntry links the SyntheticTimer into + // syntheticTimerQueue.timers when setting.Enabled == true. + // syntheticTimerEntry is protected by mu. + syntheticTimerEntry +} + +// SyntheticClock is a Clock whose current time is set manually by calling +// Store or Add. +// +// +stateify savable +type SyntheticClock struct { + mu sync.Mutex `state:"nosave"` + + // now is the Clock's current time. Writes to now require that mu is + // locked. + now atomicbitops.Int64 + + // timers maps each timer expiration time to a list of all enabled timers + // with that expiration time. timers is protected by mu. + timers syntheticTimerSet +} + +// syntheticTimerQueue is the value type of SyntheticClock.timers. +// +// +stateify savable +type syntheticTimerQueue struct { + timers syntheticTimerList +} + +// NewSyntheticTimer returns an initialized heap-allocated SyntheticTimer. +func NewSyntheticTimer(clock *SyntheticClock, listener Listener) *SyntheticTimer { + t := &SyntheticTimer{} + t.Init(clock, listener) + return t +} + +// Init makes a zero-value SyntheticTimer ready for use. +func (t *SyntheticTimer) Init(clock *SyntheticClock, listener Listener) { + t.clock = clock + t.listener = listener +} + +// Destroy implements Timer.Destroy. +func (t *SyntheticTimer) Destroy() { + // Just stop the timer. + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + if t.setting.Enabled { + t.setting.Enabled = false + t.clock.delTimerLocked(t) + } +} + +// Pause implements Timer.Pause. Since SyntheticTimer expirations are caused by +// changes to the corresponding SyntheticClock's time, Pause is a no-op; +// clients must ensure that the SyntheticClock's time cannot advance while the +// timer is paused. +func (t *SyntheticTimer) Pause() { +} + +// Resume implements Timer.Resume. Since Pause is a no-op, Resume is also a +// no-op. +func (t *SyntheticTimer) Resume() { +} + +// Clock implements Timer.Clock. +func (t *SyntheticTimer) Clock() Clock { + return t.clock +} + +// Get implements Timer.Get. +func (t *SyntheticTimer) Get() (Time, Setting) { + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + // SyntheticTimers are expired synchronously with SyntheticClock time + // changes, so t.setting is always up to date. + return t.clock.nowLocked(), t.setting +} + +// Set implements Timer.Set. +func (t *SyntheticTimer) Set(s Setting, f func()) (Time, Setting) { + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + // SyntheticTimers are expired synchronously with SyntheticClock time + // changes, so t.setting is always up to date. + now := t.clock.nowLocked() + oldS := t.setting + newS, newExp := s.At(now) + if f != nil { + f() + } + if oldS != newS { + if oldS.Enabled { + t.clock.delTimerLocked(t) + } + t.setting = newS + if newS.Enabled { + t.clock.addTimerLocked(t) + } + } + if newExp > 0 { + t.listener.NotifyTimer(newExp) + } + return now, oldS +} + +// Now implements Clock.Now. +func (c *SyntheticClock) Now() Time { + return FromNanoseconds(c.now.Load()) +} + +// Preconditions: c.mu must be locked. +func (c *SyntheticClock) nowLocked() Time { + return FromNanoseconds(c.now.RacyLoad()) +} + +// NewTimer implements Clock.NewTimer. +func (c *SyntheticClock) NewTimer(listener Listener) Timer { + return NewSyntheticTimer(c, listener) +} + +// Store sets c's current time to now and notifies expired timers. +// +// Preconditions: +// - now.Nanoseconds() >= 0. +// - The caller must not hold locks following Timer methods in the lock order +// (since Listener notification requires acquiring such locks). +func (c *SyntheticClock) Store(now Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.setTimeLocked(now.Nanoseconds()) +} + +// Add increases c's current time by d and notifies expired timers. +// +// Preconditions: +// - c's resulting current time >= 0. +// - The caller must not hold locks following Timer methods in the lock order +// (since Listener notification requires acquiring such locks). +func (c *SyntheticClock) Add(delta time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.setTimeLocked(c.now.RacyLoad() + delta.Nanoseconds()) +} + +// Preconditions: c.mu must be locked. +func (c *SyntheticClock) setTimeLocked(nowNS int64) { + if nowNS < 0 { + panic(fmt.Sprintf("invalid time %d", nowNS)) + } + c.now.Store(nowNS) + now := FromNanoseconds(nowNS) + // Expire timers. + for { + seg := c.timers.FirstSegment() + if !seg.Ok() || uint64(nowNS) < seg.Start() { + break + } + // Make a copy of the timers list, then remove seg and iterate the + // copy, since insertion of new segments (for periodic timers) will + // invalidate seg. + timers := seg.ValuePtr().timers + c.timers.Remove(seg) + for !timers.Empty() { + t := timers.Front() + timers.Remove(t) + s, exp := t.setting.At(now) + if exp == 0 { + panic(fmt.Sprintf("ktime.SyntheticClock (time=%d) contains enqueued timer %p for time=%d with unexpired setting %+v", nowNS, t, seg.Start(), t.setting)) + } + t.setting = s + t.listener.NotifyTimer(exp) + if t.setting.Enabled { + c.addTimerLocked(t) + } + } + } +} + +// Preconditions: c.mu must be locked. +func (c *SyntheticClock) addTimerLocked(t *SyntheticTimer) { + nextNS := uint64(t.setting.Next.Nanoseconds()) + seg, gap := c.timers.Find(nextNS) + if gap.Ok() { + seg = c.timers.Insert(gap, uint64Range{nextNS, nextNS + 1}, syntheticTimerQueue{}) + } + seg.ValuePtr().timers.PushBack(t) +} + +// Preconditions: c.mu must be locked. +func (c *SyntheticClock) delTimerLocked(t *SyntheticTimer) { + nextNS := uint64(t.setting.Next.Nanoseconds()) + seg := c.timers.FindSegment(nextNS) + if !seg.Ok() { + panic(fmt.Sprintf("ktime.SyntheticClock (time=%d) does not contain enqueued timer %p for time=%d with setting %+v", c.now.RacyLoad(), t, nextNS, t.setting)) + } + q := seg.ValuePtr() + q.timers.Remove(t) + if q.timers.Empty() { + c.timers.Remove(seg) + } +} + +type syntheticTimerSetFunctions struct{} + +func (syntheticTimerSetFunctions) MinKey() uint64 { + return 0 +} + +func (syntheticTimerSetFunctions) MaxKey() uint64 { + return math.MaxUint64 +} + +func (syntheticTimerSetFunctions) ClearValue(*syntheticTimerQueue) { +} + +func (syntheticTimerSetFunctions) Merge(_ uint64Range, _ syntheticTimerQueue, _ uint64Range, _ syntheticTimerQueue) (syntheticTimerQueue, bool) { + return syntheticTimerQueue{}, false +} + +func (syntheticTimerSetFunctions) Split(_ uint64Range, _ syntheticTimerQueue, _ uint64) (syntheticTimerQueue, syntheticTimerQueue) { + panic("syntheticTimerSetFunctions.Split should never be called") +} diff --git a/pkg/sentry/ktime/synthetic_timer_test.go b/pkg/sentry/ktime/synthetic_timer_test.go new file mode 100644 index 000000000..eaea12c8d --- /dev/null +++ b/pkg/sentry/ktime/synthetic_timer_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ktime + +import ( + "slices" + "testing" + "time" +) + +func TestSyntheticClockNow(t *testing.T) { + var c SyntheticClock + if got := c.Now(); got.Nanoseconds() != 0 { + t.Errorf("zero-value SyntheticClock: Now() = %v, want 0", got) + } + want := FromSeconds(1) + c.Store(want) + if got := c.Now(); got != want { + t.Errorf("after Store: Now() = %v, want %v", got, want) + } + c.Add(10 * time.Second) + if got, want := c.Now(), FromSeconds(11); got != want { + t.Errorf("after positive Add: Now() = %v, want %v", got, want) + } + c.Add(-5 * time.Second) + if got, want := c.Now(), FromSeconds(6); got != want { + t.Errorf("after negative Add: Now() = %v, want %v", got, want) + } +} + +type testRecorder struct { + vals []int +} + +type testRecorderListener struct { + r *testRecorder + val int +} + +func (l *testRecorderListener) NotifyTimer(exp uint64) { + l.r.vals = append(l.r.vals, l.val) +} + +func newTestRecorderTimer(c Clock, r *testRecorder, val int, next Time, period time.Duration) Timer { + t := c.NewTimer(&testRecorderListener{ + r: r, + val: val, + }) + t.Set(Setting{ + Enabled: true, + Next: next, + Period: period, + }, nil) + return t +} + +func checkRecordAt(t *testing.T, c *SyntheticClock, r *testRecorder, now Time, want []int) { + c.Store(now) + if !slices.Equal(r.vals, want) { + t.Errorf("at time %v: got %v, want %v", now, r.vals, want) + } +} + +func TestSyntheticTimer(t *testing.T) { + var ( + c SyntheticClock + r testRecorder + ) + + // Set up timers. + // + // t0 and t1 are "far apart". + // t1 and t2 are as close as possible. t1 is also periodic. + // t3 and t4 expire at the same time. + // t5 occurs between the second and third occurrences of t1. + newTestRecorderTimer(&c, &r, 0, FromSeconds(2), 0) + newTestRecorderTimer(&c, &r, 1, FromSeconds(4), 4*time.Second) + newTestRecorderTimer(&c, &r, 2, FromSeconds(4).Add(time.Nanosecond), 0) + newTestRecorderTimer(&c, &r, 3, FromSeconds(6), 0) + newTestRecorderTimer(&c, &r, 4, FromSeconds(6), 0) + newTestRecorderTimer(&c, &r, 5, FromSeconds(10), 0) + + // The order in which timers expire isn't specified, but is FIFO in the + // current implementation. + checkRecordAt(t, &c, &r, FromSeconds(1), []int{}) + checkRecordAt(t, &c, &r, FromSeconds(2), []int{0}) + checkRecordAt(t, &c, &r, FromSeconds(3), []int{0}) + checkRecordAt(t, &c, &r, FromSeconds(4).Add(-time.Nanosecond), []int{0}) + checkRecordAt(t, &c, &r, FromSeconds(4), []int{0, 1}) + checkRecordAt(t, &c, &r, FromSeconds(4).Add(time.Nanosecond), []int{0, 1, 2}) + checkRecordAt(t, &c, &r, FromSeconds(5), []int{0, 1, 2}) + checkRecordAt(t, &c, &r, FromSeconds(6), []int{0, 1, 2, 3, 4}) + checkRecordAt(t, &c, &r, FromSeconds(7), []int{0, 1, 2, 3, 4}) + checkRecordAt(t, &c, &r, FromSeconds(8).Add(-time.Nanosecond), []int{0, 1, 2, 3, 4}) + checkRecordAt(t, &c, &r, FromSeconds(8), []int{0, 1, 2, 3, 4, 1}) + checkRecordAt(t, &c, &r, FromSeconds(8), []int{0, 1, 2, 3, 4, 1}) + checkRecordAt(t, &c, &r, FromSeconds(9), []int{0, 1, 2, 3, 4, 1}) + checkRecordAt(t, &c, &r, FromSeconds(10), []int{0, 1, 2, 3, 4, 1, 5}) + checkRecordAt(t, &c, &r, FromSeconds(11), []int{0, 1, 2, 3, 4, 1, 5}) + checkRecordAt(t, &c, &r, FromSeconds(12), []int{0, 1, 2, 3, 4, 1, 5, 1}) + checkRecordAt(t, &c, &r, FromSeconds(13), []int{0, 1, 2, 3, 4, 1, 5, 1}) + checkRecordAt(t, &c, &r, FromSeconds(14), []int{0, 1, 2, 3, 4, 1, 5, 1}) + checkRecordAt(t, &c, &r, FromSeconds(15), []int{0, 1, 2, 3, 4, 1, 5, 1}) +} diff --git a/pkg/sentry/watchdog/BUILD b/pkg/sentry/watchdog/BUILD index 4efc554f9..0ef863b87 100644 --- a/pkg/sentry/watchdog/BUILD +++ b/pkg/sentry/watchdog/BUILD @@ -10,7 +10,6 @@ go_library( srcs = ["watchdog.go"], visibility = ["//:sandbox"], deps = [ - "//pkg/abi/linux", "//pkg/log", "//pkg/metric", "//pkg/sentry/kernel", diff --git a/pkg/sentry/watchdog/watchdog.go b/pkg/sentry/watchdog/watchdog.go index 4dc018a64..ffcef09df 100644 --- a/pkg/sentry/watchdog/watchdog.go +++ b/pkg/sentry/watchdog/watchdog.go @@ -33,7 +33,6 @@ import ( "fmt" "time" - "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/metric" "gvisor.dev/gvisor/pkg/sentry/kernel" @@ -280,7 +279,7 @@ func (w *Watchdog) runTurn() { newOffenders := make(map[*kernel.Task]*offender) newTaskFound := false - now := ktime.FromNanoseconds(int64(w.k.CPUClockNow() * uint64(linux.ClockTick))) + now := w.k.CPUClockNow() // The process may be running with low CPU limit making tasks appear stuck because // are starved of CPU cycles. An estimate is that Tasks could have been starved @@ -294,11 +293,9 @@ func (w *Watchdog) runTurn() { log.Infof("Watchdog starting loop, tasks: %d, discount: %v", len(tasks), discount) for _, t := range tasks { - tsched := t.TaskGoroutineSchedInfo() - // An offender is a task running inside the kernel for longer than the specified timeout. - if tsched.State == kernel.TaskGoroutineRunningSys { - lastUpdateTime := ktime.FromNanoseconds(int64(tsched.Timestamp * uint64(linux.ClockTick))) + tstate, lastUpdateTime := t.TaskGoroutineStateTime() + if tstate == kernel.TaskGoroutineRunningSys { elapsed := now.Sub(lastUpdateTime) - discount if elapsed > w.TaskTimeout { tc, ok := w.offenders[t]