From 16a6afe48fa06a173accc2caac15f3f1a5a3ea3c Mon Sep 17 00:00:00 2001
From: Jamie Liu <jamieliu@google.com>
Date: Thu, 24 Feb 2022 14:27:39 -0800
Subject: [PATCH] Yield P after notifying event waiters in fdnotifier.

The fdnotifier goroutine alternates between [host] epoll_wait()ing for I/O
readiness events and propagating those events through the sentry, possibly
waking task goroutines blocked in e.g. [application] epoll_wait(). Since woken
goroutines are queued on the runqueue of the waking P (cf. runtime.ready()),
these goroutines will almost always wait to execute until another OS thread
takes them (either directly via runtime.runqsteal(), or indirectly via sysmon
stealing the P and runtime.handoffp()ing it to another thread). Furthermore,
epoll_wait() is most likely to block (new events are least likely to have
arrived) shortly after a previous call (on the same epoll FD) returns events,
so this behavior utilizes the thread's OS-scheduled time slice poorly. Avoid
both of these problems by runtime.goyield()ing between goroutine wakeup and
epoll_wait().

PiperOrigin-RevId: 430793631
---
 pkg/fdnotifier/fdnotifier.go | 7 +++++++
 pkg/sync/runtime_unsafe.go   | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/pkg/fdnotifier/fdnotifier.go b/pkg/fdnotifier/fdnotifier.go
index 152557143..0858e0866 100644
--- a/pkg/fdnotifier/fdnotifier.go
+++ b/pkg/fdnotifier/fdnotifier.go
@@ -155,13 +155,20 @@ func (n *notifier) waitAndNotify() error {
 			return err
 		}
 
+		notified := false
 		n.mu.Lock()
 		for i := 0; i < v; i++ {
 			if fi, ok := n.fdMap[e[i].Fd]; ok {
 				fi.queue.Notify(waiter.EventMaskFromLinux(e[i].Events))
+				notified = true
 			}
 		}
 		n.mu.Unlock()
+		if notified {
+			// Let goroutines woken by Notify get a chance to run before we
+			// epoll_wait again.
+			sync.Goyield()
+		}
 	}
 }
 
diff --git a/pkg/sync/runtime_unsafe.go b/pkg/sync/runtime_unsafe.go
index cd17f7368..cd7b9025b 100644
--- a/pkg/sync/runtime_unsafe.go
+++ b/pkg/sync/runtime_unsafe.go
@@ -22,6 +22,15 @@ import (
 	"unsafe"
 )
 
+// Goyield is runtime.goyield, which is similar to runtime.Gosched but only
+// yields the processor to other goroutines already on the processor's
+// runqueue.
+//
+//go:nosplit
+func Goyield() {
+	goyield()
+}
+
 // Gopark is runtime.gopark. Gopark calls unlockf(pointer to runtime.g, lock);
 // if unlockf returns true, Gopark blocks until Goready(pointer to runtime.g)
 // is called. unlockf and its callees must be nosplit and norace, since stack