From f29dc0a7870ca0cfabfdaee8a6842558962acbc2 Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Wed, 16 Aug 2023 02:44:45 +0000 Subject: [PATCH] network: speed up createSocket() by giving protocal 0 in socket() In practice, we found that createSocket() could cost up to 40ms, which is an unacceptable overhead in FaaS(Function as a Service) scenario. The root cause is the bind() syscall runs into the slow path in kernel and invokes synchronize_net(), which is an expensive call. From kernel commit[1] we learned that we could bypass this slow path by adjusting the socket() syscall before we call bind(). By giving the empty protocal number, the packet_create() will not call __register_prot_hook(), in which the kernel turn on the flag packet_sock.running. So when it comes to packet_do_bind(), it won't call synchronize_net() for there is no prot_hook to unregister. Worst cost for bind() tested on Intel Xeon E5-2682 with Linux kernel 5.4 over 100 times: * before: 39633.274 us * after: 25.098 us Reference: [1] https://github.com/torvalds/linux/commit/902fefb82ef72a50c78cb4a20cc954b037a98d1c Signed-off-by: Tianyu Zhou --- runsc/sandbox/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index 5a9773f19..e17835f55 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -360,7 +360,7 @@ type socketEntry struct { func createSocket(iface net.Interface, ifaceLink netlink.Link, enableGSO bool) (*socketEntry, error) { // Create the socket. const protocol = 0x0300 // htons(ETH_P_ALL) - fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, protocol) + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, 0) // pass protocol 0 to avoid slow bind() if err != nil { return nil, fmt.Errorf("unable to create raw socket: %v", err) }