Add //pkg/sync:generic_atomicptrmap.

AtomicPtrMap is a generic concurrent map from arbitrary keys to arbitrary
pointer values.

Benchmarks:
name                                                            time/op
StoreDelete/RWMutexMap-12                                        335ns ± 1%
StoreDelete/SyncMap-12                                           705ns ± 3%
StoreDelete/AtomicPtrMap-12                                      287ns ± 4%
StoreDelete/AtomicPtrMapSharded-12                               289ns ± 1%
LoadOrStoreDelete/RWMutexMap-12                                  342ns ± 2%
LoadOrStoreDelete/SyncMap-12                                     662ns ± 2%
LoadOrStoreDelete/AtomicPtrMap-12                                290ns ± 7%
LoadOrStoreDelete/AtomicPtrMapSharded-12                         293ns ± 2%
LookupPositive/RWMutexMap-12                                     101ns ±26%
LookupPositive/SyncMap-12                                        202ns ± 2%
LookupPositive/AtomicPtrMap-12                                  71.1ns ± 2%
LookupPositive/AtomicPtrMapSharded-12                           73.2ns ± 1%
LookupNegative/RWMutexMap-12                                     119ns ± 1%
LookupNegative/SyncMap-12                                        154ns ± 1%
LookupNegative/AtomicPtrMap-12                                  84.7ns ± 3%
LookupNegative/AtomicPtrMapSharded-12                           86.8ns ± 1%
Concurrent/FixedKeys_1PercentWrites_RWMutexMap-12               1.32µs ± 2%
Concurrent/FixedKeys_1PercentWrites_SyncMap-12                  52.7ns ±10%
Concurrent/FixedKeys_1PercentWrites_AtomicPtrMap-12             31.8ns ±20%
Concurrent/FixedKeys_1PercentWrites_AtomicPtrMapSharded-12      24.0ns ±15%
Concurrent/FixedKeys_10PercentWrites_RWMutexMap-12               860ns ± 3%
Concurrent/FixedKeys_10PercentWrites_SyncMap-12                 68.8ns ±20%
Concurrent/FixedKeys_10PercentWrites_AtomicPtrMap-12            98.6ns ± 7%
Concurrent/FixedKeys_10PercentWrites_AtomicPtrMapSharded-12     42.0ns ±25%
Concurrent/FixedKeys_50PercentWrites_RWMutexMap-12              1.17µs ± 3%
Concurrent/FixedKeys_50PercentWrites_SyncMap-12                  136ns ±34%
Concurrent/FixedKeys_50PercentWrites_AtomicPtrMap-12             286ns ± 3%
Concurrent/FixedKeys_50PercentWrites_AtomicPtrMapSharded-12      115ns ±35%
Concurrent/ChangingKeys_1PercentWrites_RWMutexMap-12            1.27µs ± 2%
Concurrent/ChangingKeys_1PercentWrites_SyncMap-12               5.01µs ± 3%
Concurrent/ChangingKeys_1PercentWrites_AtomicPtrMap-12          38.1ns ± 3%
Concurrent/ChangingKeys_1PercentWrites_AtomicPtrMapSharded-12   22.6ns ± 2%
Concurrent/ChangingKeys_10PercentWrites_RWMutexMap-12           1.08µs ± 2%
Concurrent/ChangingKeys_10PercentWrites_SyncMap-12              5.97µs ± 1%
Concurrent/ChangingKeys_10PercentWrites_AtomicPtrMap-12          390ns ± 2%
Concurrent/ChangingKeys_10PercentWrites_AtomicPtrMapSharded-12  93.6ns ± 1%
Concurrent/ChangingKeys_50PercentWrites_RWMutexMap-12           1.77µs ± 2%
Concurrent/ChangingKeys_50PercentWrites_SyncMap-12              8.07µs ± 2%
Concurrent/ChangingKeys_50PercentWrites_AtomicPtrMap-12         1.61µs ± 2%
Concurrent/ChangingKeys_50PercentWrites_AtomicPtrMapSharded-12   386ns ± 1%

Updates #231

PiperOrigin-RevId: 346614776

nogo.yaml

@@ -214,6 +214,8 @@ analyzers:
   printf:
     external: # Enabled.
   shift:
+    generated: # Disabled for generated code; these shifts are well-defined.
+      exclude: [".*"]
     external: # Enabled.
   stringintconv:
     external:

pkg/sentry/vfs/mount_unsafe.go

@@ -12,11 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build go1.12
-// +build !go1.17
-
-// Check go:linkname function signatures when updating Go version.
-
 package vfs
 
 import (
@@ -41,6 +36,15 @@ type mountKey struct {
 	point  unsafe.Pointer // *Dentry
 }
 
+var (
+	mountKeyHasher = sync.MapKeyHasher(map[mountKey]struct{}(nil))
+	mountKeySeed   = sync.RandUintptr()
+)
+
+func (k *mountKey) hash() uintptr {
+	return mountKeyHasher(gohacks.Noescape(unsafe.Pointer(k)), mountKeySeed)
+}
+
 func (mnt *Mount) parent() *Mount {
 	return (*Mount)(atomic.LoadPointer(&mnt.key.parent))
 }
@@ -56,23 +60,17 @@ func (mnt *Mount) getKey() VirtualDentry {
 	}
 }
 
-func (mnt *Mount) saveKey() VirtualDentry { return mnt.getKey() }
-
 // Invariant: mnt.key.parent == nil. vd.Ok().
 func (mnt *Mount) setKey(vd VirtualDentry) {
 	atomic.StorePointer(&mnt.key.parent, unsafe.Pointer(vd.mount))
 	atomic.StorePointer(&mnt.key.point, unsafe.Pointer(vd.dentry))
 }
 
-func (mnt *Mount) loadKey(vd VirtualDentry) { mnt.setKey(vd) }
-
 // mountTable maps (mount parent, mount point) pairs to mounts. It supports
 // efficient concurrent lookup, even in the presence of concurrent mutators
 // (provided mutation is sufficiently uncommon).
 //
 // mountTable.Init() must be called on new mountTables before use.
-//
-// +stateify savable
 type mountTable struct {
 	// mountTable is implemented as a seqcount-protected hash table that
 	// resolves collisions with linear probing, featuring Robin Hood insertion
@@ -84,8 +82,7 @@ type mountTable struct {
 	// intrinsics and inline assembly, limiting the performance of this
 	// approach.)
 
-	seq  sync.SeqCount `state:"nosave"`
-	seed uint32        // for hashing keys
+	seq sync.SeqCount `state:"nosave"`
 
 	// size holds both length (number of elements) and capacity (number of
 	// slots): capacity is stored as its base-2 log (referred to as order) in
@@ -150,7 +147,6 @@ func init() {
 
 // Init must be called exactly once on each mountTable before use.
 func (mt *mountTable) Init() {
-	mt.seed = rand32()
 	mt.size = mtInitOrder
 	mt.slots = newMountTableSlots(mtInitCap)
 }
@@ -167,7 +163,7 @@ func newMountTableSlots(cap uintptr) unsafe.Pointer {
 // Lookup may be called even if there are concurrent mutators of mt.
 func (mt *mountTable) Lookup(parent *Mount, point *Dentry) *Mount {
 	key := mountKey{parent: unsafe.Pointer(parent), point: unsafe.Pointer(point)}
-	hash := memhash(gohacks.Noescape(unsafe.Pointer(&key)), uintptr(mt.seed), mountKeyBytes)
+	hash := key.hash()
 
 loop:
 	for {
@@ -247,7 +243,7 @@ func (mt *mountTable) Insert(mount *Mount) {
 // * mt.seq must be in a writer critical section.
 // * mt must not already contain a Mount with the same mount point and parent.
 func (mt *mountTable) insertSeqed(mount *Mount) {
-	hash := memhash(unsafe.Pointer(&mount.key), uintptr(mt.seed), mountKeyBytes)
+	hash := mount.key.hash()
 
 	// We're under the maximum load factor if:
 	//
@@ -346,7 +342,7 @@ func (mt *mountTable) Remove(mount *Mount) {
 // * mt.seq must be in a writer critical section.
 // * mt must contain mount.
 func (mt *mountTable) removeSeqed(mount *Mount) {
-	hash := memhash(unsafe.Pointer(&mount.key), uintptr(mt.seed), mountKeyBytes)
+	hash := mount.key.hash()
 	tcap := uintptr(1) << (mt.size & mtSizeOrderMask)
 	mask := tcap - 1
 	slots := mt.slots
@@ -386,9 +382,3 @@ func (mt *mountTable) removeSeqed(mount *Mount) {
 		off = (off + mountSlotBytes) & offmask
 	}
 }
-
-//go:linkname memhash runtime.memhash
-func memhash(p unsafe.Pointer, seed, s uintptr) uintptr
-
-//go:linkname rand32 runtime.fastrand
-func rand32() uint32

pkg/sentry/vfs/save_restore.go

@@ -101,6 +101,9 @@ func (vfs *VirtualFilesystem) saveMounts() []*Mount {
 	return mounts
 }
 
+// saveKey is called by stateify.
+func (mnt *Mount) saveKey() VirtualDentry { return mnt.getKey() }
+
 // loadMounts is called by stateify.
 func (vfs *VirtualFilesystem) loadMounts(mounts []*Mount) {
 	if mounts == nil {
@@ -112,6 +115,9 @@ func (vfs *VirtualFilesystem) loadMounts(mounts []*Mount) {
 	}
 }
 
+// loadKey is called by stateify.
+func (mnt *Mount) loadKey(vd VirtualDentry) { mnt.setKey(vd) }
+
 func (mnt *Mount) afterLoad() {
 	if atomic.LoadInt64(&mnt.refs) != 0 {
 		refsvfs2.Register(mnt)

pkg/sync/BUILD

@@ -10,15 +10,34 @@ exports_files(["LICENSE"])
 
 go_template(
     name = "generic_atomicptr",
-    srcs = ["atomicptr_unsafe.go"],
+    srcs = ["generic_atomicptr_unsafe.go"],
     types = [
         "Value",
     ],
 )
 
+go_template(
+    name = "generic_atomicptrmap",
+    srcs = ["generic_atomicptrmap_unsafe.go"],
+    opt_consts = [
+        "ShardOrder",
+    ],
+    opt_types = [
+        "Hasher",
+    ],
+    types = [
+        "Key",
+        "Value",
+    ],
+    deps = [
+        ":sync",
+        "//pkg/gohacks",
+    ],
+)
+
 go_template(
     name = "generic_seqatomic",
-    srcs = ["seqatomic_unsafe.go"],
+    srcs = ["generic_seqatomic_unsafe.go"],
     types = [
         "Value",
     ],

pkg/sync/atomicptrmaptest/BUILD

@@ -0,0 +1,57 @@
+load("//tools:defs.bzl", "go_library", "go_test")
+load("//tools/go_generics:defs.bzl", "go_template_instance")
+
+package(
+    default_visibility = ["//visibility:private"],
+    licenses = ["notice"],
+)
+
+go_template_instance(
+    name = "test_atomicptrmap",
+    out = "test_atomicptrmap_unsafe.go",
+    package = "atomicptrmap",
+    prefix = "test",
+    template = "//pkg/sync:generic_atomicptrmap",
+    types = {
+        "Key": "int64",
+        "Value": "testValue",
+    },
+)
+
+go_template_instance(
+    name = "test_atomicptrmap_sharded",
+    out = "test_atomicptrmap_sharded_unsafe.go",
+    consts = {
+        "ShardOrder": "4",
+    },
+    package = "atomicptrmap",
+    prefix = "test",
+    suffix = "Sharded",
+    template = "//pkg/sync:generic_atomicptrmap",
+    types = {
+        "Key": "int64",
+        "Value": "testValue",
+    },
+)
+
+go_library(
+    name = "atomicptrmap",
+    testonly = 1,
+    srcs = [
+        "atomicptrmap.go",
+        "test_atomicptrmap_sharded_unsafe.go",
+        "test_atomicptrmap_unsafe.go",
+    ],
+    deps = [
+        "//pkg/gohacks",
+        "//pkg/sync",
+    ],
+)
+
+go_test(
+    name = "atomicptrmap_test",
+    size = "small",
+    srcs = ["atomicptrmap_test.go"],
+    library = ":atomicptrmap",
+    deps = ["//pkg/sync"],
+)

pkg/sync/atomicptrmaptest/atomicptrmap.go

@@ -0,0 +1,20 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package atomicptrmap instantiates generic_atomicptrmap for testing.
+package atomicptrmap
+
+type testValue struct {
+	val int
+}

pkg/sync/generic_atomicptr_unsafe.go

@@ -3,9 +3,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package template doesn't exist. This file must be instantiated using the
+// Package seqatomic doesn't exist. This file must be instantiated using the
 // go_template_instance rule in tools/go_generics/defs.bzl.
-package template
+package seqatomic
 
 import (
 	"sync/atomic"

pkg/sync/generic_seqatomic_unsafe.go

@@ -3,9 +3,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package template doesn't exist. This file must be instantiated using the
+// Package seqatomic doesn't exist. This file must be instantiated using the
 // go_template_instance rule in tools/go_generics/defs.bzl.
-package template
+package seqatomic
 
 import (
 	"unsafe"

pkg/sync/runtime_unsafe.go

@@ -11,6 +11,8 @@
 package sync
 
 import (
+	"fmt"
+	"reflect"
 	"unsafe"
 )
 
@@ -61,6 +63,57 @@ const (
 	TraceEvGoBlockSelect byte = 24
 )
 
+// Rand32 returns a non-cryptographically-secure random uint32.
+func Rand32() uint32 {
+	return fastrand()
+}
+
+// Rand64 returns a non-cryptographically-secure random uint64.
+func Rand64() uint64 {
+	return uint64(fastrand())<<32 | uint64(fastrand())
+}
+
+//go:linkname fastrand runtime.fastrand
+func fastrand() uint32
+
+// RandUintptr returns a non-cryptographically-secure random uintptr.
+func RandUintptr() uintptr {
+	if unsafe.Sizeof(uintptr(0)) == 4 {
+		return uintptr(Rand32())
+	}
+	return uintptr(Rand64())
+}
+
+// MapKeyHasher returns a hash function for pointers of m's key type.
+//
+// Preconditions: m must be a map.
+func MapKeyHasher(m interface{}) func(unsafe.Pointer, uintptr) uintptr {
+	if rtyp := reflect.TypeOf(m); rtyp.Kind() != reflect.Map {
+		panic(fmt.Sprintf("sync.MapKeyHasher: m is %v, not map", rtyp))
+	}
+	mtyp := *(**maptype)(unsafe.Pointer(&m))
+	return mtyp.hasher
+}
+
+type maptype struct {
+	size       uintptr
+	ptrdata    uintptr
+	hash       uint32
+	tflag      uint8
+	align      uint8
+	fieldAlign uint8
+	kind       uint8
+	equal      func(unsafe.Pointer, unsafe.Pointer) bool
+	gcdata     *byte
+	str        int32
+	ptrToThis  int32
+	key        unsafe.Pointer
+	elem       unsafe.Pointer
+	bucket     unsafe.Pointer
+	hasher     func(unsafe.Pointer, uintptr) uintptr
+	// more fields
+}
+
 // These functions are only used within the sync package.
 
 //go:linkname semacquire sync.runtime_Semacquire

tools/go_generics/generics.go

@@ -223,7 +223,7 @@ func main() {
 		} else {
 			switch kind {
 			case globals.KindType, globals.KindVar, globals.KindConst, globals.KindFunction:
-				if ident.Name != "_" {
+				if ident.Name != "_" && !(ident.Name == "init" && kind == globals.KindFunction) {
 					ident.Name = *prefix + ident.Name + *suffix
 				}
 			case globals.KindTag: