diff --git a/pkg/atomicbitops/BUILD b/pkg/atomicbitops/BUILD index 993946e43..cebdceb0d 100644 --- a/pkg/atomicbitops/BUILD +++ b/pkg/atomicbitops/BUILD @@ -11,11 +11,22 @@ go_library( "aligned_64bit.go", "atomicbitops.go", "atomicbitops_amd64.s", + "atomicbitops_arm64.go", "atomicbitops_arm64.s", "atomicbitops_noasm.go", ], visibility = ["//:sandbox"], - deps = ["//pkg/sync"], + deps = [ + "//pkg/cpuid", + "//pkg/sync", + ], +) + +go_test( + name = "atomicbitops_benchmark_test", + size = "small", + srcs = ["atomicbitops_benchmark_test.go"], + library = ":atomicbitops", ) go_test( diff --git a/pkg/atomicbitops/atomicbitops_arm64.go b/pkg/atomicbitops/atomicbitops_arm64.go new file mode 100644 index 000000000..7139885c8 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_arm64.go @@ -0,0 +1,22 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +package atomicbitops + +import "gvisor.dev/gvisor/pkg/cpuid" + +var arm64HasATOMICS = cpuid.HostFeatureSet().HasFeature(cpuid.ARM64FeatureATOMICS) diff --git a/pkg/atomicbitops/atomicbitops_arm64.s b/pkg/atomicbitops/atomicbitops_arm64.s index 600c591cf..b2b950c74 100644 --- a/pkg/atomicbitops/atomicbitops_arm64.s +++ b/pkg/atomicbitops/atomicbitops_arm64.s @@ -19,87 +19,123 @@ TEXT ·andUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALW R2, (R0), R3 + RET +load_store_loop: LDAXRW (R0), R2 ANDW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·orUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALW R1, (R0), R2 + RET +load_store_loop: LDAXRW (R0), R2 ORRW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·xorUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALW R1, (R0), R2 + RET +load_store_loop: LDAXRW (R0), R2 EORW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·compareAndSwapUint32(SB),NOSPLIT,$0-20 - MOVD addr+0(FP), R0 - MOVW old+8(FP), R1 - MOVW new+12(FP), R2 -again: - LDAXRW (R0), R3 - CMPW R1, R3 - BNE done - STLXRW R2, (R0), R4 - CBNZ R4, again -done: - MOVW R3, prev+16(FP) + MOVD addr+0(FP), R0 + MOVW old+8(FP), R1 + MOVW new+12(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALW R1, (R0), R2 + MOVW R1, prev+16(FP) + RET +load_store_loop: + LDAXRW (R0), R3 + CMPW R1, R3 + BNE ok + STLXRW R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVW R3, prev+16(FP) RET TEXT ·andUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALD R2, (R0), R3 + RET +load_store_loop: LDAXR (R0), R2 AND R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·orUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALD R1, (R0), R2 + RET +load_store_loop: LDAXR (R0), R2 ORR R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·xorUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALD R1, (R0), R2 + RET +load_store_loop: LDAXR (R0), R2 EOR R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·compareAndSwapUint64(SB),NOSPLIT,$0-32 - MOVD addr+0(FP), R0 - MOVD old+8(FP), R1 - MOVD new+16(FP), R2 -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE done - STLXR R2, (R0), R4 - CBNZ R4, again -done: - MOVD R3, prev+24(FP) + MOVD addr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALD R1, (R0), R2 + MOVD R1, prev+24(FP) + RET +load_store_loop: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVD R3, prev+24(FP) RET diff --git a/pkg/atomicbitops/atomicbitops_benchmark_test.go b/pkg/atomicbitops/atomicbitops_benchmark_test.go new file mode 100644 index 000000000..daaa76180 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_benchmark_test.go @@ -0,0 +1,171 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atomicbitops + +import ( + "testing" +) + +func BenchmarkAndUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + AndUint32(&val32, uint32(i)) + } +} + +func BenchmarkAndUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + AndUint64(&val64, uint64(i)) + } +} + +func BenchmarkAndUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + AndUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkAndUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + AndUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkOrUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + OrUint32(&val32, uint32(i)) + } +} + +func BenchmarkOrUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + OrUint64(&val64, uint64(i)) + } +} + +func BenchmarkOrUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + OrUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkOrUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + OrUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkXorUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + XorUint32(&val32, uint32(i)) + } +} + +func BenchmarkXorUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + XorUint64(&val64, uint64(i)) + } +} + +func BenchmarkXorUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + XorUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkXorUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + XorUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkCompareAndSwapUint32(b *testing.B) { + x := FromUint32(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint32(ptr, 1, 0) + CompareAndSwapUint32(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint64(b *testing.B) { + x := FromUint64(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint64(ptr, 1, 0) + CompareAndSwapUint64(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint32Parallel(b *testing.B) { + x := FromUint32(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint32(ptr, 1, 0) + CompareAndSwapUint32(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint64Parallel(b *testing.B) { + x := FromUint64(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint64(ptr, 1, 0) + CompareAndSwapUint64(ptr, 0, 1) + } + }) +} diff --git a/pkg/cpuid/BUILD b/pkg/cpuid/BUILD index 4bcffaf61..2b63ec6b3 100644 --- a/pkg/cpuid/BUILD +++ b/pkg/cpuid/BUILD @@ -17,7 +17,6 @@ go_library( ], visibility = ["//:sandbox"], deps = [ - "//pkg/context", "//pkg/log", ], ) diff --git a/pkg/cpuid/cpuid.go b/pkg/cpuid/cpuid.go index 8e3390f07..f7fa74a3a 100644 --- a/pkg/cpuid/cpuid.go +++ b/pkg/cpuid/cpuid.go @@ -30,8 +30,6 @@ package cpuid import ( "fmt" "strings" - - "gvisor.dev/gvisor/pkg/context" ) // contextID is the package for context.Context.Value keys. @@ -42,8 +40,13 @@ const ( CtxFeatureSet contextID = iota ) +// context represents context.Context. +type context interface { + Value(key interface{}) interface{} +} + // FromContext returns the FeatureSet from the context, if available. -func FromContext(ctx context.Context) FeatureSet { +func FromContext(ctx context) FeatureSet { v := ctx.Value(CtxFeatureSet) if v == nil { return FeatureSet{} // Panics if used. diff --git a/pkg/tcpip/BUILD b/pkg/tcpip/BUILD index 46676032e..29d31c5d7 100644 --- a/pkg/tcpip/BUILD +++ b/pkg/tcpip/BUILD @@ -50,6 +50,7 @@ deps_test( "//pkg/atomicbitops", "//pkg/buffer", "//pkg/context", + "//pkg/cpuid", "//pkg/gohacks", "//pkg/goid", "//pkg/ilist", diff --git a/pkg/waiter/BUILD b/pkg/waiter/BUILD index 505d3eef5..5ad715f1e 100644 --- a/pkg/waiter/BUILD +++ b/pkg/waiter/BUILD @@ -23,7 +23,6 @@ go_library( ], visibility = ["//visibility:public"], deps = [ - "//pkg/atomicbitops", "//pkg/sync", ], )