From 1cef45605e9c9ea8a21530426f7341b206469845 Mon Sep 17 00:00:00 2001 From: howard zhang Date: Mon, 28 Mar 2022 16:29:27 +0800 Subject: [PATCH] Arm64 atomic: add LSE atomics instructions to arm64 benchmark for the atomics instructions: Without LSE instructions: BenchmarkAndUint32-80 138205774 8.562 ns/op BenchmarkAndUint64-80 139365404 8.571 ns/op BenchmarkAndUint32Parallel-80 16887577 259.6 ns/op BenchmarkAndUint64Parallel-80 6202482 281.2 ns/op BenchmarkOrUint32-80 138791865 8.592 ns/op BenchmarkOrUint64-80 138424888 8.670 ns/op BenchmarkOrUint32Parallel-80 15080332 175.1 ns/op BenchmarkOrUint64Parallel-80 8891142 262.2 ns/op BenchmarkXorUint32-80 139146337 8.611 ns/op BenchmarkXorUint64-80 138612325 8.627 ns/op BenchmarkXorUint32Parallel-80 3651159 548.5 ns/op BenchmarkXorUint64Parallel-80 17135235 350.2 ns/op BenchmarkCompareAndSwapUint32-80 11712571 95.49 ns/op BenchmarkCompareAndSwapUint64-80 12229885 87.32 ns/op BenchmarkCompareAndSwapUint32Parallel-80 11853363 115.6 ns/op BenchmarkCompareAndSwapUint64Parallel-80 15574476 87.91 ns/op With LSE instructions: BenchmarkAndUint32-80 172129803 6.933 ns/op BenchmarkAndUint64-80 170516512 6.993 ns/op BenchmarkAndUint32Parallel-80 60089396 20.14 ns/op BenchmarkAndUint64Parallel-80 59266794 20.01 ns/op BenchmarkOrUint32-80 174805162 6.835 ns/op BenchmarkOrUint64-80 174935716 6.836 ns/op BenchmarkOrUint32Parallel-80 60084607 19.89 ns/op BenchmarkOrUint64Parallel-80 51555650 19.99 ns/op BenchmarkXorUint32-80 170525718 6.835 ns/op BenchmarkXorUint64-80 175269829 6.842 ns/op BenchmarkXorUint32Parallel-80 60005954 20.15 ns/op BenchmarkXorUint64Parallel-80 51590584 19.95 ns/op BenchmarkCompareAndSwapUint32-80 27096822 40.69 ns/op BenchmarkCompareAndSwapUint64-80 27516450 39.02 ns/op BenchmarkCompareAndSwapUint32Parallel-80 27194458 40.69 ns/op BenchmarkCompareAndSwapUint64Parallel-80 27614142 39.25 ns/op Signed-off-by: howard zhang --- pkg/atomicbitops/BUILD | 14 +- pkg/atomicbitops/atomicbitops_arm64.go | 26 +++ pkg/atomicbitops/atomicbitops_arm64.s | 104 +++++++---- .../atomicbitops_benchmark_test.go | 171 ++++++++++++++++++ pkg/tcpip/BUILD | 1 + 5 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 pkg/atomicbitops/atomicbitops_arm64.go create mode 100644 pkg/atomicbitops/atomicbitops_benchmark_test.go diff --git a/pkg/atomicbitops/BUILD b/pkg/atomicbitops/BUILD index 993946e43..3622c9270 100644 --- a/pkg/atomicbitops/BUILD +++ b/pkg/atomicbitops/BUILD @@ -10,14 +10,26 @@ go_library( "aligned_32bit_unsafe.go", "aligned_64bit.go", "atomicbitops.go", + "atomicbitops_arm64.go", "atomicbitops_amd64.s", "atomicbitops_arm64.s", "atomicbitops_noasm.go", ], visibility = ["//:sandbox"], - deps = ["//pkg/sync"], + deps = [ + "//pkg/cpuid", + "//pkg/sync", + ], ) +go_test( + name = "atomicbitops_benchmark_test", + size = "small", + srcs = ["atomicbitops_benchmark_test.go"], + library = ":atomicbitops", +) + + go_test( name = "atomicbitops_test", size = "small", diff --git a/pkg/atomicbitops/atomicbitops_arm64.go b/pkg/atomicbitops/atomicbitops_arm64.go new file mode 100644 index 000000000..60903f042 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_arm64.go @@ -0,0 +1,26 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build arm64 +// +build arm64 + +// Package atomicbitops provides extensions to the sync/atomic package. +// +// All read-modify-write operations implemented by this package have +// acquire-release memory ordering (like sync/atomic). +package atomicbitops + +import "gvisor.dev/gvisor/pkg/cpuid" + +var arm64HasATOMICS = cpuid.HostFeatureSet().HasFeature(cpuid.ARM64FeatureATOMICS) diff --git a/pkg/atomicbitops/atomicbitops_arm64.s b/pkg/atomicbitops/atomicbitops_arm64.s index 600c591cf..b2b950c74 100644 --- a/pkg/atomicbitops/atomicbitops_arm64.s +++ b/pkg/atomicbitops/atomicbitops_arm64.s @@ -19,87 +19,123 @@ TEXT ·andUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALW R2, (R0), R3 + RET +load_store_loop: LDAXRW (R0), R2 ANDW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·orUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALW R1, (R0), R2 + RET +load_store_loop: LDAXRW (R0), R2 ORRW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·xorUint32(SB),NOSPLIT,$0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALW R1, (R0), R2 + RET +load_store_loop: LDAXRW (R0), R2 EORW R1, R2 STLXRW R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·compareAndSwapUint32(SB),NOSPLIT,$0-20 - MOVD addr+0(FP), R0 - MOVW old+8(FP), R1 - MOVW new+12(FP), R2 -again: - LDAXRW (R0), R3 - CMPW R1, R3 - BNE done - STLXRW R2, (R0), R4 - CBNZ R4, again -done: - MOVW R3, prev+16(FP) + MOVD addr+0(FP), R0 + MOVW old+8(FP), R1 + MOVW new+12(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALW R1, (R0), R2 + MOVW R1, prev+16(FP) + RET +load_store_loop: + LDAXRW (R0), R3 + CMPW R1, R3 + BNE ok + STLXRW R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVW R3, prev+16(FP) RET TEXT ·andUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + MVN R1, R2 + LDCLRALD R2, (R0), R3 + RET +load_store_loop: LDAXR (R0), R2 AND R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·orUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDORALD R1, (R0), R2 + RET +load_store_loop: LDAXR (R0), R2 ORR R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·xorUint64(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD val+8(FP), R1 -again: + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + LDEORALD R1, (R0), R2 + RET +load_store_loop: LDAXR (R0), R2 EOR R1, R2 STLXR R2, (R0), R3 - CBNZ R3, again + CBNZ R3, load_store_loop RET TEXT ·compareAndSwapUint64(SB),NOSPLIT,$0-32 - MOVD addr+0(FP), R0 - MOVD old+8(FP), R1 - MOVD new+16(FP), R2 -again: - LDAXR (R0), R3 - CMP R1, R3 - BNE done - STLXR R2, (R0), R4 - CBNZ R4, again -done: - MOVD R3, prev+24(FP) + MOVD addr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R2 + MOVBU ·arm64HasATOMICS(SB), R4 + CBZ R4, load_store_loop + CASALD R1, (R0), R2 + MOVD R1, prev+24(FP) + RET +load_store_loop: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R4 + CBNZ R4, load_store_loop +ok: + MOVD R3, prev+24(FP) RET diff --git a/pkg/atomicbitops/atomicbitops_benchmark_test.go b/pkg/atomicbitops/atomicbitops_benchmark_test.go new file mode 100644 index 000000000..daaa76180 --- /dev/null +++ b/pkg/atomicbitops/atomicbitops_benchmark_test.go @@ -0,0 +1,171 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atomicbitops + +import ( + "testing" +) + +func BenchmarkAndUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + AndUint32(&val32, uint32(i)) + } +} + +func BenchmarkAndUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + AndUint64(&val64, uint64(i)) + } +} + +func BenchmarkAndUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + AndUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkAndUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + AndUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkOrUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + OrUint32(&val32, uint32(i)) + } +} + +func BenchmarkOrUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + OrUint64(&val64, uint64(i)) + } +} + +func BenchmarkOrUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + OrUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkOrUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + OrUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkXorUint32(b *testing.B) { + val32 := FromUint32(0xfffffff) + for i := 0; i < b.N; i++ { + XorUint32(&val32, uint32(i)) + } +} + +func BenchmarkXorUint64(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + for i := 0; i < b.N; i++ { + XorUint64(&val64, uint64(i)) + } +} + +func BenchmarkXorUint32Parallel(b *testing.B) { + val32 := FromUint32(0xfffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint32(0) + for pb.Next() { + XorUint32(&val32, i) + i++ + } + }) +} + +func BenchmarkXorUint64Parallel(b *testing.B) { + val64 := FromUint64(0xfffffffffffffff) + b.RunParallel(func(pb *testing.PB) { + i := uint64(0) + for pb.Next() { + XorUint64(&val64, i) + i++ + } + }) +} + +func BenchmarkCompareAndSwapUint32(b *testing.B) { + x := FromUint32(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint32(ptr, 1, 0) + CompareAndSwapUint32(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint64(b *testing.B) { + x := FromUint64(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint64(ptr, 1, 0) + CompareAndSwapUint64(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint32Parallel(b *testing.B) { + x := FromUint32(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint32(ptr, 1, 0) + CompareAndSwapUint32(ptr, 0, 1) + } + }) +} + +func BenchmarkCompareAndSwapUint64Parallel(b *testing.B) { + x := FromUint64(1) + ptr := &x + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + CompareAndSwapUint64(ptr, 1, 0) + CompareAndSwapUint64(ptr, 0, 1) + } + }) +} diff --git a/pkg/tcpip/BUILD b/pkg/tcpip/BUILD index 46676032e..29d31c5d7 100644 --- a/pkg/tcpip/BUILD +++ b/pkg/tcpip/BUILD @@ -50,6 +50,7 @@ deps_test( "//pkg/atomicbitops", "//pkg/buffer", "//pkg/context", + "//pkg/cpuid", "//pkg/gohacks", "//pkg/goid", "//pkg/ilist",