mirror of
https://github.com/netbirdio/gvisor.git
synced 2026-05-22 17:12:49 -07:00
Arm64 atomic: add LSE atomics instructions to arm64
benchmark for the atomics instructions: Without LSE instructions: BenchmarkAndUint32-80 138205774 8.562 ns/op BenchmarkAndUint64-80 139365404 8.571 ns/op BenchmarkAndUint32Parallel-80 16887577 259.6 ns/op BenchmarkAndUint64Parallel-80 6202482 281.2 ns/op BenchmarkOrUint32-80 138791865 8.592 ns/op BenchmarkOrUint64-80 138424888 8.670 ns/op BenchmarkOrUint32Parallel-80 15080332 175.1 ns/op BenchmarkOrUint64Parallel-80 8891142 262.2 ns/op BenchmarkXorUint32-80 139146337 8.611 ns/op BenchmarkXorUint64-80 138612325 8.627 ns/op BenchmarkXorUint32Parallel-80 3651159 548.5 ns/op BenchmarkXorUint64Parallel-80 17135235 350.2 ns/op BenchmarkCompareAndSwapUint32-80 11712571 95.49 ns/op BenchmarkCompareAndSwapUint64-80 12229885 87.32 ns/op BenchmarkCompareAndSwapUint32Parallel-80 11853363 115.6 ns/op BenchmarkCompareAndSwapUint64Parallel-80 15574476 87.91 ns/op With LSE instructions: BenchmarkAndUint32-80 172129803 6.933 ns/op BenchmarkAndUint64-80 170516512 6.993 ns/op BenchmarkAndUint32Parallel-80 60089396 20.14 ns/op BenchmarkAndUint64Parallel-80 59266794 20.01 ns/op BenchmarkOrUint32-80 174805162 6.835 ns/op BenchmarkOrUint64-80 174935716 6.836 ns/op BenchmarkOrUint32Parallel-80 60084607 19.89 ns/op BenchmarkOrUint64Parallel-80 51555650 19.99 ns/op BenchmarkXorUint32-80 170525718 6.835 ns/op BenchmarkXorUint64-80 175269829 6.842 ns/op BenchmarkXorUint32Parallel-80 60005954 20.15 ns/op BenchmarkXorUint64Parallel-80 51590584 19.95 ns/op BenchmarkCompareAndSwapUint32-80 27096822 40.69 ns/op BenchmarkCompareAndSwapUint64-80 27516450 39.02 ns/op BenchmarkCompareAndSwapUint32Parallel-80 27194458 40.69 ns/op BenchmarkCompareAndSwapUint64Parallel-80 27614142 39.25 ns/op Signed-off-by: howard zhang <howard.zhang@arm.com>
This commit is contained in:
howard zhang
committed by
Howard Zhang
Howard Zhang
parent
f4e537843f
commit
1cef45605e
+13
-1
@@ -10,14 +10,26 @@ go_library(
|
||||
"aligned_32bit_unsafe.go",
|
||||
"aligned_64bit.go",
|
||||
"atomicbitops.go",
|
||||
"atomicbitops_arm64.go",
|
||||
"atomicbitops_amd64.s",
|
||||
"atomicbitops_arm64.s",
|
||||
"atomicbitops_noasm.go",
|
||||
],
|
||||
visibility = ["//:sandbox"],
|
||||
deps = ["//pkg/sync"],
|
||||
deps = [
|
||||
"//pkg/cpuid",
|
||||
"//pkg/sync",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "atomicbitops_benchmark_test",
|
||||
size = "small",
|
||||
srcs = ["atomicbitops_benchmark_test.go"],
|
||||
library = ":atomicbitops",
|
||||
)
|
||||
|
||||
|
||||
go_test(
|
||||
name = "atomicbitops_test",
|
||||
size = "small",
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2022 The gVisor Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build arm64
|
||||
// +build arm64
|
||||
|
||||
// Package atomicbitops provides extensions to the sync/atomic package.
|
||||
//
|
||||
// All read-modify-write operations implemented by this package have
|
||||
// acquire-release memory ordering (like sync/atomic).
|
||||
package atomicbitops
|
||||
|
||||
import "gvisor.dev/gvisor/pkg/cpuid"
|
||||
|
||||
var arm64HasATOMICS = cpuid.HostFeatureSet().HasFeature(cpuid.ARM64FeatureATOMICS)
|
||||
@@ -19,87 +19,123 @@
|
||||
TEXT ·andUint32(SB),NOSPLIT,$0-12
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVW val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
MVN R1, R2
|
||||
LDCLRALW R2, (R0), R3
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXRW (R0), R2
|
||||
ANDW R1, R2
|
||||
STLXRW R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·orUint32(SB),NOSPLIT,$0-12
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVW val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
LDORALW R1, (R0), R2
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXRW (R0), R2
|
||||
ORRW R1, R2
|
||||
STLXRW R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·xorUint32(SB),NOSPLIT,$0-12
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVW val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
LDEORALW R1, (R0), R2
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXRW (R0), R2
|
||||
EORW R1, R2
|
||||
STLXRW R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·compareAndSwapUint32(SB),NOSPLIT,$0-20
|
||||
MOVD addr+0(FP), R0
|
||||
MOVW old+8(FP), R1
|
||||
MOVW new+12(FP), R2
|
||||
again:
|
||||
LDAXRW (R0), R3
|
||||
CMPW R1, R3
|
||||
BNE done
|
||||
STLXRW R2, (R0), R4
|
||||
CBNZ R4, again
|
||||
done:
|
||||
MOVW R3, prev+16(FP)
|
||||
MOVD addr+0(FP), R0
|
||||
MOVW old+8(FP), R1
|
||||
MOVW new+12(FP), R2
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
CASALW R1, (R0), R2
|
||||
MOVW R1, prev+16(FP)
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXRW (R0), R3
|
||||
CMPW R1, R3
|
||||
BNE ok
|
||||
STLXRW R2, (R0), R4
|
||||
CBNZ R4, load_store_loop
|
||||
ok:
|
||||
MOVW R3, prev+16(FP)
|
||||
RET
|
||||
|
||||
TEXT ·andUint64(SB),NOSPLIT,$0-16
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVD val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
MVN R1, R2
|
||||
LDCLRALD R2, (R0), R3
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXR (R0), R2
|
||||
AND R1, R2
|
||||
STLXR R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·orUint64(SB),NOSPLIT,$0-16
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVD val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
LDORALD R1, (R0), R2
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXR (R0), R2
|
||||
ORR R1, R2
|
||||
STLXR R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·xorUint64(SB),NOSPLIT,$0-16
|
||||
MOVD ptr+0(FP), R0
|
||||
MOVD val+8(FP), R1
|
||||
again:
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
LDEORALD R1, (R0), R2
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXR (R0), R2
|
||||
EOR R1, R2
|
||||
STLXR R2, (R0), R3
|
||||
CBNZ R3, again
|
||||
CBNZ R3, load_store_loop
|
||||
RET
|
||||
|
||||
TEXT ·compareAndSwapUint64(SB),NOSPLIT,$0-32
|
||||
MOVD addr+0(FP), R0
|
||||
MOVD old+8(FP), R1
|
||||
MOVD new+16(FP), R2
|
||||
again:
|
||||
LDAXR (R0), R3
|
||||
CMP R1, R3
|
||||
BNE done
|
||||
STLXR R2, (R0), R4
|
||||
CBNZ R4, again
|
||||
done:
|
||||
MOVD R3, prev+24(FP)
|
||||
MOVD addr+0(FP), R0
|
||||
MOVD old+8(FP), R1
|
||||
MOVD new+16(FP), R2
|
||||
MOVBU ·arm64HasATOMICS(SB), R4
|
||||
CBZ R4, load_store_loop
|
||||
CASALD R1, (R0), R2
|
||||
MOVD R1, prev+24(FP)
|
||||
RET
|
||||
load_store_loop:
|
||||
LDAXR (R0), R3
|
||||
CMP R1, R3
|
||||
BNE ok
|
||||
STLXR R2, (R0), R4
|
||||
CBNZ R4, load_store_loop
|
||||
ok:
|
||||
MOVD R3, prev+24(FP)
|
||||
RET
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
// Copyright 2018 The gVisor Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package atomicbitops
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkAndUint32(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
AndUint32(&val32, uint32(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAndUint64(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
AndUint64(&val64, uint64(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAndUint32Parallel(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint32(0)
|
||||
for pb.Next() {
|
||||
AndUint32(&val32, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkAndUint64Parallel(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint64(0)
|
||||
for pb.Next() {
|
||||
AndUint64(&val64, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkOrUint32(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
OrUint32(&val32, uint32(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOrUint64(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
OrUint64(&val64, uint64(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOrUint32Parallel(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint32(0)
|
||||
for pb.Next() {
|
||||
OrUint32(&val32, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkOrUint64Parallel(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint64(0)
|
||||
for pb.Next() {
|
||||
OrUint64(&val64, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkXorUint32(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
XorUint32(&val32, uint32(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkXorUint64(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
for i := 0; i < b.N; i++ {
|
||||
XorUint64(&val64, uint64(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkXorUint32Parallel(b *testing.B) {
|
||||
val32 := FromUint32(0xfffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint32(0)
|
||||
for pb.Next() {
|
||||
XorUint32(&val32, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkXorUint64Parallel(b *testing.B) {
|
||||
val64 := FromUint64(0xfffffffffffffff)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := uint64(0)
|
||||
for pb.Next() {
|
||||
XorUint64(&val64, i)
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkCompareAndSwapUint32(b *testing.B) {
|
||||
x := FromUint32(1)
|
||||
ptr := &x
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
CompareAndSwapUint32(ptr, 1, 0)
|
||||
CompareAndSwapUint32(ptr, 0, 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkCompareAndSwapUint64(b *testing.B) {
|
||||
x := FromUint64(1)
|
||||
ptr := &x
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
CompareAndSwapUint64(ptr, 1, 0)
|
||||
CompareAndSwapUint64(ptr, 0, 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkCompareAndSwapUint32Parallel(b *testing.B) {
|
||||
x := FromUint32(1)
|
||||
ptr := &x
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
CompareAndSwapUint32(ptr, 1, 0)
|
||||
CompareAndSwapUint32(ptr, 0, 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkCompareAndSwapUint64Parallel(b *testing.B) {
|
||||
x := FromUint64(1)
|
||||
ptr := &x
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
CompareAndSwapUint64(ptr, 1, 0)
|
||||
CompareAndSwapUint64(ptr, 0, 1)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -50,6 +50,7 @@ deps_test(
|
||||
"//pkg/atomicbitops",
|
||||
"//pkg/buffer",
|
||||
"//pkg/context",
|
||||
"//pkg/cpuid",
|
||||
"//pkg/gohacks",
|
||||
"//pkg/goid",
|
||||
"//pkg/ilist",
|
||||
|
||||
Reference in New Issue
Block a user