diff --git a/BUILD b/BUILD index 6cbd2c413..a3295b470 100644 --- a/BUILD +++ b/BUILD @@ -54,7 +54,7 @@ doc( yaml_test( name = "nogo_config_test", srcs = glob(["nogo*.yaml"]), - schema = "//tools/nogo:config-schema.json", + schema = "//tools/nogo/config:schema.json", ) yaml_test( diff --git a/Makefile b/Makefile index 0ef24c9f4..ae33685f5 100644 --- a/Makefile +++ b/Makefile @@ -196,7 +196,7 @@ nogo-tests: # For unit tests, we take everything in the root, pkg/... and tools/..., and # pull in all directories in runsc except runsc/container. unit-tests: ## Local package unit tests in pkg/..., tools/.., etc. - @$(call test,--build_tag_filters=-nogo --test_tag_filters=-nogo --test_filter=-//runsc/container/... //:all pkg/... tools/... runsc/...) + @$(call test,--build_tag_filters=-nogo --test_tag_filters=-nogo --test_filter=-//runsc/container/... //:all pkg/... tools/... runsc/... vdso/...) .PHONY: unit-tests # See unit-tests: this includes runsc/container. diff --git a/WORKSPACE b/WORKSPACE index 9dab4825f..8efe045f6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -36,8 +36,6 @@ http_archive( name = "io_bazel_rules_go", patch_args = ["-p1"], patches = [ - # Ensure we don't destroy the facts visibility. - "//tools:rules_go_visibility.patch", # Newer versions of the rules_go rules will automatically strip test # binaries of symbols, which we don't want. "//tools:rules_go_symbols.patch", @@ -51,13 +49,6 @@ http_archive( http_archive( name = "bazel_gazelle", - patch_args = ["-p1"], - patches = [ - # Fix permissions for facts for go_library, not just tool library. - # This is actually a no-op with the hacky patch above, but should - # slightly future proof this mechanism. - "//tools:bazel_gazelle_generate.patch", - ], sha256 = "62ca106be173579c0a167deb23358fdfe71ffa1e4cfdddf5582af26520f1c66f", urls = [ "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz", diff --git a/pkg/ring0/BUILD b/pkg/ring0/BUILD index 525c5d1f8..e04080eda 100644 --- a/pkg/ring0/BUILD +++ b/pkg/ring0/BUILD @@ -1,69 +1,50 @@ -load("//tools:defs.bzl", "arch_genrule", "go_library") -load("//tools/go_generics:defs.bzl", "go_template", "go_template_instance") +load("//tools:defs.bzl", "arch_genrule", "go_library", "select_arch") +load("//tools/nogo:defs.bzl", "nogo_facts") package(licenses = ["notice"]) -go_template( - name = "defs_amd64", - srcs = [ - "defs.go", - "defs_amd64.go", - "offsets_amd64.go", - "x86.go", - ], - visibility = [":__subpackages__"], -) +exports_files(glob(["*.go"])) -go_template( - name = "defs_arm64", +nogo_facts( + name = "entry_impl", srcs = [ "aarch64.go", "defs.go", + "defs_amd64.go", "defs_arm64.go", - "offsets_arm64.go", + "entry_template.go", + "x86.go", + ], + output = "entry_impl.s", + template = select_arch( + amd64 = "entry_amd64.s", + arm64 = "entry_arm64.s", + ), + deps = [ + "//pkg/abi/linux", + "//pkg/cpuid", + "//pkg/hostarch", + "//pkg/ring0/pagetables", + "//pkg/sentry/arch", + "//pkg/sentry/arch/fpu", ], - visibility = [":__subpackages__"], -) - -go_template_instance( - name = "defs_impl_amd64", - out = "defs_impl_amd64.go", - package = "ring0", - template = ":defs_amd64", -) - -go_template_instance( - name = "defs_impl_arm64", - out = "defs_impl_arm64.go", - package = "ring0", - template = ":defs_arm64", ) arch_genrule( - name = "entry_impl_amd64", - srcs = ["entry_amd64.s"], - outs = ["entry_impl_amd64.s"], - cmd = "(echo -e '// build +amd64\\n' && QEMU $(location //pkg/ring0/gen_offsets) && cat $(location entry_amd64.s)) > $@", - tools = ["//pkg/ring0/gen_offsets"], -) - -arch_genrule( - name = "entry_impl_arm64", - srcs = ["entry_arm64.s"], - outs = ["entry_impl_arm64.s"], - cmd = "(echo -e '// build +arm64\\n' && QEMU $(location //pkg/ring0/gen_offsets) && cat $(location entry_arm64.s)) > $@", - tools = ["//pkg/ring0/gen_offsets"], + name = "entry_impl_arch", + src = ":entry_impl", + template = "entry_impl_%s.s", ) go_library( name = "ring0", srcs = [ - "defs_impl_amd64.go", - "defs_impl_arm64.go", + "aarch64.go", + "defs.go", + "defs_amd64.go", + "defs_arm64.go", "entry_amd64.go", "entry_arm64.go", - "entry_impl_amd64.s", - "entry_impl_arm64.s", "kernel.go", "kernel_amd64.go", "kernel_arm64.go", @@ -73,15 +54,15 @@ go_library( "lib_arm64.go", "lib_arm64.s", "ring0.go", + "x86.go", + ":entry_impl_arch", ], visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/cpuid", "//pkg/hostarch", "//pkg/ring0/pagetables", - "//pkg/safecopy", "//pkg/sentry/arch", "//pkg/sentry/arch/fpu", - "//pkg/sync", ], ) diff --git a/pkg/ring0/defs_arm64.go b/pkg/ring0/defs_arm64.go index bce925b5d..2a296c588 100644 --- a/pkg/ring0/defs_arm64.go +++ b/pkg/ring0/defs_arm64.go @@ -137,6 +137,3 @@ type SwitchArchOpts struct { // KernelASID indicates that the kernel ASID to be used on return, KernelASID uint16 } - -func init() { -} diff --git a/pkg/ring0/entry_amd64.s b/pkg/ring0/entry_amd64.s index 3bcaf89ad..f2c3608d4 100644 --- a/pkg/ring0/entry_amd64.s +++ b/pkg/ring0/entry_amd64.s @@ -15,9 +15,77 @@ #include "funcdata.h" #include "textflag.h" -// NB: Offsets are programmatically generated (see BUILD). -// -// This file is concatenated with the definitions. +// CPU offsets. +#define CPU_REGISTERS {{ .CPU.registers.Offset }} +#define CPU_FPU_STATE {{ .CPU.floatingPointState.Offset }} +#define CPU_ERROR_CODE ({{ .CPU.CPUArchState.Offset }}+{{ .CPUArchState.errorCode.Offset }}) +#define CPU_ERROR_TYPE ({{ .CPU.CPUArchState.Offset }}+{{ .CPUArchState.errorType.Offset }}) +#define CPU_ENTRY ({{ .CPU.CPUArchState.Offset }}+{{ .CPUArchState.kernelEntry.Offset }}) +#define CPU_HAS_XSAVE ({{ .CPU.CPUArchState.Offset }}+{{ .CPUArchState.hasXSAVE.Offset }}) +#define CPU_HAS_XSAVEOPT ({{ .CPU.CPUArchState.Offset }}+{{ .CPUArchState.hasXSAVEOPT.Offset }}) + +{{ with .kernelEntry }} +#define ENTRY_SCRATCH0 {{ .scratch0.Offset }} +#define ENTRY_STACK_TOP {{ .stackTop.Offset }} +#define ENTRY_CPU_SELF {{ .cpuSelf.Offset }} +#define ENTRY_KERNEL_CR3 {{ .kernelCR3.Offset }} +{{ end }} + +// Bits. +#define _RFLAGS_IF {{ ._RFLAGS_IF.Value }} +#define _RFLAGS_IOPL0 {{ ._RFLAGS_IOPL0.Value }} +#define _KERNEL_FLAGS {{ .KernelFlagsSet.Value }} + +// Vectors. +#define DivideByZero {{ .DivideByZero.Value }} +#define Debug {{ .Debug.Value }} +#define NMI {{ .NMI.Value }} +#define Breakpoint {{ .Breakpoint.Value }} +#define Overflow {{ .Overflow.Value }} +#define BoundRangeExceeded {{ .BoundRangeExceeded.Value }} +#define InvalidOpcode {{ .InvalidOpcode.Value }} +#define DeviceNotAvailable {{ .DeviceNotAvailable.Value }} +#define DoubleFault {{ .DoubleFault.Value }} +#define CoprocessorSegmentOverrun {{ .CoprocessorSegmentOverrun.Value }} +#define InvalidTSS {{ .InvalidTSS.Value }} +#define SegmentNotPresent {{ .SegmentNotPresent.Value }} +#define StackSegmentFault {{ .StackSegmentFault.Value }} +#define GeneralProtectionFault {{ .GeneralProtectionFault.Value }} +#define PageFault {{ .PageFault.Value }} +#define X87FloatingPointException {{ .X87FloatingPointException.Value }} +#define AlignmentCheck {{ .AlignmentCheck.Value }} +#define MachineCheck {{ .MachineCheck.Value }} +#define SIMDFloatingPointException {{ .SIMDFloatingPointException.Value }} +#define VirtualizationException {{ .VirtualizationException.Value }} +#define SecurityException {{ .SecurityException.Value }} +#define SyscallInt80 {{ .SyscallInt80.Value }} +#define Syscall {{ .Syscall.Value }} + +{{ with .import.linux.PtraceRegs }} +#define PTRACE_R15 {{ .R15.Offset }} +#define PTRACE_R14 {{ .R14.Offset }} +#define PTRACE_R13 {{ .R13.Offset }} +#define PTRACE_R12 {{ .R12.Offset }} +#define PTRACE_RBP {{ .Rbp.Offset }} +#define PTRACE_RBX {{ .Rbx.Offset }} +#define PTRACE_R11 {{ .R11.Offset }} +#define PTRACE_R10 {{ .R10.Offset }} +#define PTRACE_R9 {{ .R9.Offset }} +#define PTRACE_R8 {{ .R8.Offset }} +#define PTRACE_RAX {{ .Rax.Offset }} +#define PTRACE_RCX {{ .Rcx.Offset }} +#define PTRACE_RDX {{ .Rdx.Offset }} +#define PTRACE_RSI {{ .Rsi.Offset }} +#define PTRACE_RDI {{ .Rdi.Offset }} +#define PTRACE_ORIGRAX {{ .Orig_rax.Offset }} +#define PTRACE_RIP {{ .Rip.Offset }} +#define PTRACE_CS {{ .Cs.Offset }} +#define PTRACE_FLAGS {{ .Eflags.Offset }} +#define PTRACE_RSP {{ .Rsp.Offset }} +#define PTRACE_SS {{ .Ss.Offset }} +#define PTRACE_FS_BASE {{ .Fs_base.Offset }} +#define PTRACE_GS_BASE {{ .Gs_base.Offset }} +{{ end }} // Saves a register set. // diff --git a/pkg/ring0/entry_arm64.s b/pkg/ring0/entry_arm64.s index f801b8e11..fe2583569 100644 --- a/pkg/ring0/entry_arm64.s +++ b/pkg/ring0/entry_arm64.s @@ -15,9 +15,98 @@ #include "funcdata.h" #include "textflag.h" -// NB: Offsets are programatically generated (see BUILD). -// -// This file is concatenated with the definitions. +{{ with .CPU }} +#define CPU_SELF {{ .self.Offset }} +#define CPU_REGISTERS {{ .registers.Offset }} +{{ end }} +{{ with .CPUArchState }} +#define CPU_STACK_TOP ({{ .stack.Offset }} + {{ .stack.Size }}) +#define CPU_ERROR_CODE {{ .errorCode.Offset }} +#define CPU_ERROR_TYPE {{ .errorType.Offset }} +#define CPU_FAULT_ADDR {{ .faultAddr.Offset }} +#define CPU_FPSTATE_EL0 {{ .el0Fp.Offset }} +#define CPU_TTBR0_KVM {{ .ttbr0Kvm.Offset }} +#define CPU_TTBR0_APP {{ .ttbr0App.Offset }} +#define CPU_VECTOR_CODE {{ .vecCode.Offset }} +#define CPU_APP_ADDR {{ .appAddr.Offset }} +#define CPU_LAZY_VFP {{ .lazyVFP.Offset }} +#define CPU_APP_ASID {{ .appASID.Offset }} +{{ end }} + +// Bits. +#define _KERNEL_FLAGS {{ .KernelFlagsSet.Value }} + +// Vectors. +#define El1Sync {{ .El1Sync.Value }} +#define El1Irq {{ .El1Irq.Value }} +#define El1Fiq {{ .El1Fiq.Value }} +#define El1Err {{ .El1Err.Value }} +#define El0Sync {{ .El0Sync.Value }} +#define El0Irq {{ .El0Irq.Value }} +#define El0Fiq {{ .El0Fiq.Value }} +#define El0Err {{ .El0Err.Value }} +#define El1SyncDa {{ .El1SyncDa.Value }} +#define El1SyncIa {{ .El1SyncIa.Value }} +#define El1SyncSpPc {{ .El1SyncSpPc.Value }} +#define El1SyncUndef {{ .El1SyncUndef.Value }} +#define El1SyncDbg {{ .El1SyncDbg.Value }} +#define El1SyncInv {{ .El1SyncInv.Value }} +#define El0SyncSVC {{ .El0SyncSVC.Value }} +#define El0SyncDa {{ .El0SyncDa.Value }} +#define El0SyncIa {{ .El0SyncIa.Value }} +#define El0SyncFpsimdAcc {{ .El0SyncFpsimdAcc.Value }} +#define El0SyncSveAcc {{ .El0SyncSveAcc.Value }} +#define El0SyncFpsimdExc {{ .El0SyncFpsimdExc.Value }} +#define El0SyncSys {{ .El0SyncSys.Value }} +#define El0SyncSpPc {{ .El0SyncSpPc.Value }} +#define El0SyncUndef {{ .El0SyncUndef.Value }} +#define El0SyncDbg {{ .El0SyncDbg.Value }} +#define El0SyncWfx {{ .El0SyncWfx.Value }} +#define El0SyncInv {{ .El0SyncInv.Value }} +#define El0ErrNMI {{ .El0ErrNMI.Value }} +#define PageFault {{ .PageFault.Value }} +#define Syscall {{ .Syscall.Value }} +#define VirtualizationException {{ .VirtualizationException.Value }} + +{{ with .import.linux.PtraceRegs }} +#define PTRACE_R0 ({{ .Regs.Offset }} + 0*8) +#define PTRACE_R1 ({{ .Regs.Offset }} + 1*8) +#define PTRACE_R2 ({{ .Regs.Offset }} + 2*8) +#define PTRACE_R3 ({{ .Regs.Offset }} + 3*8) +#define PTRACE_R4 ({{ .Regs.Offset }} + 4*8) +#define PTRACE_R5 ({{ .Regs.Offset }} + 5*8) +#define PTRACE_R6 ({{ .Regs.Offset }} + 6*8) +#define PTRACE_R7 ({{ .Regs.Offset }} + 7*8) +#define PTRACE_R8 ({{ .Regs.Offset }} + 8*8) +#define PTRACE_R9 ({{ .Regs.Offset }} + 9*8) +#define PTRACE_R10 ({{ .Regs.Offset }} + 10*8) +#define PTRACE_R11 ({{ .Regs.Offset }} + 11*8) +#define PTRACE_R12 ({{ .Regs.Offset }} + 12*8) +#define PTRACE_R13 ({{ .Regs.Offset }} + 13*8) +#define PTRACE_R14 ({{ .Regs.Offset }} + 14*8) +#define PTRACE_R15 ({{ .Regs.Offset }} + 15*8) +#define PTRACE_R16 ({{ .Regs.Offset }} + 16*8) +#define PTRACE_R17 ({{ .Regs.Offset }} + 17*8) +#define PTRACE_R18 ({{ .Regs.Offset }} + 18*8) +#define PTRACE_R19 ({{ .Regs.Offset }} + 19*8) +#define PTRACE_R20 ({{ .Regs.Offset }} + 20*8) +#define PTRACE_R21 ({{ .Regs.Offset }} + 21*8) +#define PTRACE_R22 ({{ .Regs.Offset }} + 22*8) +#define PTRACE_R23 ({{ .Regs.Offset }} + 23*8) +#define PTRACE_R24 ({{ .Regs.Offset }} + 24*8) +#define PTRACE_R25 ({{ .Regs.Offset }} + 25*8) +#define PTRACE_R26 ({{ .Regs.Offset }} + 26*8) +#define PTRACE_R27 ({{ .Regs.Offset }} + 27*8) +#define PTRACE_R28 ({{ .Regs.Offset }} + 28*8) +#define PTRACE_R29 ({{ .Regs.Offset }} + 29*8) +#define PTRACE_R30 ({{ .Regs.Offset }} + 30*8) +#define PTRACE_SP {{ .Sp.Offset }} +#define PTRACE_PC {{ .Pc.Offset }} +#define PTRACE_PSTATE {{ .Pstate.Offset }} +{{ end }} +{{ with .import.arch.Registers }} +#define PTRACE_TLS {{ .TPIDR_EL0.Offset }} +{{ end }} // Saves a register set. // diff --git a/pkg/ring0/gen_offsets/main.go b/pkg/ring0/entry_template.go similarity index 83% rename from pkg/ring0/gen_offsets/main.go rename to pkg/ring0/entry_template.go index a4927da2f..c51abf586 100644 --- a/pkg/ring0/gen_offsets/main.go +++ b/pkg/ring0/entry_template.go @@ -12,13 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Binary gen_offsets is a helper for generating offset headers. -package main +package ring0 import ( - "os" + // Used for template generation. + _ "gvisor.dev/gvisor/pkg/abi/linux" ) - -func main() { - Emit(os.Stdout) -} diff --git a/pkg/ring0/gen_offsets/BUILD b/pkg/ring0/gen_offsets/BUILD deleted file mode 100644 index 9ea8f9a4f..000000000 --- a/pkg/ring0/gen_offsets/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -load("//tools:defs.bzl", "go_binary") -load("//tools/go_generics:defs.bzl", "go_template_instance") - -package(licenses = ["notice"]) - -go_template_instance( - name = "defs_impl_arm64", - out = "defs_impl_arm64.go", - package = "main", - template = "//pkg/ring0:defs_arm64", -) - -go_template_instance( - name = "defs_impl_amd64", - out = "defs_impl_amd64.go", - package = "main", - template = "//pkg/ring0:defs_amd64", -) - -go_binary( - name = "gen_offsets", - srcs = [ - "defs_impl_amd64.go", - "defs_impl_arm64.go", - "main.go", - ], - # Use the libc malloc to avoid any extra dependencies. This is required to - # pass the sentry deps test. - system_malloc = True, - visibility = [ - "//pkg/ring0:__pkg__", - "//pkg/sentry/platform/kvm:__pkg__", - ], - deps = [ - "//pkg/cpuid", - "//pkg/hostarch", - "//pkg/ring0/pagetables", - "//pkg/sentry/arch", - "//pkg/sentry/arch/fpu", - ], -) diff --git a/pkg/ring0/offsets_amd64.go b/pkg/ring0/offsets_amd64.go deleted file mode 100644 index 38fe27c35..000000000 --- a/pkg/ring0/offsets_amd64.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build amd64 -// +build amd64 - -package ring0 - -import ( - "fmt" - "io" - "reflect" - - "gvisor.dev/gvisor/pkg/sentry/arch" -) - -// Emit prints architecture-specific offsets. -func Emit(w io.Writer) { - fmt.Fprintf(w, "// Automatically generated, do not edit.\n") - - c := &CPU{} - fmt.Fprintf(w, "\n// CPU offsets.\n") - fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ENTRY 0x%02x\n", reflect.ValueOf(&c.kernelEntry).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_HAS_XSAVE 0x%02x\n", reflect.ValueOf(&c.hasXSAVE).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_HAS_XSAVEOPT 0x%02x\n", reflect.ValueOf(&c.hasXSAVEOPT).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_FPU_STATE 0x%02x\n", reflect.ValueOf(&c.floatingPointState).Pointer()-reflect.ValueOf(c).Pointer()) - - e := &kernelEntry{} - fmt.Fprintf(w, "\n// CPU entry offsets.\n") - fmt.Fprintf(w, "#define ENTRY_SCRATCH0 0x%02x\n", reflect.ValueOf(&e.scratch0).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_STACK_TOP 0x%02x\n", reflect.ValueOf(&e.stackTop).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_CPU_SELF 0x%02x\n", reflect.ValueOf(&e.cpuSelf).Pointer()-reflect.ValueOf(e).Pointer()) - fmt.Fprintf(w, "#define ENTRY_KERNEL_CR3 0x%02x\n", reflect.ValueOf(&e.kernelCR3).Pointer()-reflect.ValueOf(e).Pointer()) - - fmt.Fprintf(w, "\n// Bits.\n") - fmt.Fprintf(w, "#define _RFLAGS_IF 0x%02x\n", _RFLAGS_IF) - fmt.Fprintf(w, "#define _RFLAGS_IOPL0 0x%02x\n", _RFLAGS_IOPL0) - fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet) - - fmt.Fprintf(w, "\n// Vectors.\n") - fmt.Fprintf(w, "#define DivideByZero 0x%02x\n", DivideByZero) - fmt.Fprintf(w, "#define Debug 0x%02x\n", Debug) - fmt.Fprintf(w, "#define NMI 0x%02x\n", NMI) - fmt.Fprintf(w, "#define Breakpoint 0x%02x\n", Breakpoint) - fmt.Fprintf(w, "#define Overflow 0x%02x\n", Overflow) - fmt.Fprintf(w, "#define BoundRangeExceeded 0x%02x\n", BoundRangeExceeded) - fmt.Fprintf(w, "#define InvalidOpcode 0x%02x\n", InvalidOpcode) - fmt.Fprintf(w, "#define DeviceNotAvailable 0x%02x\n", DeviceNotAvailable) - fmt.Fprintf(w, "#define DoubleFault 0x%02x\n", DoubleFault) - fmt.Fprintf(w, "#define CoprocessorSegmentOverrun 0x%02x\n", CoprocessorSegmentOverrun) - fmt.Fprintf(w, "#define InvalidTSS 0x%02x\n", InvalidTSS) - fmt.Fprintf(w, "#define SegmentNotPresent 0x%02x\n", SegmentNotPresent) - fmt.Fprintf(w, "#define StackSegmentFault 0x%02x\n", StackSegmentFault) - fmt.Fprintf(w, "#define GeneralProtectionFault 0x%02x\n", GeneralProtectionFault) - fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault) - fmt.Fprintf(w, "#define X87FloatingPointException 0x%02x\n", X87FloatingPointException) - fmt.Fprintf(w, "#define AlignmentCheck 0x%02x\n", AlignmentCheck) - fmt.Fprintf(w, "#define MachineCheck 0x%02x\n", MachineCheck) - fmt.Fprintf(w, "#define SIMDFloatingPointException 0x%02x\n", SIMDFloatingPointException) - fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException) - fmt.Fprintf(w, "#define SecurityException 0x%02x\n", SecurityException) - fmt.Fprintf(w, "#define SyscallInt80 0x%02x\n", SyscallInt80) - fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall) - - p := &arch.Registers{} - fmt.Fprintf(w, "\n// Ptrace registers.\n") - fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.R15).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.R14).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.R13).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.R12).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RBP 0x%02x\n", reflect.ValueOf(&p.Rbp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RBX 0x%02x\n", reflect.ValueOf(&p.Rbx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.R11).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.R10).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.R9).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.R8).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RAX 0x%02x\n", reflect.ValueOf(&p.Rax).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RCX 0x%02x\n", reflect.ValueOf(&p.Rcx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RDX 0x%02x\n", reflect.ValueOf(&p.Rdx).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RSI 0x%02x\n", reflect.ValueOf(&p.Rsi).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RDI 0x%02x\n", reflect.ValueOf(&p.Rdi).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_ORIGRAX 0x%02x\n", reflect.ValueOf(&p.Orig_rax).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RIP 0x%02x\n", reflect.ValueOf(&p.Rip).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_CS 0x%02x\n", reflect.ValueOf(&p.Cs).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_FLAGS 0x%02x\n", reflect.ValueOf(&p.Eflags).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_RSP 0x%02x\n", reflect.ValueOf(&p.Rsp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_SS 0x%02x\n", reflect.ValueOf(&p.Ss).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_FS_BASE 0x%02x\n", reflect.ValueOf(&p.Fs_base).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_GS_BASE 0x%02x\n", reflect.ValueOf(&p.Gs_base).Pointer()-reflect.ValueOf(p).Pointer()) -} diff --git a/pkg/ring0/offsets_arm64.go b/pkg/ring0/offsets_arm64.go deleted file mode 100644 index 60b2c4074..000000000 --- a/pkg/ring0/offsets_arm64.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build arm64 -// +build arm64 - -package ring0 - -import ( - "fmt" - "io" - "reflect" - - "gvisor.dev/gvisor/pkg/sentry/arch" -) - -// Emit prints architecture-specific offsets. -func Emit(w io.Writer) { - fmt.Fprintf(w, "// Automatically generated, do not edit.\n") - - c := &CPU{} - fmt.Fprintf(w, "\n// CPU offsets.\n") - fmt.Fprintf(w, "#define CPU_SELF 0x%02x\n", reflect.ValueOf(&c.self).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_REGISTERS 0x%02x\n", reflect.ValueOf(&c.registers).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_STACK_TOP 0x%02x\n", reflect.ValueOf(&c.stack[0]).Pointer()-reflect.ValueOf(c).Pointer()+uintptr(len(c.stack))) - fmt.Fprintf(w, "#define CPU_ERROR_CODE 0x%02x\n", reflect.ValueOf(&c.errorCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_ERROR_TYPE 0x%02x\n", reflect.ValueOf(&c.errorType).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_FAULT_ADDR 0x%02x\n", reflect.ValueOf(&c.faultAddr).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_FPSTATE_EL0 0x%02x\n", reflect.ValueOf(&c.el0Fp).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_TTBR0_KVM 0x%02x\n", reflect.ValueOf(&c.ttbr0Kvm).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_TTBR0_APP 0x%02x\n", reflect.ValueOf(&c.ttbr0App).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_VECTOR_CODE 0x%02x\n", reflect.ValueOf(&c.vecCode).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_APP_ADDR 0x%02x\n", reflect.ValueOf(&c.appAddr).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_LAZY_VFP 0x%02x\n", reflect.ValueOf(&c.lazyVFP).Pointer()-reflect.ValueOf(c).Pointer()) - fmt.Fprintf(w, "#define CPU_APP_ASID 0x%02x\n", reflect.ValueOf(&c.appASID).Pointer()-reflect.ValueOf(c).Pointer()) - - fmt.Fprintf(w, "\n// Bits.\n") - fmt.Fprintf(w, "#define _KERNEL_FLAGS 0x%02x\n", KernelFlagsSet) - - fmt.Fprintf(w, "\n// Vectors.\n") - - fmt.Fprintf(w, "#define El1Sync 0x%02x\n", El1Sync) - fmt.Fprintf(w, "#define El1Irq 0x%02x\n", El1Irq) - fmt.Fprintf(w, "#define El1Fiq 0x%02x\n", El1Fiq) - fmt.Fprintf(w, "#define El1Err 0x%02x\n", El1Err) - - fmt.Fprintf(w, "#define El0Sync 0x%02x\n", El0Sync) - fmt.Fprintf(w, "#define El0Irq 0x%02x\n", El0Irq) - fmt.Fprintf(w, "#define El0Fiq 0x%02x\n", El0Fiq) - fmt.Fprintf(w, "#define El0Err 0x%02x\n", El0Err) - - fmt.Fprintf(w, "#define El1SyncDa 0x%02x\n", El1SyncDa) - fmt.Fprintf(w, "#define El1SyncIa 0x%02x\n", El1SyncIa) - fmt.Fprintf(w, "#define El1SyncSpPc 0x%02x\n", El1SyncSpPc) - fmt.Fprintf(w, "#define El1SyncUndef 0x%02x\n", El1SyncUndef) - fmt.Fprintf(w, "#define El1SyncDbg 0x%02x\n", El1SyncDbg) - fmt.Fprintf(w, "#define El1SyncInv 0x%02x\n", El1SyncInv) - - fmt.Fprintf(w, "#define El0SyncSVC 0x%02x\n", El0SyncSVC) - fmt.Fprintf(w, "#define El0SyncDa 0x%02x\n", El0SyncDa) - fmt.Fprintf(w, "#define El0SyncIa 0x%02x\n", El0SyncIa) - fmt.Fprintf(w, "#define El0SyncFpsimdAcc 0x%02x\n", El0SyncFpsimdAcc) - fmt.Fprintf(w, "#define El0SyncSveAcc 0x%02x\n", El0SyncSveAcc) - fmt.Fprintf(w, "#define El0SyncFpsimdExc 0x%02x\n", El0SyncFpsimdExc) - fmt.Fprintf(w, "#define El0SyncSys 0x%02x\n", El0SyncSys) - fmt.Fprintf(w, "#define El0SyncSpPc 0x%02x\n", El0SyncSpPc) - fmt.Fprintf(w, "#define El0SyncUndef 0x%02x\n", El0SyncUndef) - fmt.Fprintf(w, "#define El0SyncDbg 0x%02x\n", El0SyncDbg) - fmt.Fprintf(w, "#define El0SyncWfx 0x%02x\n", El0SyncWfx) - fmt.Fprintf(w, "#define El0SyncInv 0x%02x\n", El0SyncInv) - - fmt.Fprintf(w, "#define El0ErrNMI 0x%02x\n", El0ErrNMI) - - fmt.Fprintf(w, "#define PageFault 0x%02x\n", PageFault) - fmt.Fprintf(w, "#define Syscall 0x%02x\n", Syscall) - fmt.Fprintf(w, "#define VirtualizationException 0x%02x\n", VirtualizationException) - - p := &arch.Registers{} - fmt.Fprintf(w, "\n// Ptrace registers.\n") - fmt.Fprintf(w, "#define PTRACE_R0 0x%02x\n", reflect.ValueOf(&p.Regs[0]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R1 0x%02x\n", reflect.ValueOf(&p.Regs[1]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R2 0x%02x\n", reflect.ValueOf(&p.Regs[2]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R3 0x%02x\n", reflect.ValueOf(&p.Regs[3]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R4 0x%02x\n", reflect.ValueOf(&p.Regs[4]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R5 0x%02x\n", reflect.ValueOf(&p.Regs[5]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R6 0x%02x\n", reflect.ValueOf(&p.Regs[6]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R7 0x%02x\n", reflect.ValueOf(&p.Regs[7]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R8 0x%02x\n", reflect.ValueOf(&p.Regs[8]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R9 0x%02x\n", reflect.ValueOf(&p.Regs[9]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R10 0x%02x\n", reflect.ValueOf(&p.Regs[10]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R11 0x%02x\n", reflect.ValueOf(&p.Regs[11]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R12 0x%02x\n", reflect.ValueOf(&p.Regs[12]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R13 0x%02x\n", reflect.ValueOf(&p.Regs[13]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R14 0x%02x\n", reflect.ValueOf(&p.Regs[14]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R15 0x%02x\n", reflect.ValueOf(&p.Regs[15]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R16 0x%02x\n", reflect.ValueOf(&p.Regs[16]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R17 0x%02x\n", reflect.ValueOf(&p.Regs[17]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R18 0x%02x\n", reflect.ValueOf(&p.Regs[18]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R19 0x%02x\n", reflect.ValueOf(&p.Regs[19]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R20 0x%02x\n", reflect.ValueOf(&p.Regs[20]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R21 0x%02x\n", reflect.ValueOf(&p.Regs[21]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R22 0x%02x\n", reflect.ValueOf(&p.Regs[22]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R23 0x%02x\n", reflect.ValueOf(&p.Regs[23]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R24 0x%02x\n", reflect.ValueOf(&p.Regs[24]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R25 0x%02x\n", reflect.ValueOf(&p.Regs[25]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R26 0x%02x\n", reflect.ValueOf(&p.Regs[26]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R27 0x%02x\n", reflect.ValueOf(&p.Regs[27]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R28 0x%02x\n", reflect.ValueOf(&p.Regs[28]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R29 0x%02x\n", reflect.ValueOf(&p.Regs[29]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_R30 0x%02x\n", reflect.ValueOf(&p.Regs[30]).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_SP 0x%02x\n", reflect.ValueOf(&p.Sp).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_PC 0x%02x\n", reflect.ValueOf(&p.Pc).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_PSTATE 0x%02x\n", reflect.ValueOf(&p.Pstate).Pointer()-reflect.ValueOf(p).Pointer()) - fmt.Fprintf(w, "#define PTRACE_TLS 0x%02x\n", reflect.ValueOf(&p.TPIDR_EL0).Pointer()-reflect.ValueOf(p).Pointer()) -} diff --git a/pkg/sentry/platform/kvm/BUILD b/pkg/sentry/platform/kvm/BUILD index 834d72408..abdb6c383 100644 --- a/pkg/sentry/platform/kvm/BUILD +++ b/pkg/sentry/platform/kvm/BUILD @@ -1,5 +1,6 @@ -load("//tools:defs.bzl", "go_library", "go_test") +load("//tools:defs.bzl", "arch_genrule", "go_library", "go_test", "select_arch") load("//tools/go_generics:defs.bzl", "go_template_instance") +load("//tools/nogo:defs.bzl", "nogo_facts") package(licenses = ["notice"]) @@ -14,6 +15,37 @@ go_template_instance( }, ) +nogo_facts( + name = "bluepill_impl", + srcs = [ + "//pkg/ring0:aarch64.go", + "//pkg/ring0:defs.go", + "//pkg/ring0:defs_amd64.go", + "//pkg/ring0:defs_arm64.go", + "//pkg/ring0:entry_template.go", + "//pkg/ring0:x86.go", + ], + output = "bluepill_impl.s", + template = select_arch( + amd64 = "bluepill_amd64.s", + arm64 = "bluepill_arm64.s", + ), + deps = [ + "//pkg/abi/linux", + "//pkg/cpuid", + "//pkg/hostarch", + "//pkg/ring0/pagetables", + "//pkg/sentry/arch", + "//pkg/sentry/arch/fpu", + ], +) + +arch_genrule( + name = "bluepill_impl_arch", + src = ":bluepill_impl", + template = "bluepill_impl_%s.s", +) + go_library( name = "kvm", srcs = [ @@ -26,10 +58,8 @@ go_library( "bluepill_amd64.go", "bluepill_amd64_unsafe.go", "bluepill_arm64.go", - "bluepill_arm64.s", "bluepill_arm64_unsafe.go", "bluepill_fault.go", - "bluepill_impl_amd64.s", "bluepill_unsafe.go", "context.go", "filters_amd64.go", @@ -51,6 +81,7 @@ go_library( "physical_map_amd64.go", "physical_map_arm64.go", "virtual_map.go", + ":bluepill_impl_arch", ], visibility = ["//pkg/sentry:internal"], deps = [ @@ -113,11 +144,3 @@ go_test( "@org_golang_x_sys//unix:go_default_library", ], ) - -genrule( - name = "bluepill_impl_amd64", - srcs = ["bluepill_amd64.s"], - outs = ["bluepill_impl_amd64.s"], - cmd = "(echo -e '// build +amd64\\n' && $(location //pkg/ring0/gen_offsets) && cat $(SRCS)) > $@", - tools = ["//pkg/ring0/gen_offsets"], -) diff --git a/pkg/sentry/platform/kvm/bluepill_amd64.s b/pkg/sentry/platform/kvm/bluepill_amd64.s index 5d8358f64..71a3ca560 100644 --- a/pkg/sentry/platform/kvm/bluepill_amd64.s +++ b/pkg/sentry/platform/kvm/bluepill_amd64.s @@ -19,6 +19,11 @@ // This is guaranteed to be zero. #define VCPU_CPU 0x0 +// ENTRY_CPU_SELF is the location of the CPU in the entry struct. +// +// This is sourced from ring0. +#define ENTRY_CPU_SELF {{ .kernelEntry.cpuSelf.Offset }} + // Context offsets. // // Only limited use of the context is done in the assembly stub below, most is @@ -32,6 +37,7 @@ // This is checked as the source of the fault. #define CLI $0xfa +// System call definitions. #define SYS_MMAP 9 // See bluepill.go. diff --git a/runsc/flag/flag.go b/runsc/flag/flag.go index 6b25da904..1c4df5b98 100644 --- a/runsc/flag/flag.go +++ b/runsc/flag/flag.go @@ -30,9 +30,11 @@ var ( Bool = flag.Bool CommandLine = flag.CommandLine Int = flag.Int + Int64 = flag.Int64 NewFlagSet = flag.NewFlagSet Parse = flag.Parse String = flag.String + StringVar = flag.StringVar Uint = flag.Uint Var = flag.Var ) diff --git a/tools/bazel_gazelle_generate.patch b/tools/bazel_gazelle_generate.patch deleted file mode 100644 index fd1e1bda6..000000000 --- a/tools/bazel_gazelle_generate.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/language/go/generate.go b/language/go/generate.go -index 2892948..feb4ad6 100644 ---- a/language/go/generate.go -+++ b/language/go/generate.go -@@ -691,6 +691,10 @@ func (g *generator) setImportAttrs(r *rule.Rule, importPath string) { - } - - func (g *generator) commonVisibility(importPath string) []string { -+ if importPath == "golang.org/x/tools/go/analysis/internal/facts" { -+ // Imported by nogo main. We add a visibility exception. -+ return []string{"//visibility:public"} -+ } - // If the Bazel package name (rel) contains "internal", add visibility for - // subpackages of the parent. - // If the import path contains "internal" but rel does not, this is diff --git a/tools/bazeldefs/defs.bzl b/tools/bazeldefs/defs.bzl index b1cb4796d..71fd9776f 100644 --- a/tools/bazeldefs/defs.bzl +++ b/tools/bazeldefs/defs.bzl @@ -33,6 +33,28 @@ def select_system(linux = ["__linux__"], darwin = [], **kwargs): "//conditions:default": linux, }) +arch_config = [ + "//command_line_option:cpu", + "//command_line_option:crosstool_top", + "//command_line_option:platforms", +] + +def arm64_config(settings, attr): + return { + "//command_line_option:cpu": "aarch64", + "//command_line_option:crosstool_top": "@crosstool//:toolchains", + "//command_line_option:platforms": "@io_bazel_rules_go//go/toolchain:linux_arm64", + } + +def amd64_config(settings, attr): + return { + "//command_line_option:cpu": "k8", + "//command_line_option:crosstool_top": "@crosstool//:toolchains", + "//command_line_option:platforms": "@io_bazel_rules_go//go/toolchain:linux_amd64", + } + +transition_allowlist = "@bazel_tools//tools/allowlists/function_transition_allowlist" + def default_installer(): return None @@ -41,44 +63,3 @@ def default_net_util(): def coreutil(): return [] # Nothing needed. - -def select_native_vs_cross(native = [], amd64 = [], arm64 = [], cross = []): - values = { - "//tools/bazeldefs:linux_arm64_cross": arm64 + cross, - "//tools/bazeldefs:linux_amd64_cross": amd64 + cross, - "//conditions:default": native, - } - return select(values) - -def arch_genrule(name, srcs, outs, cmd, tools): - """Runs a gen command on the target architecture. - - If the target architecture isn't match the host architecture, it will build - a command for the target architecture and run it via qemu. - - The native genrule runs the command on the host architecture. - - Args: - name: name of generated target. - srcs: A list of inputs for this rule. - cmd: The command to run. It has to contain " QEMU " before executed binaries. - outs: A list of files generated by this rule. - tools: A list of tool dependencies for this rule. - """ - qemu_arm64 = "qemu-aarch64-static" - qemu_amd64 = "qemu-x86_64-static" - srcs = select_native_vs_cross( - cross = srcs + tools, - native = srcs, - ) - tools = select_native_vs_cross( - cross = [], - native = tools, - ) - cmd = select_native_vs_cross( - arm64 = cmd.replace("QEMU", qemu_arm64), - amd64 = cmd.replace("QEMU", qemu_amd64), - native = cmd.replace("QEMU", ""), - cross = "", - ) - native.genrule(name = name, srcs = srcs, outs = outs, cmd = cmd, tools = tools) diff --git a/tools/bazeldefs/go.bzl b/tools/bazeldefs/go.bzl index af8694626..0c360df1f 100644 --- a/tools/bazeldefs/go.bzl +++ b/tools/bazeldefs/go.bzl @@ -135,17 +135,13 @@ def go_context(ctx, goos = None, goarch = None, std = False): go_ctx = _go_context(ctx) if goos == None: goos = go_ctx.sdk.goos - elif goos != go_ctx.sdk.goos: - fail("Internal GOOS (%s) doesn't match GoSdk GOOS (%s)." % (goos, go_ctx.sdk.goos)) if goarch == None: goarch = go_ctx.sdk.goarch - elif goarch != go_ctx.sdk.goarch: - fail("Internal GOARCH (%s) doesn't match GoSdk GOARCH (%s)." % (goarch, go_ctx.sdk.goarch)) return struct( env = go_ctx.env, go = go_ctx.go, - goarch = go_ctx.sdk.goarch, - goos = go_ctx.sdk.goos, + goarch = goarch, + goos = goos, gotags = go_ctx.tags, nogo_args = [], runfiles = depset([go_ctx.go] + go_ctx.sdk.srcs + go_ctx.sdk.tools + go_ctx.stdlib.libs), diff --git a/tools/checkescape/BUILD b/tools/checkescape/BUILD index 109b5410c..833aa1269 100644 --- a/tools/checkescape/BUILD +++ b/tools/checkescape/BUILD @@ -8,7 +8,7 @@ go_library( nogo = False, visibility = ["//tools/nogo:__subpackages__"], deps = [ - "//tools/nogo/objdump", + "//tools/nogo/flags", "@org_golang_x_tools//go/analysis:go_default_library", "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library", "@org_golang_x_tools//go/ssa:go_default_library", diff --git a/tools/checkescape/checkescape.go b/tools/checkescape/checkescape.go index ddd1212d7..0ddfae17a 100644 --- a/tools/checkescape/checkescape.go +++ b/tools/checkescape/checkescape.go @@ -66,14 +66,17 @@ import ( "go/token" "go/types" "io" + "io/ioutil" "log" + "os" + "os/exec" "path/filepath" "strings" "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/analysis/passes/buildssa" "golang.org/x/tools/go/ssa" - "gvisor.dev/gvisor/tools/nogo/objdump" + "gvisor.dev/gvisor/tools/nogo/flags" ) const ( @@ -177,21 +180,30 @@ type packageEscapeFacts struct { // AFact implements analysis.Fact.AFact. func (*packageEscapeFacts) AFact() {} -// Analyzer includes specific results. -var Analyzer = &analysis.Analyzer{ - Name: "checkescape", - Doc: "escape analysis checks based on +checkescape annotations", - Run: runSelectEscapes, - Requires: []*analysis.Analyzer{buildssa.Analyzer}, - FactTypes: []analysis.Fact{(*packageEscapeFacts)(nil)}, +// objdumpAnalyzer accepts the objdump parameter. +type objdumpAnalyzer struct { + analysis.Analyzer } -// EscapeAnalyzer includes all local escape results. -var EscapeAnalyzer = &analysis.Analyzer{ - Name: "checkescape", - Doc: "complete local escape analysis results (requires Analyzer facts)", - Run: runAllEscapes, - Requires: []*analysis.Analyzer{buildssa.Analyzer}, +// Run implements nogo.binaryAnalyzer.Run. +func (ob *objdumpAnalyzer) Run(pass *analysis.Pass, binary io.Reader) (interface{}, error) { + return run(pass, binary) +} + +// Legacy implements nogo.analyzer.Legacy. +func (ob *objdumpAnalyzer) Legacy() *analysis.Analyzer { + return &ob.Analyzer +} + +// Analyzer includes specific results. +var Analyzer = &objdumpAnalyzer{ + Analyzer: analysis.Analyzer{ + Name: "checkescape", + Doc: "escape analysis checks based on +checkescape annotations", + Run: nil, // Must be invoked via Run above. + Requires: []*analysis.Analyzer{buildssa.Analyzer}, + FactTypes: []analysis.Fact{(*packageEscapeFacts)(nil)}, + }, } // LinePosition is a low-resolution token.Position. @@ -356,7 +368,65 @@ func MergeAll(others []Escapes) (es Escapes) { // // Note that the map uses : because that is all that is // provided in the objdump format. Since this is all local, it is sufficient. -func loadObjdump() (map[string][]string, error) { +func loadObjdump(binary io.Reader) (finalResults map[string][]string, finalErr error) { + // Do we have a binary? If it's missing, then the nil will simply be + // plumbed all the way down here. + if binary == nil { + return nil, fmt.Errorf("no binary provided") + } + + // Construct & start our command. The 'go tool objdump' command + // requires a seekable input passed on the command line. Therefore, we + // may need to generate a temporary file here. + input, ok := binary.(*os.File) + if ok { + // Ensure that the file is seekable and that the offset is + // zero, since we can't control that. + if offset, err := input.Seek(0, os.SEEK_CUR); err != nil || offset != 0 { + ok = false // Not usable. + } + } + if !ok { + // Copy to a temporary path. + f, err := ioutil.TempFile("", "") + if err != nil { + return nil, fmt.Errorf("unable to create temp file: %w", err) + } + // Ensure the file is deleted. + defer os.Remove(f.Name()) + // Populate the file contents. + if _, err := io.Copy(f, binary); err != nil { + return nil, fmt.Errorf("unable to populate temp file: %w", err) + } + // Seek to the beginning. + if _, err := f.Seek(0, os.SEEK_SET); err != nil { + return nil, fmt.Errorf("unable to seek in temp file: %w", err) + } + input = f + } + + // Execute go tool objdump ggiven the input. + cmd := exec.Command(flags.Go, "tool", "objdump", input.Name()) + cmd.Stderr = os.Stderr + pipeOut, pipeErr := cmd.StdoutPipe() + if pipeErr != nil { + return nil, fmt.Errorf("unable to load objdump: %w", pipeErr) + } + if startErr := cmd.Start(); startErr != nil { + return nil, fmt.Errorf("unable to start objdump: %w", startErr) + } + + // Ensure that the command has finished successfully. Note that even if + // we parse the first few lines correctly, and early exit could + // indicate that the dump was incomplete and we could be missed some + // escapes that would have appeared. We need to force failure. + defer func() { + if waitErr := cmd.Wait(); finalErr == nil && waitErr != nil { + // Override the function's return value in this case. + finalErr = fmt.Errorf("error running objdump: %v", waitErr) + } + }() + // Identify calls by address or name. Note that this is also // constructed dynamically below, as we encounted the addresses. // This is because some of the functions (duffzero) may have @@ -389,83 +459,78 @@ func loadObjdump() (map[string][]string, error) { // Build the map. nextFunc := "" // For funcsAllowed. m := make(map[string][]string) - if err := objdump.Load(func(origR io.Reader) error { - r := bufio.NewReader(origR) - NextLine: - for { - line, err := r.ReadString('\n') - if err != nil && err != io.EOF { - return err + r := bufio.NewReader(pipeOut) +NextLine: + for { + line, err := r.ReadString('\n') + if err != nil && err != io.EOF { + return nil, err + } + fields := strings.Fields(line) + + // Is this an "allowed" function definition? + if len(fields) >= 2 && fields[0] == "TEXT" { + nextFunc = strings.TrimSuffix(fields[1], "(SB)") + if _, ok := funcsAllowed[nextFunc]; !ok { + nextFunc = "" // Don't record addresses. } - fields := strings.Fields(line) + } + if nextFunc != "" && len(fields) > 2 { + // Save the given address (in hex form, as it appears). + addrsAllowed[fields[1]] = struct{}{} + } - // Is this an "allowed" function definition? - if len(fields) >= 2 && fields[0] == "TEXT" { - nextFunc = strings.TrimSuffix(fields[1], "(SB)") - if _, ok := funcsAllowed[nextFunc]; !ok { - nextFunc = "" // Don't record addresses. - } + // We recognize lines corresponding to actual code (not the + // symbol name or other metadata) and annotate them if they + // correspond to an explicit CALL instruction. We assume that + // the lack of a CALL for a given line is evidence that escape + // analysis has eliminated an allocation. + // + // Lines look like this (including the first space): + // gohacks_unsafe.go:33 0xa39 488b442408 MOVQ 0x8(SP), AX + if len(fields) >= 5 && line[0] == ' ' { + if !strings.Contains(fields[3], "CALL") { + continue } - if nextFunc != "" && len(fields) > 2 { - // Save the given address (in hex form, as it appears). - addrsAllowed[fields[1]] = struct{}{} + site := fields[0] + target := strings.TrimSuffix(fields[4], "(SB)") + + // Ignore strings containing allowed functions. + if _, ok := funcsAllowed[target]; ok { + continue } - - // We recognize lines corresponding to actual code (not the - // symbol name or other metadata) and annotate them if they - // correspond to an explicit CALL instruction. We assume that - // the lack of a CALL for a given line is evidence that escape - // analysis has eliminated an allocation. - // - // Lines look like this (including the first space): - // gohacks_unsafe.go:33 0xa39 488b442408 MOVQ 0x8(SP), AX - if len(fields) >= 5 && line[0] == ' ' { - if !strings.Contains(fields[3], "CALL") { - continue - } - site := fields[0] - target := strings.TrimSuffix(fields[4], "(SB)") - - // Ignore strings containing allowed functions. - if _, ok := funcsAllowed[target]; ok { - continue - } - if _, ok := addrsAllowed[target]; ok { - continue - } - if len(fields) > 5 { - // This may be a future relocation. Some - // objdump versions describe this differently. - // If it contains any of the functions allowed - // above as a string, we let it go. - softTarget := strings.Join(fields[5:], " ") - for name := range funcsAllowed { - if strings.Contains(softTarget, name) { - continue NextLine - } - } - } - - // Does this exist already? - existing, ok := m[site] - if !ok { - existing = make([]string, 0, 1) - } - for _, other := range existing { - if target == other { + if _, ok := addrsAllowed[target]; ok { + continue + } + if len(fields) > 5 { + // This may be a future relocation. Some + // objdump versions describe this differently. + // If it contains any of the functions allowed + // above as a string, we let it go. + softTarget := strings.Join(fields[5:], " ") + for name := range funcsAllowed { + if strings.Contains(softTarget, name) { continue NextLine } } - existing = append(existing, target) - m[site] = existing // Update. } - if err == io.EOF { - break + + // Does this exist already? + existing, ok := m[site] + if !ok { + existing = make([]string, 0, 1) } + for _, other := range existing { + if target == other { + continue NextLine + } + } + existing = append(existing, target) + m[site] = existing // Update. + } + if err == io.EOF { + break } - return nil - }); err != nil { - return nil, err } // Zap any accidental false positives. @@ -489,16 +554,6 @@ type poser interface { Pos() token.Pos } -// runSelectEscapes runs with only select escapes. -func runSelectEscapes(pass *analysis.Pass) (interface{}, error) { - return run(pass, false) -} - -// runAllEscapes runs with all escapes included. -func runAllEscapes(pass *analysis.Pass) (interface{}, error) { - return run(pass, true) -} - // findReasons extracts reasons from the function. func findReasons(pass *analysis.Pass, fdecl *ast.FuncDecl) ([]EscapeReason, bool, map[EscapeReason]bool) { // Is there a comment? @@ -575,8 +630,8 @@ func findReasons(pass *analysis.Pass, fdecl *ast.FuncDecl) ([]EscapeReason, bool } // run performs the analysis. -func run(pass *analysis.Pass, localEscapes bool) (interface{}, error) { - calls, callsErr := loadObjdump() +func run(pass *analysis.Pass, binary io.Reader) (interface{}, error) { + calls, callsErr := loadObjdump(binary) if callsErr != nil { // Note that if this analysis fails, then we don't actually // fail the analyzer itself. We simply report every possible @@ -794,15 +849,6 @@ func run(pass *analysis.Pass, localEscapes bool) (interface{}, error) { loadFunc(fn) } - if !localEscapes { - // Export all findings for future packages. We only do this in - // non-local escapes mode, and expect to run this analysis - // after the SelectAnalysis. - pass.ExportPackageFact(&packageEscapeFacts{ - Funcs: mergedEscapes, - }) - } - // Scan all functions for violations. for _, f := range pass.Files { // Scan all declarations. @@ -812,18 +858,8 @@ func run(pass *analysis.Pass, localEscapes bool) (interface{}, error) { if !ok { continue } - var ( - reasons []EscapeReason - local bool - testReasons map[EscapeReason]bool - ) - if localEscapes { - // Find all hard escapes. - reasons = hardReasons - } else { - // Find all declared reasons. - reasons, local, testReasons = findReasons(pass, fdecl) - } + // Find all declared reasons. + reasons, local, testReasons := findReasons(pass, fdecl) // Scan for matches. fn := pass.TypesInfo.Defs[fdecl.Name].(*types.Func) diff --git a/tools/checkinfo/BUILD b/tools/checkinfo/BUILD new file mode 100644 index 000000000..5cdc1244a --- /dev/null +++ b/tools/checkinfo/BUILD @@ -0,0 +1,11 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "checkinfo", + srcs = ["checkinfo.go"], + nogo = False, + visibility = ["//tools/nogo:__subpackages__"], + deps = ["@org_golang_x_tools//go/analysis:go_default_library"], +) diff --git a/tools/checkinfo/checkinfo.go b/tools/checkinfo/checkinfo.go new file mode 100644 index 000000000..cc7ea7d0f --- /dev/null +++ b/tools/checkinfo/checkinfo.go @@ -0,0 +1,136 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package checkinfo attaches basic info to types. +package checkinfo + +import ( + "go/types" + + "golang.org/x/tools/go/analysis" +) + +// Analyzer defines the entrypoint. +var Analyzer = &analysis.Analyzer{ + Name: "checkinfo", + Doc: "annotates types with basic information", + Run: run, + FactTypes: []analysis.Fact{ + (*Align)(nil), + (*Offset)(nil), + (*Size)(nil), + (*Value)(nil), + }, +} + +// Align is a fact. +type Align int64 + +// AFact implements analysis.Fact.AFact. +func (*Align) AFact() {} + +// Offset is a fact. +type Offset int64 + +// AFact implements analysis.Fact.AFact. +func (*Offset) AFact() {} + +// Size is a fact. +type Size int64 + +// AFact implements analysis.Fact.AFact. +func (*Size) AFact() {} + +// Value is a trivial fact. +type Value string + +// AFact implements analysis.Fact.AFact. +func (*Value) AFact() {} + +func walkObject(pass *analysis.Pass, obj types.Object) { + switch x := obj.(type) { + case *types.Const: + // Add a special constant value. This is supported for + // constants only, and appears as a "Value" fact. + v := Value(x.Val().ExactString()) + pass.ExportObjectFact(obj, &v) + case *types.PkgName: + // Don't walk to other packages. + case *types.Var: + // Add information as a field. + a := Align(pass.TypesSizes.Alignof(x.Type())) + s := Size(pass.TypesSizes.Sizeof(x.Type())) + pass.ExportObjectFact(obj, &a) + pass.ExportObjectFact(obj, &s) + case *types.TypeName: + // Skip if just an alias, or if not underlying type. If it is + // not an alias, then it must be package-local. + typ := x.Type() + if x.IsAlias() || typ == nil || typ.Underlying() == nil { + break + } + // Add basic information. + a := Align(pass.TypesSizes.Alignof(typ)) + s := Size(pass.TypesSizes.Sizeof(typ)) + pass.ExportObjectFact(obj, &a) + pass.ExportObjectFact(obj, &s) + // Recurse to fields if this is a definition. + if structType, ok := typ.Underlying().(*types.Struct); ok { + fields := make([]*types.Var, 0, structType.NumFields()) + for i := 0; i < structType.NumFields(); i++ { + fieldObj := structType.Field(i) + fields = append(fields, fieldObj) + walkObject(pass, fieldObj) + } + offsets := pass.TypesSizes.Offsetsof(fields) + for i, field := range fields { + pass.ExportObjectFact(field, (*Offset)(&offsets[i])) + } + } + case *types.Func: + // Skip if no underlying type. + if x.Type() == nil { + break + } + // Recurse to all parameters. + sig := x.Type().(*types.Signature) + if recv := sig.Recv(); recv != nil { + walkObject(pass, recv) + } + if params := sig.Params(); params != nil { + for i := 0; i < params.Len(); i++ { + walkObject(pass, params.At(i)) + } + } + if results := sig.Results(); results != nil { + for i := 0; i < results.Len(); i++ { + walkObject(pass, results.At(i)) + } + } + walkScope(pass, x.Scope()) + } +} + +// walkScope recursively resolves a scope. +func walkScope(pass *analysis.Pass, scope *types.Scope) { + for _, name := range scope.Names() { + walkObject(pass, scope.Lookup(name)) + } +} + +func run(pass *analysis.Pass) (interface{}, error) { + // Export all facts. + walkScope(pass, pass.Pkg.Scope()) + return nil, nil +} diff --git a/tools/defs.bzl b/tools/defs.bzl index f4266e1de..d588241b6 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -8,7 +8,7 @@ change for Google-internal and bazel-compatible rules. load("//tools/go_stateify:defs.bzl", "go_stateify") load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps") load("//tools/nogo:defs.bzl", "nogo_test") -load("//tools/bazeldefs:defs.bzl", _arch_genrule = "arch_genrule", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _version = "version") +load("//tools/bazeldefs:defs.bzl", _amd64_config = "amd64_config", _arch_config = "arch_config", _arm64_config = "arm64_config", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _transition_allowlist = "transition_allowlist", _version = "version") load("//tools/bazeldefs:cc.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _gbenchmark = "gbenchmark", _gbenchmark_internal = "gbenchmark_internal", _grpcpp = "grpcpp", _gtest = "gtest", _vdso_linker_option = "vdso_linker_option") load("//tools/bazeldefs:go.bzl", _bazel_worker_proto = "bazel_worker_proto", _gazelle = "gazelle", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_rule = "go_rule", _go_test = "go_test", _select_goarch = "select_goarch", _select_goos = "select_goos") load("//tools/bazeldefs:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar") @@ -16,10 +16,8 @@ load("//tools/bazeldefs:platforms.bzl", _default_platform = "default_platform", load("//tools/bazeldefs:tags.bzl", "go_suffixes") # Core rules. -arch_genrule = _arch_genrule build_test = _build_test bzl_library = _bzl_library -default_installer = _default_installer default_net_util = _default_net_util select_arch = _select_arch select_system = _select_system @@ -348,3 +346,58 @@ def proto_library(name, srcs, deps = None, has_services = 0, **kwargs): deps = [":" + name + "_cc_proto"], **kwargs ) + +def _arch_transition_impl(settings, attr): + return { + "arm64": _arm64_config(settings, attr), + "amd64": _amd64_config(settings, attr), + } + +arch_transition = transition( + implementation = _arch_transition_impl, + inputs = [], + outputs = _arch_config, +) + +def _arch_genrule_impl(ctx): + """Runs a command with inputs from multiple architectures. + + The command will be run multiple times, with the provided + template rendered using the architecture for the output. + """ + outputs = [] + for (arch, src) in ctx.split_attr.src.items(): + # Calculate the template for this output file. + output = ctx.actions.declare_file(ctx.attr.template % arch) + outputs.append(output) + + # Copy the specific generated source. + input_files = src[DefaultInfo].files + ctx.actions.run_shell( + inputs = input_files, + outputs = [output], + command = "cp %s %s" % ( + " ".join([f.path for f in input_files.to_list()]), + output.path, + ), + ) + return [DefaultInfo( + files = depset(outputs), + )] + +arch_genrule = rule( + implementation = _arch_genrule_impl, + attrs = { + "src": attr.label( + doc = "Sources for the genrule.", + cfg = arch_transition, + ), + "template": attr.string( + doc = "Template for the output files.", + mandatory = True, + ), + "_allowlist_function_transition": attr.label( + default = _transition_allowlist, + ), + }, +) diff --git a/tools/go_branch.sh b/tools/go_branch.sh index 2cfd5b5c3..1f8a55870 100755 --- a/tools/go_branch.sh +++ b/tools/go_branch.sh @@ -45,36 +45,10 @@ origpwd=$(pwd) othersrc=("go.mod" "go.sum" "AUTHORS" "LICENSE") readonly module origpwd othersrc -# Build an amd64 & arm64 gopath. -declare -r go_amd64="${tmp_dir}/amd64" -declare -r go_arm64="${tmp_dir}/arm64" +# Build a full gopath. +declare -r go_output="${tmp_dir}/output" make build BAZEL_OPTIONS="" TARGETS="//:gopath" -rsync --recursive --delete --copy-links bazel-bin/gopath/ "${go_amd64}" -make build BAZEL_OPTIONS=--config=cross-aarch64 TARGETS="//:gopath" 2>/dev/null -rsync --recursive --delete --copy-links bazel-bin/gopath/ "${go_arm64}" - -# Strip irrelevant files, i.e. use only arm64 files from the arm64 build. -# This is because bazel may generate incorrect files for non-target platforms -# as a workaround. See pkg/sentry/loader/vdsodata as an example. -find "${go_amd64}/src/${module}" -name '*_arm64*.go' -exec rm -f {} \; -find "${go_amd64}/src/${module}" -name '*_arm64*.s' -exec rm -f {} \; -find "${go_arm64}/src/${module}" -name '*_amd64*.go' -exec rm -f {} \; -find "${go_arm64}/src/${module}" -name '*_amd64*.s' -exec rm -f {} \; - -# Check that all files are compatible. This means that if the files exist in -# both architectures, then they must be identical. The only ones that we expect -# to exist in a single architecture (due to binary builds) may be different. -function cross_check() { - (cd "${1}" && find "src/${module}" -type f | \ - xargs -n 1 -I {} sh -c "diff '${1}/{}' '${2}/{}' 2>/dev/null; test \$? -ne 1") -} -cross_check "${go_arm64}" "${go_amd64}" -cross_check "${go_amd64}" "${go_arm64}" - -# Merge the two for a complete set of source files. -declare -r go_merged="${tmp_dir}/merged" -rsync --recursive "${go_amd64}/" "${go_merged}" -rsync --recursive "${go_arm64}/" "${go_merged}" +rsync --recursive --delete --copy-links bazel-bin/gopath/ "${go_output}" # We expect to have an existing go branch that we will use as the basis for this # commit. That branch may be empty, but it must exist. We search for this branch @@ -120,7 +94,7 @@ find . -type d -exec chmod 0755 {} \; # will change here. Otherwise, it adds a tremendous amount of noise to commits. # If this file disappears in the future, then presumably we will still delete # the underlying directory. -declare -r gopath="${go_merged}/src/${module}" +declare -r gopath="${go_output}/src/${module}" rsync --recursive --delete \ --exclude .git \ "${gopath}/" . diff --git a/tools/nogo/BUILD b/tools/nogo/BUILD index d72821377..47357329b 100644 --- a/tools/nogo/BUILD +++ b/tools/nogo/BUILD @@ -1,10 +1,8 @@ -load("//tools:defs.bzl", "bzl_library", "go_library", "go_test", "select_goarch", "select_goos") -load("//tools/nogo:defs.bzl", "nogo_objdump_tool", "nogo_stdlib", "nogo_target") +load("//tools:defs.bzl", "bzl_library", "go_binary", "select_goarch", "select_goos") +load("//tools/nogo:defs.bzl", "nogo_stdlib", "nogo_target") package(licenses = ["notice"]) -exports_files(["config-schema.json"]) - nogo_target( name = "target", goarch = select_goarch(), @@ -12,72 +10,16 @@ nogo_target( visibility = ["//visibility:public"], ) -nogo_objdump_tool( - name = "objdump_tool", - visibility = ["//visibility:public"], -) - nogo_stdlib( name = "stdlib", visibility = ["//visibility:public"], ) -go_library( +go_binary( name = "nogo", - srcs = [ - "analyzers.go", - "build.go", - "config.go", - "findings.go", - "nogo.go", - ], - nogo = False, - visibility = ["//:sandbox"], - deps = [ - "//tools/checkescape", - "//tools/checklinkname", - "//tools/checklocks", - "//tools/checkunsafe", - "//tools/nogo/objdump", - "//tools/worker", - "@co_honnef_go_tools//staticcheck:go_default_library", - "@co_honnef_go_tools//stylecheck:go_default_library", - "@org_golang_x_tools//go/analysis:go_default_library", - "@org_golang_x_tools//go/analysis/internal/facts:go_default_library", - "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library", - "@org_golang_x_tools//go/analysis/passes/assign:go_default_library", - "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library", - "@org_golang_x_tools//go/analysis/passes/bools:go_default_library", - "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library", - "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library", - "@org_golang_x_tools//go/analysis/passes/composite:go_default_library", - "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library", - "@org_golang_x_tools//go/analysis/passes/errorsas:go_default_library", - "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library", - "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library", - "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library", - "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library", - "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library", - "@org_golang_x_tools//go/analysis/passes/printf:go_default_library", - "@org_golang_x_tools//go/analysis/passes/shadow:go_default_library", - "@org_golang_x_tools//go/analysis/passes/shift:go_default_library", - "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library", - "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library", - "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library", - "@org_golang_x_tools//go/analysis/passes/tests:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library", - "@org_golang_x_tools//go/gcexportdata:go_default_library", - "@org_golang_x_tools//go/types/objectpath:go_default_library", - ], -) - -go_test( - name = "nogo_test", - srcs = ["config_test.go"], - library = ":nogo", + srcs = ["main.go"], + visibility = ["//visibility:public"], + deps = ["//tools/nogo/cli"], ) bzl_library( diff --git a/tools/nogo/analyzers.go b/tools/nogo/analyzers.go deleted file mode 100644 index db8bbdb8a..000000000 --- a/tools/nogo/analyzers.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nogo - -import ( - "encoding/gob" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/passes/asmdecl" - "golang.org/x/tools/go/analysis/passes/assign" - "golang.org/x/tools/go/analysis/passes/atomic" - "golang.org/x/tools/go/analysis/passes/bools" - "golang.org/x/tools/go/analysis/passes/buildtag" - "golang.org/x/tools/go/analysis/passes/cgocall" - "golang.org/x/tools/go/analysis/passes/composite" - "golang.org/x/tools/go/analysis/passes/copylock" - "golang.org/x/tools/go/analysis/passes/errorsas" - "golang.org/x/tools/go/analysis/passes/httpresponse" - "golang.org/x/tools/go/analysis/passes/loopclosure" - "golang.org/x/tools/go/analysis/passes/lostcancel" - "golang.org/x/tools/go/analysis/passes/nilfunc" - "golang.org/x/tools/go/analysis/passes/nilness" - "golang.org/x/tools/go/analysis/passes/printf" - "golang.org/x/tools/go/analysis/passes/shadow" - "golang.org/x/tools/go/analysis/passes/shift" - "golang.org/x/tools/go/analysis/passes/stdmethods" - "golang.org/x/tools/go/analysis/passes/stringintconv" - "golang.org/x/tools/go/analysis/passes/structtag" - "golang.org/x/tools/go/analysis/passes/tests" - "golang.org/x/tools/go/analysis/passes/unmarshal" - "golang.org/x/tools/go/analysis/passes/unreachable" - "golang.org/x/tools/go/analysis/passes/unsafeptr" - "golang.org/x/tools/go/analysis/passes/unusedresult" - "honnef.co/go/tools/staticcheck" - "honnef.co/go/tools/stylecheck" - - "gvisor.dev/gvisor/tools/checkescape" - "gvisor.dev/gvisor/tools/checklinkname" - "gvisor.dev/gvisor/tools/checklocks" - "gvisor.dev/gvisor/tools/checkunsafe" -) - -// AllAnalyzers is a list of all available analyzers. -var AllAnalyzers = []*analysis.Analyzer{ - asmdecl.Analyzer, - assign.Analyzer, - atomic.Analyzer, - bools.Analyzer, - buildtag.Analyzer, - cgocall.Analyzer, - composite.Analyzer, - copylock.Analyzer, - errorsas.Analyzer, - httpresponse.Analyzer, - loopclosure.Analyzer, - lostcancel.Analyzer, - nilfunc.Analyzer, - nilness.Analyzer, - printf.Analyzer, - shift.Analyzer, - stdmethods.Analyzer, - stringintconv.Analyzer, - shadow.Analyzer, - structtag.Analyzer, - tests.Analyzer, - unmarshal.Analyzer, - unreachable.Analyzer, - unsafeptr.Analyzer, - unusedresult.Analyzer, - checkescape.Analyzer, - checkunsafe.Analyzer, - checklinkname.Analyzer, - checklocks.Analyzer, -} - -func register(all []*analysis.Analyzer) { - // Register all fact types. - // - // N.B. This needs to be done recursively, because there may be - // analyzers in the Requires list that do not appear explicitly above. - registered := make(map[*analysis.Analyzer]struct{}) - var registerOne func(*analysis.Analyzer) - registerOne = func(a *analysis.Analyzer) { - if _, ok := registered[a]; ok { - return - } - - // Register dependencies. - for _, da := range a.Requires { - registerOne(da) - } - - // Register local facts. - for _, f := range a.FactTypes { - gob.Register(f) - } - - registered[a] = struct{}{} // Done. - } - for _, a := range all { - registerOne(a) - } -} - -func init() { - // Add all staticcheck analyzers. - for _, a := range staticcheck.Analyzers { - AllAnalyzers = append(AllAnalyzers, a.Analyzer) - } - // Add all stylecheck analyzers. - for _, a := range stylecheck.Analyzers { - AllAnalyzers = append(AllAnalyzers, a.Analyzer) - } - - // Register lists. - register(AllAnalyzers) -} diff --git a/tools/nogo/check/BUILD b/tools/nogo/check/BUILD index 666780dd3..514442531 100644 --- a/tools/nogo/check/BUILD +++ b/tools/nogo/check/BUILD @@ -1,14 +1,54 @@ -load("//tools:defs.bzl", "go_binary") +load("//tools:defs.bzl", "go_library") package(licenses = ["notice"]) -go_binary( +go_library( name = "check", - srcs = ["main.go"], - nogo = False, - visibility = ["//visibility:public"], + srcs = [ + "analyzers.go", + "build.go", + "check.go", + "findings.go", + ], + visibility = ["//tools/nogo:__subpackages__"], deps = [ - "//tools/nogo", + "//runsc/flag", + "//tools/checkescape", + "//tools/checkinfo", + "//tools/checklinkname", + "//tools/checklocks", + "//tools/checkunsafe", + "//tools/nogo/facts", + "//tools/nogo/flags", "//tools/worker", + "@co_honnef_go_tools//staticcheck:go_default_library", + "@co_honnef_go_tools//stylecheck:go_default_library", + "@org_golang_x_tools//go/analysis:go_default_library", + "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library", + "@org_golang_x_tools//go/analysis/passes/assign:go_default_library", + "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library", + "@org_golang_x_tools//go/analysis/passes/bools:go_default_library", + "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library", + "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library", + "@org_golang_x_tools//go/analysis/passes/composite:go_default_library", + "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library", + "@org_golang_x_tools//go/analysis/passes/errorsas:go_default_library", + "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library", + "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library", + "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library", + "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library", + "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library", + "@org_golang_x_tools//go/analysis/passes/printf:go_default_library", + "@org_golang_x_tools//go/analysis/passes/shadow:go_default_library", + "@org_golang_x_tools//go/analysis/passes/shift:go_default_library", + "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library", + "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library", + "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library", + "@org_golang_x_tools//go/analysis/passes/tests:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library", + "@org_golang_x_tools//go/gcexportdata:go_default_library", ], ) diff --git a/tools/nogo/check/analyzers.go b/tools/nogo/check/analyzers.go new file mode 100644 index 000000000..dc3f3bb06 --- /dev/null +++ b/tools/nogo/check/analyzers.go @@ -0,0 +1,195 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package check + +import ( + "encoding/gob" + "io" + "reflect" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/asmdecl" + "golang.org/x/tools/go/analysis/passes/assign" + "golang.org/x/tools/go/analysis/passes/atomic" + "golang.org/x/tools/go/analysis/passes/bools" + "golang.org/x/tools/go/analysis/passes/buildtag" + "golang.org/x/tools/go/analysis/passes/cgocall" + "golang.org/x/tools/go/analysis/passes/composite" + "golang.org/x/tools/go/analysis/passes/copylock" + "golang.org/x/tools/go/analysis/passes/errorsas" + "golang.org/x/tools/go/analysis/passes/httpresponse" + "golang.org/x/tools/go/analysis/passes/loopclosure" + "golang.org/x/tools/go/analysis/passes/lostcancel" + "golang.org/x/tools/go/analysis/passes/nilfunc" + "golang.org/x/tools/go/analysis/passes/nilness" + "golang.org/x/tools/go/analysis/passes/printf" + "golang.org/x/tools/go/analysis/passes/shadow" + "golang.org/x/tools/go/analysis/passes/shift" + "golang.org/x/tools/go/analysis/passes/stdmethods" + "golang.org/x/tools/go/analysis/passes/stringintconv" + "golang.org/x/tools/go/analysis/passes/structtag" + "golang.org/x/tools/go/analysis/passes/tests" + "golang.org/x/tools/go/analysis/passes/unmarshal" + "golang.org/x/tools/go/analysis/passes/unreachable" + "golang.org/x/tools/go/analysis/passes/unsafeptr" + "golang.org/x/tools/go/analysis/passes/unusedresult" + "honnef.co/go/tools/staticcheck" + "honnef.co/go/tools/stylecheck" + + "gvisor.dev/gvisor/tools/checkescape" + "gvisor.dev/gvisor/tools/checkinfo" + "gvisor.dev/gvisor/tools/checklinkname" + "gvisor.dev/gvisor/tools/checklocks" + "gvisor.dev/gvisor/tools/checkunsafe" +) + +// binaryAnalyzer is a special class of analyzer which supports an additional +// operation to run an analyzer with the object binary data. +type binaryAnalyzer interface { + // Run runs the analyzer with the given binary data. + Run(*analysis.Pass, io.Reader) (interface{}, error) +} + +// analyzer is a simple analysis.Analyzer interface. +// +// This is implemented by plainAnalyzer, and is used to allow calls to +// non-standard analyzers (e.g. checkescape, which requires the objdump output +// in addition to the existing pass information). +type analyzer interface { + Legacy() *analysis.Analyzer +} + +// plainAnalyzer implements analyzer. +type plainAnalyzer struct { + *analysis.Analyzer +} + +// Legacy implements analyzer.Legacy. +func (pa *plainAnalyzer) Legacy() *analysis.Analyzer { + return pa.Analyzer +} + +var ( + // allAnalyzers is a list of all available analyzers. + // + // This is guaranteed to be complete closure around the dependency + // graph of all analyzers (via the "Requires" attribute, below). + // Therefore, to map an *analysis.Analyzer to a runner, you may safely + // use "findAnalyzer". + allAnalyzers = make(map[*analysis.Analyzer]analyzer) + + // allFactTypes is a list of all fact types, useful as a filter. + allFactTypes = make(map[reflect.Type]bool) + + // allFactNames is a list with all fact names. + allFactNames = make(map[reflect.Type]string) +) + +// findAnalyzer maps orig to an analyzer instance. +// +// This is guaranteed to work provided allAnalyzers is made into a transitive +// closure of all known analyzers (see init). +func findAnalyzer(orig *analysis.Analyzer) analyzer { + return allAnalyzers[orig] +} + +// registerFactType registers a analysis.Fact. +func registerFactType(f analysis.Fact) { + // Already registered? + t := reflect.TypeOf(f) + if _, ok := allFactTypes[t]; ok { + return + } + + // Register the type. + gob.Register(f) + allFactTypes[t] = true + s := t.String() + for len(s) > 0 && s[0] == '*' { + s = s[1:] + } + + // Take only the final element. + parts := strings.Split(s, ".") + allFactNames[t] = parts[len(parts)-1] +} + +// register recurisvely registers an analyzer. +func register(a analyzer) { + // Already registered? + if _, ok := allAnalyzers[a.Legacy()]; ok { + return + } + + // Register all fact types. + for _, f := range a.Legacy().FactTypes { + registerFactType(f) + } + + // Register dependencies. + for _, orig := range a.Legacy().Requires { + if findAnalyzer(orig) == nil { + register(&plainAnalyzer{orig}) + } + } + + // Save the analyzer. + allAnalyzers[a.Legacy()] = a +} + +func init() { + // Standard & internal analyzers. + register(&plainAnalyzer{asmdecl.Analyzer}) + register(&plainAnalyzer{assign.Analyzer}) + register(&plainAnalyzer{atomic.Analyzer}) + register(&plainAnalyzer{bools.Analyzer}) + register(&plainAnalyzer{buildtag.Analyzer}) + register(&plainAnalyzer{cgocall.Analyzer}) + register(&plainAnalyzer{composite.Analyzer}) + register(&plainAnalyzer{copylock.Analyzer}) + register(&plainAnalyzer{errorsas.Analyzer}) + register(&plainAnalyzer{httpresponse.Analyzer}) + register(&plainAnalyzer{loopclosure.Analyzer}) + register(&plainAnalyzer{lostcancel.Analyzer}) + register(&plainAnalyzer{nilfunc.Analyzer}) + register(&plainAnalyzer{nilness.Analyzer}) + register(&plainAnalyzer{printf.Analyzer}) + register(&plainAnalyzer{shift.Analyzer}) + register(&plainAnalyzer{stdmethods.Analyzer}) + register(&plainAnalyzer{stringintconv.Analyzer}) + register(&plainAnalyzer{shadow.Analyzer}) + register(&plainAnalyzer{structtag.Analyzer}) + register(&plainAnalyzer{tests.Analyzer}) + register(&plainAnalyzer{unmarshal.Analyzer}) + register(&plainAnalyzer{unreachable.Analyzer}) + register(&plainAnalyzer{unsafeptr.Analyzer}) + register(&plainAnalyzer{unusedresult.Analyzer}) + register(checkescape.Analyzer) + register(&plainAnalyzer{checkinfo.Analyzer}) + register(&plainAnalyzer{checkunsafe.Analyzer}) + register(&plainAnalyzer{checklinkname.Analyzer}) + register(&plainAnalyzer{checklocks.Analyzer}) + + // Add all staticcheck analyzers. + for _, a := range staticcheck.Analyzers { + register(&plainAnalyzer{a.Analyzer}) + } + + // Add all stylecheck analyzers. + for _, a := range stylecheck.Analyzers { + register(&plainAnalyzer{a.Analyzer}) + } +} diff --git a/tools/nogo/build.go b/tools/nogo/check/build.go similarity index 60% rename from tools/nogo/build.go rename to tools/nogo/check/build.go index 003533c71..098252df5 100644 --- a/tools/nogo/build.go +++ b/tools/nogo/check/build.go @@ -12,27 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.1 -// +build go1.1 - -package nogo +package check import ( "fmt" "io" "os" + + "gvisor.dev/gvisor/tools/nogo/flags" ) // findStdPkg needs to find the bundled standard library packages. -func findStdPkg(GOOS, GOARCH, path string) (io.ReadCloser, error) { +var findStdPkg = func(path string) (io.ReadCloser, error) { if path == "C" { // Cgo builds cannot be analyzed. Skip. return nil, ErrSkip } - return os.Open(fmt.Sprintf("external/go_sdk/pkg/%s_%s/%s.a", GOOS, GOARCH, path)) + + // Attempt to use the root, if available. + root, envErr := flags.Env("GOROOT") + if envErr != nil { + return nil, fmt.Errorf("unable to resolve GOROOT: %w", envErr) + } + + // Attempt to resolve the library, and propagate this error. + return os.Open(fmt.Sprintf("%s/pkg/%s_%s/%s.a", root, flags.GOOS, flags.GOARCH, path)) } -// ReleaseTags returns nil, indicating that the defaults should be used. -func ReleaseTags() ([]string, error) { +// releaseTags returns nil, indicating that the defaults should be used. +var releaseTags = func() ([]string, error) { return nil, nil } diff --git a/tools/nogo/check/check.go b/tools/nogo/check/check.go new file mode 100644 index 000000000..d01866ccc --- /dev/null +++ b/tools/nogo/check/check.go @@ -0,0 +1,783 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package check implements binary analysis similar to bazel's nogo, or the +// unitchecker package. It exists in order to provide additional facilities for +// analysis, namely plumbing through the output from dumping the generated +// binary (to analyze actual produced code). +package check + +import ( + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "io" + "log" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/gcexportdata" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/tools/nogo/facts" + "gvisor.dev/gvisor/tools/nogo/flags" + "gvisor.dev/gvisor/tools/worker" +) + +var ( + // ErrSkip indicates the package should be skipped. + ErrSkip = errors.New("skipped2") + + // cachedFacts caches by file (just byte data). + cachedFacts = worker.NewCache("facts") + + // bundleCachedFacts caches the standard library (bundleFacts). + bundleCachedFacts = worker.NewCache("stdlib") + + // showTimes indicates we should show analyzer times. + showTimes = flag.Bool("show_times", false, "show all analyzer times") +) + +var ( + tagsOnce sync.Once + buildTags []string + releaseTagsVal []string + releaseTagsErr error +) + +// versionTags generates all version tags. +// +// This function will panic if passed an invalid version. +func versionTags(v string) (tags []string) { + if len(v) < 2 || string(v[:2]) != "go" { + panic(fmt.Errorf("version %q is not valid", v)) + } + v = v[2:] // Strip go prefix. + v = strings.Split(v, " ")[0] + v = strings.Split(v, "-")[0] + parts := strings.Split(v, ".") + if len(parts) < 2 { + panic(fmt.Errorf("version %q lacks major and minor number", v)) + } + major, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + panic(fmt.Errorf("version %q contains invalid major: %w", v, err)) + } + minor, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + panic(fmt.Errorf("version %q contains invalid minor: %w", v, err)) + } + // Generate all compliant tags. + for i := int64(0); i <= minor; i++ { + tags = append(tags, fmt.Sprintf("go%d.%d", major, i)) + } + return tags +} + +// shouldInclude indicates whether the file should be included. +func shouldInclude(path string) (bool, error) { + tagsOnce.Do(func() { + if len(flags.BuildTags) > 0 { + buildTags = strings.Split(flags.BuildTags, ",") + } + if v, err := flags.Env("GOVERSION"); err == nil { + buildTags = append(buildTags, versionTags(v)...) + } else { + buildTags = append(buildTags, versionTags(runtime.Version())...) + } + releaseTagsVal, releaseTagsErr = releaseTags() + }) + if releaseTagsErr != nil { + return false, releaseTagsErr + } + ctx := build.Default + ctx.GOOS = flags.GOOS + ctx.GOARCH = flags.GOARCH + ctx.BuildTags = buildTags + ctx.ReleaseTags = releaseTagsVal + return ctx.MatchFile(filepath.Dir(path), filepath.Base(path)) +} + +// sortSrcs sorts a set of src files into Go files and non-Go files. +func sortSrcs(srcs []string) (goFiles []string, nonGoFiles []string) { + for _, filename := range srcs { + if strings.HasSuffix(filename, ".go") { + goFiles = append(goFiles, filename) + } else { + nonGoFiles = append(nonGoFiles, filename) + } + } + return +} + +// importerEntry is a single entry in the importer. +type importerEntry struct { + ready sync.WaitGroup + pkg *types.Package + findings FindingSet + facts *facts.Package + err error +} + +// importer is an almost-implementation of go/types.Importer. +// +// This wraps a configuration, which provides the map of package names to +// files, and the facts. Note that this importer implementation will always +// pass when a given package is not available. +type importer struct { + fset *token.FileSet + sources map[string][]string + + // mu protects cache. + mu sync.Mutex + cache map[string]*importerEntry + + // importsMu protects imports. + importsMu sync.Mutex + imports map[string]*types.Package +} + +// allFacts returns all package facts for the given name. +// +// This attempts to load via the FactMap (global flags) or the Bundles (global +// flags), but falls back to attempting a direct import. +func (i *importer) allFacts(pkg *types.Package) (*facts.Package, error) { + // Attempt to load from the fact map. + filename, ok := flags.FactMap[pkg.Path()] + if ok { + cb, err := cachedFacts.Lookup([]string{filename}, func() (worker.Sizer, error) { + r, openErr := os.Open(filename) + if openErr != nil { + return nil, fmt.Errorf("error loading facts from %q: %w", filename, openErr) + } + defer r.Close() + loadedFacts := facts.NewPackage(pkg) + if _, readErr := loadedFacts.ReadFrom(r); readErr != nil { + return nil, fmt.Errorf("error loading facts: %w", readErr) + } + return loadedFacts, nil + }) + if err != nil { + return nil, err + } + return cb.(*facts.Package), nil + } + + // Attempt to load any bundles. + for _, filename := range flags.Bundles { + cb, err := bundleCachedFacts.Lookup([]string{filename}, func() (worker.Sizer, error) { + r, openErr := os.Open(filename) + if openErr != nil { + return nil, fmt.Errorf("error loading bundled facts from %q: %w", filename, openErr) + } + defer r.Close() + loadedFacts := facts.NewBundle(i) + if _, readErr := loadedFacts.ReadFrom(r); readErr != nil { + // If the file is length zero, we skip it. This + // is because stray fact files may been left + // behind that are attempting to recreate now. + fi, err := r.Stat() + if err == nil && fi.Size() == 0 { + return nil, ErrSkip + } + return nil, fmt.Errorf("error loading bundled facts: %w", readErr) + } + return loadedFacts, nil + }) + if err == ErrSkip { + continue // See above. + } + if err != nil { + return nil, err + } + if loadedFacts, ok := cb.(*facts.Bundle).Packages[pkg.Path()]; ok { + return loadedFacts, nil + } + } + + // Attempt to resolve the package via import. + _, parsedFacts, err := i.importPackage(pkg.Path()) + return parsedFacts, err +} + +// fastFact returns facts for the given package. +func (i *importer) fastFact(pkg *types.Package, obj types.Object, ptr analysis.Fact) bool { + foundFacts, err := i.allFacts(pkg) + if err != nil || foundFacts == nil { + return false + } + return foundFacts.ImportFact(obj, ptr) +} + +// findBinary finds the binary for the given package. +func (i *importer) findBinary(path string) (rc io.ReadCloser, err error) { + realPath, ok := flags.ImportMap[path] + if !ok { + // Not found in the import path. Attempt to find the package + // via the standard library. + rc, err = findStdPkg(path) + } else { + // Open the file. + rc, err = os.Open(realPath) + } + return rc, err +} + +// importPackage almost-implements types.Importer.Import. +// +// This must be called by other methods directly. +func (i *importer) importPackage(path string) (*types.Package, *facts.Package, error) { + if path == "unsafe" { + // Special case: go/types has pre-defined type information for + // unsafe. We ensure that this package is correct, in case any + // analyzers are specifically looking for this. + return types.Unsafe, nil, nil + } + + // Pull the internal entry. + i.mu.Lock() + entry, ok := i.cache[path] + if ok { + i.mu.Unlock() + entry.ready.Wait() + return entry.pkg, entry.facts, entry.err + } + + // Start preparing this entry. + entry = new(importerEntry) + entry.ready.Add(1) + defer entry.ready.Done() + i.cache[path] = entry + i.mu.Unlock() + + // If we have the srcs for this package, then we can actually do an + // analysis from first principles to validate the package and derive + // the types. We strictly prefer this to the gcexportdata. + if srcs, ok := i.sources[path]; ok && len(srcs) > 0 { + start := time.Now() + entry.pkg, entry.findings, entry.facts, entry.err = i.checkPackage(path, srcs) + if entry.err != nil { + return nil, nil, entry.err + } + // Why does the news already need to be bad? Note that this is + // printed here because this will only happen when multiple + // packages are being analyzed. + log.Printf("SUCCESS: all analyzers successfully completed %q (%v).", path, time.Since(start)) + i.importsMu.Lock() + defer i.importsMu.Unlock() + i.imports[path] = entry.pkg + return entry.pkg, entry.facts, entry.err + } + + // Load all exported data. Unfortunately, we will have to hold the lock + // during this time. The imported may access imports directly. + rc, err := i.findBinary(path) + if err != nil { + return nil, nil, err + } + defer rc.Close() + r, err := gcexportdata.NewReader(rc) + if err != nil { + return nil, nil, err + } + i.importsMu.Lock() + defer i.importsMu.Unlock() + entry.pkg, entry.err = gcexportdata.Read(r, i.fset, i.imports, path) + return entry.pkg, entry.facts, entry.err +} + +// Import implements types.Importer.Import. +func (i *importer) Import(path string) (*types.Package, error) { + pkg, _, err := i.importPackage(path) + return pkg, err +} + +// errorImporter tracks the last error. +type errorImporter struct { + *importer + lastErr atomic.Value +} + +// Import implements types.Importer.Import. +func (i *errorImporter) Import(path string) (*types.Package, error) { + pkg, _, err := i.importer.importPackage(path) + if err != nil { + i.lastErr.Store(err) + } + return pkg, err +} + +// checkPackage is the backing implementation for CheckPackage and others. +// +// The implementation was adapted from [1], which was in turn adpated from [2]. +// This returns a list of matching analysis issues, or an error if the analysis +// could not be completed. +// +// Note that a partial result may be returned if an error occurred on at least +// one analyzer. This may be expected if e.g. a binary is not provided but a +// binaryAnalyzer is used. +// +// [1] bazelbuid/rules_go/tools/builders/nogo_main.go +// [2] golang.org/x/tools/go/checker/internal/checker +func (i *importer) checkPackage(path string, srcs []string) (*types.Package, FindingSet, *facts.Package, error) { + // Load all source files. + goFiles, _ := sortSrcs(srcs) + syntax := make([]*ast.File, 0, len(goFiles)) + for _, file := range goFiles { + include, err := shouldInclude(file) + if err != nil { + return nil, nil, nil, fmt.Errorf("error evaluating file %q: %w", file, err) + } + if !include { + continue + } + s, err := parser.ParseFile(i.fset, file, nil, parser.ParseComments) + if err != nil { + return nil, nil, nil, fmt.Errorf("error parsing file %q: %w", file, err) + } + syntax = append(syntax, s) + } + + // Check type information. + ei := &errorImporter{ + importer: i, + } + typesSizes := types.SizesFor("gc", flags.GOARCH) + typeConfig := types.Config{ + Importer: ei, + } + typesInfo := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Uses: make(map[*ast.Ident]types.Object), + Defs: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Scopes: make(map[ast.Node]*types.Scope), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + } + astPackage, err := typeConfig.Check(path, i.fset, syntax, typesInfo) + if err != nil && ei.lastErr.Load() != ErrSkip { + return nil, nil, nil, fmt.Errorf("error checking types: %w", err) + } + + // We start with completely empty facts. All of our facts are sourced + // via the fastFact function that hits the local caches. + // + // Note that facts should be reconcilable between types as of go/tools + // commit ee04797aa0b6be5ce3d5f7ac0f91e34716b3acdf. We previously used + // to do a sanity check to ensure that binary import data was + // compatible with ast-derived data, but this is no longer necessary. + // If packages are available locally, we can refer to those directly. + astFacts := facts.NewPackage(astPackage) + + // Recursively visit all analyzers. + var ( + resultsMu sync.RWMutex // protects results & errs, findings. + factsMu sync.RWMutex // protects facts. + ready = make(map[*analysis.Analyzer]*sync.WaitGroup) + results = make(map[*analysis.Analyzer]interface{}) + errs = make(map[*analysis.Analyzer]error) + findings = make(FindingSet, 0) + ) + for a := range allAnalyzers { + wg := new(sync.WaitGroup) + wg.Add(1) // For analysis. + ready[a] = wg + } + limit := make(chan struct{}, 1) + for a, wg := range ready { + go func(a *analysis.Analyzer, wg *sync.WaitGroup) { + defer wg.Done() + + // Wait for all requirements. + for _, orig := range a.Requires { + ready[orig].Wait() + + // Should we bail early? + resultsMu.RLock() + if err := errs[orig]; err != nil { + resultsMu.RUnlock() + resultsMu.Lock() + defer resultsMu.Unlock() + errs[a] = err + return + } + resultsMu.RUnlock() + } + + limit <- struct{}{} + defer func() { <-limit }() + + // Collect local fact types. + localFactTypes := make(map[reflect.Type]bool) + for _, ft := range a.FactTypes { + localFactTypes[reflect.TypeOf(ft)] = true + } + + // Run the analysis. + var localFindings FindingSet + p := &analysis.Pass{ + Analyzer: a, + Fset: i.fset, + Files: syntax, + Pkg: astPackage, + TypesInfo: typesInfo, + ResultOf: results, // All results. + Report: func(d analysis.Diagnostic) { + localFindings = append(localFindings, Finding{ + Category: a.Name, + Position: i.fset.Position(d.Pos), + Message: d.Message, + }) + }, + ImportPackageFact: func(pkg *types.Package, ptr analysis.Fact) bool { + if pkg != astPackage { + return i.fastFact(pkg, nil, ptr) + } + factsMu.RLock() + defer factsMu.RUnlock() + return astFacts.ImportFact(nil, ptr) + }, + ExportPackageFact: func(fact analysis.Fact) { + factsMu.Lock() + defer factsMu.Unlock() + astFacts.ExportFact(nil, fact) + }, + ImportObjectFact: func(obj types.Object, ptr analysis.Fact) bool { + if pkg := obj.Pkg(); pkg != nil && pkg != astPackage { + return i.fastFact(pkg, obj, ptr) + } + factsMu.RLock() + defer factsMu.RUnlock() + return astFacts.ImportFact(obj, ptr) + }, + ExportObjectFact: func(obj types.Object, fact analysis.Fact) { + if obj == nil { + // Tried to export nil object? + log.Printf("WARNING: attempted to export fact for nil object") + return + } + if obj.Pkg() != astPackage { + // This is not allowed: the + // built-in facts library will + // also panic in this case. + log.Printf("WARNING: attempted to export fact for package %s", obj.Pkg().Name()) + return + } + factsMu.Lock() + defer factsMu.Unlock() + astFacts.ExportFact(obj, fact) + }, + AllPackageFacts: func() (rv []analysis.PackageFact) { + factsMu.RLock() + defer factsMu.RUnlock() + // Pull all dependencies. + for _, importedPkg := range astPackage.Imports() { + otherFacts, err := i.allFacts(importedPkg) + if err != nil || otherFacts == nil { + continue + } + for typ := range localFactTypes { + v := reflect.New(typ.Elem()) + if otherFacts.ImportFact(nil, v.Interface().(analysis.Fact)) { + rv = append(rv, analysis.PackageFact{ + Package: importedPkg, + Fact: v.Interface().(analysis.Fact), + }) + } + } + } + // Pull all local facts. + for typ := range localFactTypes { + v := reflect.New(typ.Elem()) + if astFacts.ImportFact(nil, v.Interface().(analysis.Fact)) { + rv = append(rv, analysis.PackageFact{ + Package: astPackage, + Fact: v.Interface().(analysis.Fact), + }) + } + } + return + }, + AllObjectFacts: func() (rv []analysis.ObjectFact) { + factsMu.RLock() + defer factsMu.RUnlock() + // Pull all local facts. + for obj, _ := range astFacts.Objects { + for typ := range localFactTypes { + v := reflect.New(typ.Elem()) + if astFacts.ImportFact(obj, v.Interface().(analysis.Fact)) { + rv = append(rv, analysis.ObjectFact{ + Object: obj, + Fact: v.Interface().(analysis.Fact), + }) + } + } + } + return + }, + TypesSizes: typesSizes, + } + + // Ensure any analyzer panics are captured. This may + // happen for packages that are not supported by + // specific analyzers. The only panic that can happen + // is while resultsMu is held as a read-only lock. + var ( + result interface{} + err error + ) + defer func() { + if r := recover(); r != nil { + // In order to make the multiple + // analyzers running concurrently + // debuggable, capture panic exceptions + // and propagate as an analyzer error. + err = fmt.Errorf("panic recovered: %s", r) + resultsMu.RUnlock() // +checklocksignore + resultsMu.Lock() + errs[a] = err + resultsMu.Unlock() + } + }() + found := findAnalyzer(a) + resultsMu.RLock() + if ba, ok := found.(binaryAnalyzer); ok { + // Load the binary and analyze. + rc, loadErr := i.findBinary(path) + if loadErr != nil { + err = loadErr + } else { + result, err = ba.Run(p, rc) + rc.Close() + } + } else { + result, err = a.Run(p) + } + resultsMu.RUnlock() + resultsMu.Lock() + findings = append(findings, localFindings...) + results[a] = result + errs[a] = err + resultsMu.Unlock() + }(a, wg) + } + for _, wg := range ready { + // Wait for completion. + wg.Wait() + } + for a := range ready { + // Check the error. If we generate an error here, we report + // this as a finding that can be suppressed. Some analyzers + // will fail on some packages. + if errs[a] != nil { + findings = append(findings, Finding{ + Category: a.Name, + Position: token.Position{Filename: path}, + Message: errs[a].Error(), + }) + continue + } + + // Check the result. Per above, we check that the type is what + // we expected and that an error did not occur during analysis. + if got, want := reflect.TypeOf(results[a]), a.ResultType; got != want { + return astPackage, findings, astFacts, fmt.Errorf("error: analyzer %s returned %v (expected type %v)", a.Name, results[a], want) + } + } + + // Return all findings. + return astPackage, findings, astFacts, nil +} + +// allFindingsAndFacts returns the complete set. +func (i *importer) allFindingsAndFacts() (FindingSet, *facts.Bundle, error) { + var ( + findings = make(FindingSet, 0) + allFacts = facts.NewBundle(i) + ) + for path, entry := range i.cache { + findings = append(findings, entry.findings...) + if entry.facts != nil { + allFacts.Packages[path] = entry.facts + } else if entry.pkg != nil { + pkgFacts, err := i.allFacts(entry.pkg) + if err != nil { + // This should not happen, we should load facts for all packages. + return nil, nil, fmt.Errorf("no facts available for %s: %v", entry.pkg.Path(), err) + } + allFacts.Packages[path] = pkgFacts + } + } + + // Return the results. + return findings, allFacts, nil +} + +// Package runs all analyzer on a single package. +func Package(path string, srcs []string) (FindingSet, facts.Writer, error) { + i := &importer{ + fset: token.NewFileSet(), + cache: make(map[string]*importerEntry), + imports: make(map[string]*types.Package), + } + _, findings, facts, err := i.checkPackage(path, srcs) + if err != nil { + return nil, nil, err + } + return findings, facts, nil +} + +// Facts runs all analyzers, and returns human-readable facts. +// +// These facts are essentially a dictionary tree (split across all '.' +// characters in the canonical human representation) that can be used for +// rendering via a template. +func Facts(path string, srcs []string) (facts.Resolved, error) { + i := &importer{ + fset: token.NewFileSet(), + cache: make(map[string]*importerEntry), + imports: make(map[string]*types.Package), + } + pkg, _, localFacts, err := i.checkPackage(path, srcs) + if localFacts == nil && err != nil { + // Allow failure here, since we may not care about some + // analyzers for these packages. + return nil, err + } + _, allFacts, err := i.allFindingsAndFacts() + if err != nil { + return nil, err + } + return facts.Resolve(pkg, localFacts, allFacts, allFactNames), nil +} + +// FindRoot finds a package root. +func FindRoot(srcs []string, srcRootRegex string) (string, error) { + if srcRootRegex == "" { + return "", nil + } + + // Calculate the root source directory. This is always a directory + // named 'src', of which we simply take the first we find. This is a + // bit fragile, but works for all currently known Go source + // configurations. + // + // Note that there may be extra files outside of the root source + // directory; we simply ignore those. + re, err := regexp.Compile(srcRootRegex) + if err != nil { + return "", fmt.Errorf("srcRootRegex is not valid: %w", err) + } + srcRootPrefix := "" + for _, filename := range srcs { + if s := re.FindString(filename); len(s) > len(srcRootPrefix) { + srcRootPrefix = s + } + } + if srcRootPrefix == "" { + // For whatever reason, we didn't identify a good common prefix to use here. + return "", fmt.Errorf("unable to identify src prefix for %v with regex %s", srcs, srcRootRegex) + } + return srcRootPrefix, nil +} + +// SplitPackages splits a typical package structure into packages. +func SplitPackages(srcs []string, srcRootPrefix string) map[string][]string { + sources := make(map[string][]string) + for _, filename := range srcs { + if !strings.HasPrefix(filename, srcRootPrefix) { + continue // Superflouous file. + } + d := path.Dir(filename) + if len(srcRootPrefix) >= len(d) { + continue // Not a file. + } + pkg := d[len(srcRootPrefix):] + for len(pkg) > 0 && pkg[0] == '/' { + pkg = pkg[1:] + } + if len(pkg) == 0 { + continue // Also not a file. + } + + // Skip commands where possible. These also have package names + // that do not match the tree structure and will never be + // dependencies. + if strings.HasPrefix(filename, "cmd/") { + continue + } + + // Skip obvious test files; they have bizarre package semantics + // and are never direct dependencies of anything else. + if strings.HasSuffix(filename, "_test.go") { + continue + } + + // Skip unsupported packages explicitly. + if _, ok := usesTypeParams[pkg]; ok { + log.Printf("WARNING: Skipping package %q: type param analysis not yet supported.", pkg) + continue + } + + // Add to the package. + sources[pkg] = append(sources[pkg], filename) + } + + return sources +} + +// Go standard library packages using Go 1.18 type parameter features. +// +// As of writing, analysis tooling is not updated to support type parameters +// and will choke on these packages. We skip these packages entirely for now. +// +// TODO(b/201686256): remove once tooling can handle type parameters. +var usesTypeParams = map[string]struct{}{ + "constraints": struct{}{}, // golang.org/issue/45458 + "maps": struct{}{}, // golang.org/issue/47649 + "slices": struct{}{}, // golang.org/issue/45955 +} + +// Bundle checks a bundle of files (typically the standard library). +func Bundle(sources map[string][]string) (FindingSet, facts.Writer, error) { + // Process all packages. + i := &importer{ + fset: token.NewFileSet(), + sources: sources, + cache: make(map[string]*importerEntry), + imports: make(map[string]*types.Package), + } + for pkg, _ := range sources { + // Was there an error processing this package? Just print a warning. + if _, _, err := i.importPackage(pkg); err != nil && err != ErrSkip { + log.Printf("WARNING: %v.", err) + } + } + + // Build our findings and facts. + return i.allFindingsAndFacts() +} diff --git a/tools/nogo/findings.go b/tools/nogo/check/findings.go similarity index 98% rename from tools/nogo/findings.go rename to tools/nogo/check/findings.go index a73bf1a09..3845990ca 100644 --- a/tools/nogo/findings.go +++ b/tools/nogo/check/findings.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package nogo +package check import ( "encoding/gob" @@ -27,7 +27,7 @@ import ( // Finding is a single finding. type Finding struct { - Category AnalyzerName + Category string Position token.Position Message string } diff --git a/tools/nogo/check/main.go b/tools/nogo/check/main.go deleted file mode 100644 index 17ca0d846..000000000 --- a/tools/nogo/check/main.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary check is the nogo entrypoint. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - - "gvisor.dev/gvisor/tools/nogo" - "gvisor.dev/gvisor/tools/worker" -) - -var ( - packageFile = flag.String("package", "", "package configuration file (in JSON format)") - stdlibFile = flag.String("stdlib", "", "stdlib configuration file (in JSON format)") - findingsOutput = flag.String("findings", "", "output file (or stdout, if not specified)") - factsOutput = flag.String("facts", "", "output file for facts (optional)") -) - -func loadConfig(file string, config interface{}) interface{} { - // Load the configuration. - f, err := os.Open(file) - if err != nil { - log.Fatalf("unable to open configuration %q: %v", file, err) - } - defer f.Close() - dec := json.NewDecoder(f) - dec.DisallowUnknownFields() - if err := dec.Decode(config); err != nil { - log.Fatalf("unable to decode configuration: %v", err) - } - return config -} - -func main() { - worker.Work(run) -} - -func run([]string) int { - var ( - findings []nogo.Finding - factData []byte - err error - ) - - // Check & load the configuration. - if *packageFile != "" && *stdlibFile != "" { - fmt.Fprintf(os.Stderr, "unable to perform stdlib and package analysis; provide only one!") - return 1 - } - - releaseTags, err := nogo.ReleaseTags() - if err != nil { - fmt.Fprintf(os.Stderr, "error determining release tags: %v", err) - return 1 - } - - // Run the configuration. - if *stdlibFile != "" { - // Perform stdlib analysis. - c := loadConfig(*stdlibFile, new(nogo.StdlibConfig)).(*nogo.StdlibConfig) - c.ReleaseTags = releaseTags - findings, factData, err = nogo.CheckStdlib(c, nogo.AllAnalyzers) - } else if *packageFile != "" { - // Perform standard analysis. - c := loadConfig(*packageFile, new(nogo.PackageConfig)).(*nogo.PackageConfig) - c.ReleaseTags = releaseTags - findings, factData, err = nogo.CheckPackage(c, nogo.AllAnalyzers, nil) - } else { - fmt.Fprintf(os.Stderr, "please provide at least one of package or stdlib!") - return 1 - } - - // Check that analysis was successful. - if err != nil { - fmt.Fprintf(os.Stderr, "error performing analysis: %v", err) - return 1 - } - - // Save facts. - if *factsOutput != "" { - if err := ioutil.WriteFile(*factsOutput, factData, 0644); err != nil { - fmt.Fprintf(os.Stderr, "error saving findings to %q: %v", *factsOutput, err) - return 1 - } - } - - // Write all findings. - if *findingsOutput != "" { - w, err := os.OpenFile(*findingsOutput, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - fmt.Fprintf(os.Stderr, "error opening output file %q: %v", *findingsOutput, err) - return 1 - } - if err := nogo.WriteFindingsTo(w, findings, false /* json */); err != nil { - fmt.Fprintf(os.Stderr, "error writing findings to %q: %v", *findingsOutput, err) - return 1 - } - } else { - for _, finding := range findings { - fmt.Fprintf(os.Stdout, "%s\n", finding.String()) - } - } - - return 0 -} diff --git a/tools/nogo/cli/BUILD b/tools/nogo/cli/BUILD new file mode 100644 index 000000000..902eddd86 --- /dev/null +++ b/tools/nogo/cli/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "cli", + srcs = ["cli.go"], + visibility = ["//tools:__subpackages__"], + deps = [ + "//runsc/flag", + "//tools/nogo/check", + "//tools/nogo/config", + "//tools/nogo/facts", + "//tools/nogo/flags", + "//tools/worker", + "@com_github_google_subcommands//:go_default_library", + "@in_gopkg_yaml_v2//:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/tools/nogo/cli/cli.go b/tools/nogo/cli/cli.go new file mode 100644 index 000000000..48a57ee31 --- /dev/null +++ b/tools/nogo/cli/cli.go @@ -0,0 +1,570 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cli implements a basic command line interface. +package cli + +import ( + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "text/template" + + "github.com/google/subcommands" + "golang.org/x/sys/unix" + yaml "gopkg.in/yaml.v2" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/tools/nogo/check" + "gvisor.dev/gvisor/tools/nogo/config" + "gvisor.dev/gvisor/tools/nogo/facts" + "gvisor.dev/gvisor/tools/nogo/flags" + "gvisor.dev/gvisor/tools/worker" +) + +// openOutput opens an output file. +func openOutput(filename string, def io.Writer) (io.Writer, error) { + if filename == "" { + return def, nil + } + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + if err != nil { + // See above. + return nil, err + } + return f, nil +} + +// closeOutput closes an output if necessary. +// +// If an error occurs during close, this function will panic. +func closeOutput(w io.Writer) { + if c, ok := w.(io.Closer); ok { + if err := c.Close(); err != nil { + panic(err) + } + } +} + +// failure exits with the given failure message. +func failure(fmtStr string, v ...interface{}) subcommands.ExitStatus { + fmt.Fprintf(os.Stderr, fmtStr+"\n", v...) + return subcommands.ExitFailure +} + +// isTerminal return true if the file is a terminal. +func isTerminal(w io.Writer) bool { + f, ok := w.(*os.File) + if !ok { + return false + } + _, err := unix.IoctlGetTermios(int(f.Fd()), unix.TCGETS) + return err == nil +} + +// collectAllFiles collects all files from a directory tree. +func collectAllFiles(dir string) (files []string, err error) { + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + files = append(files, path) + } + return nil + }) + return +} + +// checkCommon is a common set of flags for check-like commands. +type checkCommon struct { + Facts string + Findings string + Text bool +} + +// setFlags may be called by embedding types. +// +// Note that the default file names here depend on the command name. See init +// at the bottom, where this files will be registered if they exist already. +func (c *checkCommon) setFlags(fs *flag.FlagSet, commandType string) { + fs.StringVar(&c.Facts, "facts", fmt.Sprintf(".nogo.%s.facts", commandType), "facts output file (optional)") + fs.StringVar(&c.Findings, "findings", "", "findings output file (optional)") + fs.BoolVar(&c.Text, "text", false, "force text output (by default, only if output is a terminal)") +} + +// execute runs the common bits for a check command. +func (c *checkCommon) execute(fn func() (check.FindingSet, facts.Writer, error)) error { + // Open outputs. + factsOutput, err := openOutput(c.Facts, io.Discard) + if err != nil { + return fmt.Errorf("opening facts: %w", err) + } + defer closeOutput(factsOutput) + findingsOutput, err := openOutput(c.Findings, os.Stdout) + if err != nil { + return fmt.Errorf("opening findings: %w", err) + } + defer closeOutput(findingsOutput) + + // Perform the analysis. + findings, factData, err := fn() + if err != nil { + return err + } + + // Save the data. + if _, err := factData.WriteTo(factsOutput); err != nil { + return fmt.Errorf("writing facts: %w", err) + } + if !c.Text && !isTerminal(findingsOutput) { + // Write in the default internal format (GOB encoded). + if err := check.WriteFindingsTo(findingsOutput, findings, false /* json */); err != nil { + return fmt.Errorf("writing findings: %w", err) + } + } else { + // Use a human readable text. + for _, finding := range findings { + fmt.Fprintf(findingsOutput, "%s\n", finding.String()) + } + } + + return nil +} + +// Check implements subcommands.Command for the "check" command. +type Check struct { + checkCommon + Package string + Binary string +} + +// Name implements subcommands.Command.Name. +func (*Check) Name() string { + return "check" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Check) Synopsis() string { + return "Generate facts and findings for a specific named package and sources." +} + +// Usage implements subcommands.Command.Usage. +func (*Check) Usage() string { + return `check + + Generates facts and findings for a specific named package and sources. + This command should generally be considered a "low-level" command, and + it is recommend that you use bundle or mod instead. + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (c *Check) SetFlags(fs *flag.FlagSet) { + c.setFlags(fs, "check") + fs.StringVar(&c.Package, "package", "", "package for analysis (required)") + fs.StringVar(&c.Binary, "binary", "", "binary for analysis (optional, omitting may cause some analyzers to fail)") +} + +// Execute implements subcommands.Command.Execute. +func (c *Check) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus { + if c.Package == "" { + c.Package = "main" // Default, no imports. + } + + // Add the binary to the import map. Note that it may already be + // provided in the import map via the global command line flags, but + // this is able to override that path. + if c.Binary != "" { + flags.ImportMap[c.Package] = c.Binary + } + + // Perform the analysis. + if err := c.execute(func() (check.FindingSet, facts.Writer, error) { + return check.Package(c.Package /* path */, fs.Args() /* srcs */) + }); err != nil { + return failure("%v", err) + } + + return subcommands.ExitSuccess +} + +// Bundle implements subcommands.Command for the "bundle" command. +type Bundle struct { + checkCommon + Root string + Prefix string +} + +// Name implements subcommands.Command.Name. +func (*Bundle) Name() string { + return "bundle" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Bundle) Synopsis() string { + return "Generate facts and findings for a set of sources." +} + +// Usage implements subcommands.Command.Usage. +func (*Bundle) Usage() string { + return `bundle + + Generates facts and findings for a collection of source files. Each + package name is inferred from the path, assuming a standard package + structure. The stripped prefix is determined by regular expression. + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (b *Bundle) SetFlags(fs *flag.FlagSet) { + b.setFlags(fs, "bundle") + fs.StringVar(&b.Root, "root", "", "root regular expression (for package discovery)") + fs.StringVar(&b.Prefix, "prefix", "", "package prefix to apply (for complete names)") +} + +// Execute implements subcommands.Command.Execute. +func (b *Bundle) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus { + // Perform the analysis. + if err := b.execute(func() (check.FindingSet, facts.Writer, error) { + // Discover the correct common root. + srcRootPrefix, err := check.FindRoot(fs.Args(), b.Root) + if err != nil { + return nil, nil, err + } + // Split into packages. + sources := make(map[string][]string) + for pkg, srcs := range check.SplitPackages(fs.Args(), srcRootPrefix) { + path := pkg + if b.Prefix != "" { + path = b.Prefix + "/" + path // Subpackage. + } + sources[path] = append(sources[path], srcs...) + } + return check.Bundle(sources) + }); err != nil { + return failure("%v", err) + } + + return subcommands.ExitSuccess +} + +// Stdlib implements subcommands.Command for the "stdlib" command. +type Stdlib struct { + checkCommon +} + +// Name implements subcommands.Command.Name. +func (*Stdlib) Name() string { + return "stdlib" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Stdlib) Synopsis() string { + return "Generate facts and findings for the standard library." +} + +// Usage implements subcommands.Command.Usage. +func (*Stdlib) Usage() string { + return `stdlib + + Generates facts and findings for the standard library. This wraps + bundle with a mechansim that discovers the standard library source. + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (s *Stdlib) SetFlags(fs *flag.FlagSet) { + s.setFlags(fs, "stdlib") +} + +// Execute implements subcommands.Command.Execute. +func (s *Stdlib) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus { + if fs.NArg() != 0 { + return subcommands.ExitUsageError // Need no arguments. + } + + if err := s.execute(func() (check.FindingSet, facts.Writer, error) { + root, err := flags.Env("GOROOT") + if err != nil { + return nil, nil, err + } + root = path.Join(root, "src") + srcs, err := collectAllFiles(root) + if err != nil { + return nil, nil, err + } + return check.Bundle(check.SplitPackages(srcs, root)) + }); err != nil { + return failure("%v", err) + } + + return subcommands.ExitSuccess +} + +// Filter implements subcommands.Command for the "filter" command. +type Filter struct { + Configs flags.StringList + Output string + Text bool +} + +// Name implements subcommands.Command.Name. +func (*Filter) Name() string { + return "filter" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Filter) Synopsis() string { + return "Filters findings based on merged configurations." +} + +// Usage implements subcommands.Command.Usage. +func (*Filter) Usage() string { + return `filter [findings...] + + Merges the set of provided configurations and applies to all findings. + The filtered findings are merged and written to the output. + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (f *Filter) SetFlags(fs *flag.FlagSet) { + fs.Var(&f.Configs, "config", "filter configuration files (in JSON format)") + fs.StringVar(&f.Output, "output", "", "findings output (in JSON format by default, unless attached to a terminal)") + fs.BoolVar(&f.Text, "text", false, "force text format in all cases (even not attached to a terminal)") +} + +var ( + cachedFindings = worker.NewCache("findings") // With check.FindingSet. + cachedFiltered = worker.NewCache("filtered") // With check.FindingSet. + cachedConfigs = worker.NewCache("configs") // With config.Config. + cachedFullConfigs = worker.NewCache("compiled") // With config.Config. +) + +func loadFindings(filename string) (check.FindingSet, error) { + v, err := cachedFindings.Lookup([]string{filename}, func() (worker.Sizer, error) { + r, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("unable to open input: %w", err) + } + inputFindings, err := check.ExtractFindingsFrom(r, false /* json */) + if err != nil { + // Seek to reread the file. + if _, err := r.Seek(0, os.SEEK_SET); err != nil { + return nil, fmt.Errorf("unable to reseek in findings %q: %w", filename, err) + } + // Attempt to interpret as a json input. + inputFindings, err = check.ExtractFindingsFrom(r, true /* json */) + if err != nil { + return nil, fmt.Errorf("unable to extract findings from %q: %w", filename, err) + } + } + return inputFindings, nil + }) + if err != nil { + return nil, err + } + return v.(check.FindingSet), nil +} + +func loadConfig(filename string) (*config.Config, error) { + v, err := cachedConfigs.Lookup([]string{filename}, func() (worker.Sizer, error) { + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("unable to open config: %w", err) + } + var newConfig config.Config // For current file. + dec := yaml.NewDecoder(f) + dec.SetStrict(true) + if err := dec.Decode(&newConfig); err != nil { + return nil, fmt.Errorf("unable to decode %q: %w", filename, err) + } + return &newConfig, nil + }) + if err != nil { + return nil, err + } + return v.(*config.Config), nil +} + +func loadConfigs(filenames []string) (*config.Config, error) { + v, err := cachedFullConfigs.Lookup(filenames, func() (worker.Sizer, error) { + config := &config.Config{ + Global: make(config.AnalyzerConfig), + Analyzers: make(map[string]config.AnalyzerConfig), + } + for _, filename := range filenames { + next, err := loadConfig(filename) + if err != nil { + return nil, err + } + config.Merge(next) + } + if err := config.Compile(); err != nil { + return nil, fmt.Errorf("error compiling config: %w", err) + } + return config, nil + }) + if err != nil { + return nil, err + } + return v.(*config.Config), nil +} + +// Execute implements subcommands.Command.Execute. +func (f *Filter) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus { + // Open and merge all configuations. + config, err := loadConfigs(f.Configs) + if err != nil { + return failure("unable to load configurations: %v", err) + } + + // Open the output file. + output, err := openOutput(f.Output, os.Stdout) + if err != nil { + return failure("opening output: %v", err) + } + defer closeOutput(output) + + // Load and filer available findings. + var filteredFindings check.FindingSet + for _, filename := range fs.Args() { + // Note that this applies a caching strategy to the filtered + // findings, because *this is by far the most expensive part of + // evaluation*. The set of findings is large and applying the + // configuration is complex. Therefore, we segment this cache + // on each individual raw findings input file and the + // configuration files. Note that this cache is keyed on all + // the configuration files and each individual raw findings, so + // is guaranteed to be safe. This allows us to reuse the same + // filter result many times over, because e.g. all standard + // library findings will be available to all packages. + v, err := cachedFiltered.Lookup(append(f.Configs, filename), func() (worker.Sizer, error) { + inputFindings, err := loadFindings(filename) + if err != nil { + return nil, err + } + filteredFindings := make(check.FindingSet, 0, len(inputFindings)) + for _, finding := range inputFindings { + if ok := config.ShouldReport(finding); ok { + filteredFindings = append(filteredFindings, finding) + } + } + return filteredFindings, nil + }) + if err != nil { + return failure("unable to load filtered findings from %q: %v", filename, err) + } + filteredFindings = append(filteredFindings, v.(check.FindingSet)...) + } + + // Write the output. + if !f.Text && !isTerminal(output) { + if err := check.WriteFindingsTo(output, filteredFindings, true /* json */); err != nil { + return failure("write findings: %v", err) + } + } else { + for _, finding := range filteredFindings { + fmt.Fprintf(output, "%s\n", finding.String()) + } + } + + // Treat the run as a test. + if (f.Text || isTerminal(output)) && len(filteredFindings) == 0 { + fmt.Fprintf(output, "PASS\n") + return subcommands.ExitSuccess + } + + return subcommands.ExitSuccess +} + +// Render implements subcommands.Command for the "render" command. +type Render struct { + Template string + Output string +} + +// Name implements subcommands.Command.Name. +func (*Render) Name() string { + return "render" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*Render) Synopsis() string { + return "Renders facts about a package using a template." +} + +// Usage implements subcommands.Command.Usage. +func (*Render) Usage() string { + return `render + + Loads all data and renders all known facts. Note that render is not + currently compatible with binary analyzers, and these facts will not + be included (unless they come from dependencies). + +` +} + +// SetFlags implements subcommands.Command.SetFlags. +func (r *Render) SetFlags(fs *flag.FlagSet) { + fs.StringVar(&r.Template, "template", "", "text template file for rendering (required)") + fs.StringVar(&r.Output, "output", "", "output file for rendering (or empty for stdout)") +} + +// Execute implements subcommands.Command.Execute. +func (r *Render) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus { + // Open the output file. + output, err := openOutput(r.Output, os.Stdout) + if err != nil { + return failure("opening output: %v", err) + } + defer closeOutput(output) + + // Open the template file. + t, err := template.ParseFiles(r.Template) + if err != nil { + return failure("loading template: %v", err) + } + + // Process the facts. + facts, err := check.Facts("main", fs.Args()) + if err != nil { + return failure("%v", err) + } + + // Render as a template. + if err := t.Execute(output, facts); err != nil { + return failure("during render: %v", err) + } + + return subcommands.ExitSuccess +} + +// Main is the main entrypoint. +func Main() { + subcommands.Register(&Check{}, "") + subcommands.Register(&Bundle{}, "") + subcommands.Register(&Stdlib{}, "") + subcommands.Register(&Filter{}, "") + subcommands.Register(&Render{}, "") + subcommands.Register(subcommands.HelpCommand(), "") + subcommands.Register(subcommands.FlagsCommand(), "") + worker.Work(func(args []string) int { + return int(subcommands.Execute(context.Background())) + }) +} diff --git a/tools/nogo/config/BUILD b/tools/nogo/config/BUILD new file mode 100644 index 000000000..9d4461327 --- /dev/null +++ b/tools/nogo/config/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +exports_files(["schema.json"]) + +go_library( + name = "config", + srcs = ["config.go"], + visibility = ["//tools/nogo:__subpackages__"], + deps = ["//tools/nogo/check"], +) + +go_test( + name = "config_test", + size = "small", + srcs = ["config_test.go"], + library = ":config", + deps = ["//tools/nogo/check"], +) diff --git a/tools/nogo/config.go b/tools/nogo/config/config.go similarity index 97% rename from tools/nogo/config.go rename to tools/nogo/config/config.go index ee2533610..427cfd8e3 100644 --- a/tools/nogo/config.go +++ b/tools/nogo/config/config.go @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package nogo +// Package config defines a filter configuration for nogo findings. +package config import ( "fmt" "regexp" + + "gvisor.dev/gvisor/tools/nogo/check" ) // GroupName is a named group. type GroupName string -// AnalyzerName is a named analyzer. -type AnalyzerName string - // Group represents a named collection of files. type Group struct { // Name is the short name for the group. @@ -215,7 +215,7 @@ type Config struct { // key for each analyzer is the name of the analyzer. The // value is either a boolean (enable/disable), or a map to // the groups above. - Analyzers map[AnalyzerName]AnalyzerConfig `yaml:"analyzers"` + Analyzers map[string]AnalyzerConfig `yaml:"analyzers"` } // Size implements worker.Sizer.Size. @@ -282,7 +282,7 @@ func (c *Config) Compile() error { } // ShouldReport returns true iff the finding should match the Config. -func (c *Config) ShouldReport(finding Finding) bool { +func (c *Config) ShouldReport(finding check.Finding) bool { fullPos := finding.Position.String() // Find the matching group. diff --git a/tools/nogo/config_test.go b/tools/nogo/config/config_test.go similarity index 92% rename from tools/nogo/config_test.go rename to tools/nogo/config/config_test.go index 685cffbec..cbd2845f2 100644 --- a/tools/nogo/config_test.go +++ b/tools/nogo/config/config_test.go @@ -11,11 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License.package nogo -package nogo + +package config import ( "go/token" "testing" + + "gvisor.dev/gvisor/tools/nogo/check" ) // TestShouldReport validates the suppression behavior of Config.ShouldReport. @@ -50,7 +53,7 @@ func TestShouldReport(t *testing.T) { // Omitting default-disabled-omitted-from-global here // has no effect on configuration below. }, - Analyzers: map[AnalyzerName]AnalyzerConfig{ + Analyzers: map[string]AnalyzerConfig{ "analyzer-suppressions": AnalyzerConfig{ // Suppress some. "default-enabled": &ItemConfig{ @@ -73,12 +76,12 @@ func TestShouldReport(t *testing.T) { cases := []struct { name string - finding Finding + finding check.Finding want bool }{ { name: "enabled", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "default-enabled/file.go", @@ -92,7 +95,7 @@ func TestShouldReport(t *testing.T) { }, { name: "ungrouped", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "ungrouped/file.go", @@ -106,7 +109,7 @@ func TestShouldReport(t *testing.T) { }, { name: "suppressed", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "default-enabled/file.go", @@ -120,7 +123,7 @@ func TestShouldReport(t *testing.T) { }, { name: "excluded", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "default-enabled/excluded.go", @@ -134,7 +137,7 @@ func TestShouldReport(t *testing.T) { }, { name: "disabled", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "default-disabled/file.go", @@ -148,7 +151,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer suppressed", - finding: Finding{ + finding: check.Finding{ Category: "analyzer-suppressions", Position: token.Position{ Filename: "default-enabled/file.go", @@ -162,7 +165,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer suppressed not global", - finding: Finding{ + finding: check.Finding{ // Doesn't apply outside of analyzer-suppressions. Category: "foo", Position: token.Position{ @@ -177,7 +180,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer suppressed grouped", - finding: Finding{ + finding: check.Finding{ Category: "analyzer-suppressions", Position: token.Position{ // Doesn't apply outside of default-enabled. @@ -192,7 +195,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer excluded", - finding: Finding{ + finding: check.Finding{ Category: "analyzer-suppressions", Position: token.Position{ Filename: "default-enabled/limited-exclude.go", @@ -206,7 +209,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer excluded not global", - finding: Finding{ + finding: check.Finding{ // Doesn't apply outside of analyzer-suppressions. Category: "foo", Position: token.Position{ @@ -221,7 +224,7 @@ func TestShouldReport(t *testing.T) { }, { name: "analyzer excluded grouped", - finding: Finding{ + finding: check.Finding{ Category: "analyzer-suppressions", Position: token.Position{ // Doesn't apply outside of default-enabled. @@ -236,7 +239,7 @@ func TestShouldReport(t *testing.T) { }, { name: "disabled-omitted", - finding: Finding{ + finding: check.Finding{ Category: "foo", Position: token.Position{ Filename: "default-disabled-omitted-from-global/file.go", @@ -250,7 +253,7 @@ func TestShouldReport(t *testing.T) { }, { name: "default enabled applies to customized analyzer", - finding: Finding{ + finding: check.Finding{ Category: "enabled-for-default-disabled", Position: token.Position{ Filename: "default-enabled/file.go", @@ -264,7 +267,7 @@ func TestShouldReport(t *testing.T) { }, { name: "default overridden in customized analyzer", - finding: Finding{ + finding: check.Finding{ Category: "enabled-for-default-disabled", Position: token.Position{ Filename: "default-disabled/file.go", @@ -278,7 +281,7 @@ func TestShouldReport(t *testing.T) { }, { name: "default overridden in customized analyzer even when omitted from global", - finding: Finding{ + finding: check.Finding{ Category: "enabled-for-default-disabled", Position: token.Position{ Filename: "default-disabled-omitted-from-global/file.go", diff --git a/tools/nogo/config-schema.json b/tools/nogo/config/schema.json similarity index 100% rename from tools/nogo/config-schema.json rename to tools/nogo/config/schema.json diff --git a/tools/nogo/defs.bzl b/tools/nogo/defs.bzl index bc0d6874f..67dfa8b4b 100644 --- a/tools/nogo/defs.bzl +++ b/tools/nogo/defs.bzl @@ -29,7 +29,6 @@ NogoTargetInfo = provider( fields = { "goarch": "the build architecture (GOARCH)", "goos": "the build OS target (GOOS)", - "worker_debug": "transitive debugging", }, ) @@ -37,7 +36,6 @@ def _nogo_target_impl(ctx): return [NogoTargetInfo( goarch = ctx.attr.goarch, goos = ctx.attr.goos, - worker_debug = ctx.attr.worker_debug, )] nogo_target = go_rule( @@ -52,58 +50,6 @@ nogo_target = go_rule( doc = "the Go OS target (propagated to other rules).", mandatory = True, ), - "worker_debug": attr.bool( - doc = "whether worker debugging should be enabled.", - default = False, - ), - }, -) - -def _nogo_objdump_tool_impl(ctx): - # Construct the magic dump command. - # - # Note that in some cases, the input is being fed into the tool via stdin. - # Unfortunately, the Go objdump tool expects to see a seekable file [1], so - # we need the tool to handle this case by creating a temporary file. - # - # [1] https://github.com/golang/go/issues/41051 - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - env_prefix = " ".join(["%s=%s" % (key, value) for (key, value) in go_ctx.env.items()]) - dumper = ctx.actions.declare_file(ctx.label.name) - ctx.actions.write(dumper, "\n".join([ - "#!/bin/bash", - "set -euo pipefail", - "if [[ $# -eq 0 ]]; then", - " T=$(mktemp -u -t libXXXXXX.a)", - " cat /dev/stdin > ${T}", - "else", - " T=$1;", - "fi", - "%s %s tool objdump ${T}" % ( - env_prefix, - go_ctx.go.path, - ), - "if [[ $# -eq 0 ]]; then", - " rm -rf ${T}", - "fi", - "", - ]), is_executable = True) - - # Include the full runfiles. - return [DefaultInfo( - runfiles = ctx.runfiles(files = go_ctx.runfiles.to_list()), - executable = dumper, - )] - -nogo_objdump_tool = go_rule( - rule, - implementation = _nogo_objdump_tool_impl, - attrs = { - "_target": attr.label( - default = "//tools/nogo:target", - cfg = "target", - ), }, ) @@ -117,66 +63,55 @@ NogoStdlibInfo = provider( ) def _nogo_stdlib_impl(ctx): - # Build the standard library facts. - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - facts = ctx.actions.declare_file(ctx.label.name + ".facts") - raw_findings = ctx.actions.declare_file(ctx.label.name + ".raw_findings") - config = struct( - Srcs = [f.path for f in go_ctx.stdlib_srcs], - GOOS = go_ctx.goos, - GOARCH = go_ctx.goarch, - BuildTags = go_ctx.gotags, - ) - config_file = ctx.actions.declare_file(ctx.label.name + ".cfg") - ctx.actions.write(config_file, config.to_json()) + # Build the configuration for the stdlib. + go_ctx, args, inputs, raw_findings = _nogo_config(ctx, deps = []) + + # Build the analyzer command. + facts_file = ctx.actions.declare_file(ctx.label.name + ".facts") + findings_file = ctx.actions.declare_file(ctx.label.name + ".raw_findings") args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") ctx.actions.write( output = args_file, - content = "\n".join(go_ctx.nogo_args + [ - "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, - "-stdlib=%s" % config_file.path, - "-findings=%s" % raw_findings.path, - "-facts=%s" % facts.path, - ]), + content = "\n".join(args + [ + "bundle", + "-findings=%s" % findings_file.path, + "-facts=%s" % facts_file.path, + "-root=.*?/src/", + ] + [f.path for f in go_ctx.stdlib_srcs]), ) ctx.actions.run( - inputs = [config_file] + go_ctx.stdlib_srcs + [args_file], - outputs = [facts, raw_findings], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), - executable = ctx.files._check[0], + # For the standard library, we need to include the full set of Go + # sources in the inputs. + inputs = inputs + go_ctx.stdlib_srcs + [args_file], + outputs = [facts_file, findings_file], + tools = depset(go_ctx.runfiles.to_list() + ctx.files._nogo), + executable = ctx.files._nogo[0], + env = go_ctx.env, mnemonic = "GoStandardLibraryAnalysis", # Note that this does not support work execution currently. There is an # issue with stdout pollution that is not yet resolved, so this is kept # as a separate menomic. progress_message = "Analyzing Go Standard Library", - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], + arguments = ["@%s" % args_file.path], ) # Return the stdlib facts as output. return [NogoStdlibInfo( - facts = facts, - raw_findings = raw_findings, + facts = facts_file, + raw_findings = raw_findings + [findings_file], ), DefaultInfo( # Declare the facts and findings as default outputs. This is not # strictly required, but ensures that the target still perform analysis # when built directly rather than just indirectly via a nogo_test. - files = depset([facts, raw_findings]), + files = depset([facts_file, findings_file]), )] nogo_stdlib = go_rule( rule, implementation = _nogo_stdlib_impl, attrs = { - "_check": attr.label( - default = "//tools/nogo/check:check", - cfg = "host", - ), - "_objdump_tool": attr.label( - default = "//tools/nogo:objdump_tool", + "_nogo": attr.label( + default = "//tools/nogo:nogo", cfg = "host", ), "_target": attr.label( @@ -204,20 +139,95 @@ NogoInfo = provider( ) def _select_objfile(files): - """Returns (.a file, .x file, is_archive). + """Returns (.a file, .x file). - If no .a file is available, then the first .x file will be returned + If no .x file is available, then the first .x file will be returned instead, and vice versa. If neither are available, then the first provided file will be returned.""" a_files = [f for f in files if f.path.endswith(".a")] x_files = [f for f in files if f.path.endswith(".x")] if not len(x_files) and not len(a_files): - return (files[0], files[0], False) + if not len(files): + return (None, None) + return (files[0], files[0]) if not len(x_files): x_files = a_files if not len(a_files): a_files = x_files - return a_files[0], x_files[0], True + return a_files[0], x_files[0] + +def _nogo_config(ctx, deps): + # Build a configuration for the given set of deps. This is most basic + # configuration and is used by the stdlib. For a more complete config, the + # _nogo_package_config function may be used. + # + # Returns (go_ctx, args, inputs, raw_findings). + nogo_target_info = ctx.attr._target[NogoTargetInfo] + go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) + args = go_ctx.nogo_args + [ + "-go=%s" % go_ctx.go.path, + "-GOOS=%s" % go_ctx.goos, + "-GOARCH=%s" % go_ctx.goarch, + "-tags=%s" % (",".join(go_ctx.gotags)), + ] + inputs = [] + raw_findings = [] + for dep in deps: + # There will be no file attribute set for all transitive dependencies + # that are not go_library or go_binary rules, such as a proto rules. + # This is handled by the ctx.rule.kind check above. + info = dep[NogoInfo] + if not hasattr(info, "facts"): + continue + + # Configure where to find the binary & fact files. Note that this will + # use .x and .a regardless of whether this is a go_binary rule, since + # these dependencies must be go_library rules. + _, x_file = _select_objfile(info.binaries) + args.append("-import=%s=%s" % (info.importpath, x_file.path)) + args.append("-facts=%s=%s" % (info.importpath, info.facts.path)) + + # Collect all findings; duplicates are resolved at the end. + raw_findings.extend(info.raw_findings) + + # Ensure the above are available as inputs. + inputs.append(x_file) + inputs.append(info.facts) + + return (go_ctx, args, inputs, raw_findings) + +def _nogo_package_config(ctx, deps, importpath = None, target = None): + # See _nogo_config. This includes package details. + # + # Returns (go_ctx, args, inputs, raw_findings). + go_ctx, args, inputs, raw_findings = _nogo_config(ctx, deps) + + # Add the module itself, for the type sanity check. This applies only to + # the libraries, and not binaries or tests. + binaries = [] + if target != None: + binaries.extend(target.files.to_list()) + target_objfile, target_xfile = _select_objfile(binaries) + if target_objfile != None: + inputs.append(target_objfile) + if target_xfile != None: + inputs.append(target_xfile) + args.append("-import=%s=%s" % (importpath, target_xfile.path)) + + # Add the standard library facts. + stdlib_info = ctx.attr._nogo_stdlib[NogoStdlibInfo] + stdlib_facts = stdlib_info.facts + inputs.append(stdlib_facts) + args.append("-bundle=%s" % stdlib_facts.path) + + # Flatten all findings from all dependencies. + # + # This is done because all the filtering must be done at the + # top-level nogo_test to dynamically apply a configuration. + # This does not actually add any additional work here, but + # will simply propagate the full list of files. + raw_findings = stdlib_info.raw_findings + depset(raw_findings).to_list() + return go_ctx, args, inputs, raw_findings def _nogo_aspect_impl(target, ctx): # If this is a nogo rule itself (and not the shadow of a go_library or @@ -252,123 +262,47 @@ def _nogo_aspect_impl(target, ctx): if hasattr(info, "deps"): deps = deps + info.deps - # Start with all target files and srcs as input. - binaries = target.files.to_list() - inputs = binaries + srcs - - # Generate a shell script that dumps the binary. Annoyingly, this seems - # necessary as the context in which a run_shell command runs does not seem - # to cleanly allow us redirect stdout to the actual output file. Perhaps - # I'm missing something here, but the intermediate script does work. - target_objfile, target_xfile, has_objfile = _select_objfile(binaries) - inputs.append(target_objfile) - # Extract the importpath for this package. if ctx.rule.kind == "go_test": - # If this is a test, then it will not be imported by anything else. - # We can safely set the importapth to just "test". Note that this - # is necessary if the library also imports the core library (in - # addition to including the sources directly), which happens in - # some complex cases (seccomp_victim). importpath = "test" else: importpath = go_importpath(target) - # Collect all info from shadow dependencies. - fact_map = dict() - import_map = dict() - all_raw_findings = [] - for dep in deps: - # There will be no file attribute set for all transitive dependencies - # that are not go_library or go_binary rules, such as a proto rules. - # This is handled by the ctx.rule.kind check above. - info = dep[NogoInfo] - if not hasattr(info, "facts"): - continue + # Build a complete configuration, referring to the library rule. + go_ctx, args, inputs, raw_findings = _nogo_package_config(ctx, deps, importpath = importpath, target = target) - # Configure where to find the binary & fact files. Note that this will - # use .x and .a regardless of whether this is a go_binary rule, since - # these dependencies must be go_library rules. - _, x_file, _ = _select_objfile(info.binaries) - import_map[info.importpath] = x_file.path - fact_map[info.importpath] = info.facts.path - - # Collect all findings; duplicates are resolved at the end. - all_raw_findings.extend(info.raw_findings) - - # Ensure the above are available as inputs. - inputs.append(info.facts) - inputs += info.binaries - - # Add the module itself, for the type sanity check. This applies only to - # the libraries, and not binaries or tests. - if has_objfile: - import_map[importpath] = target_xfile.path - - # Add the standard library facts. - stdlib_info = ctx.attr._nogo_stdlib[NogoStdlibInfo] - stdlib_facts = stdlib_info.facts - inputs.append(stdlib_facts) - - # The nogo tool operates on a configuration serialized in JSON format. - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - facts = ctx.actions.declare_file(target.label.name + ".facts") - raw_findings = ctx.actions.declare_file(target.label.name + ".raw_findings") - config = struct( - ImportPath = importpath, - GoFiles = [src.path for src in srcs if src.path.endswith(".go")], - NonGoFiles = [src.path for src in srcs if not src.path.endswith(".go")], - GOOS = go_ctx.goos, - GOARCH = go_ctx.goarch, - BuildTags = go_ctx.gotags, - FactMap = fact_map, - ImportMap = import_map, - StdlibFacts = stdlib_facts.path, - ) - config_file = ctx.actions.declare_file(target.label.name + ".cfg") - ctx.actions.write(config_file, config.to_json()) - inputs.append(config_file) + # Build the argument file, and the runner. args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") + facts_file = ctx.actions.declare_file(ctx.label.name + ".facts") + findings_file = ctx.actions.declare_file(ctx.label.name + ".findings") ctx.actions.write( output = args_file, - content = "\n".join(go_ctx.nogo_args + [ - "-binary=%s" % target_objfile.path, - "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, - "-package=%s" % config_file.path, - "-findings=%s" % raw_findings.path, - "-facts=%s" % facts.path, - ]), + content = "\n".join(args + [ + "check", + "-findings=%s" % findings_file.path, + "-facts=%s" % facts_file.path, + "-package=%s" % importpath, + ] + [src.path for src in srcs]), ) ctx.actions.run( - inputs = inputs + [args_file], - outputs = [facts, raw_findings], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), - executable = ctx.files._check[0], + inputs = inputs + srcs + [args_file], + outputs = [findings_file, facts_file], + tools = depset(go_ctx.runfiles.to_list() + ctx.files._nogo), + executable = ctx.files._nogo[0], + env = go_ctx.env, mnemonic = "GoStaticAnalysis", progress_message = "Analyzing %s" % target.label, execution_requirements = {"supports-workers": "1"}, - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], + arguments = ["@%s" % args_file.path], ) - # Flatten all findings from all dependencies. - # - # This is done because all the filtering must be done at the - # top-level nogo_test to dynamically apply a configuration. - # This does not actually add any additional work here, but - # will simply propagate the full list of files. - all_raw_findings = [stdlib_info.raw_findings] + depset(all_raw_findings).to_list() + [raw_findings] - # Return the package facts as output. return [ NogoInfo( - facts = facts, - raw_findings = all_raw_findings, + facts = facts_file, + raw_findings = raw_findings + [findings_file], importpath = importpath, - binaries = binaries, + binaries = target.files.to_list(), srcs = srcs, deps = deps, ), @@ -383,12 +317,8 @@ nogo_aspect = go_rule( "embed", ], attrs = { - "_check": attr.label( - default = "//tools/nogo/check:check", - cfg = "host", - ), - "_objdump_tool": attr.label( - default = "//tools/nogo:objdump_tool", + "_nogo": attr.label( + default = "//tools/nogo:nogo", cfg = "host", ), "_target": attr.label( @@ -420,23 +350,21 @@ def _nogo_test_impl(ctx): ctx.actions.write( output = args_file, content = "\n".join( - ["-input=%s" % f.path for f in raw_findings] + + ["filter"] + ["-config=%s" % f.path for f in config_srcs] + - ["-output=%s" % findings.path], + ["-output=%s" % findings.path] + + [f.path for f in raw_findings], ), ) ctx.actions.run( inputs = raw_findings + ctx.files.srcs + config_srcs + [args_file], outputs = [findings], - tools = depset(ctx.files._filter), - executable = ctx.files._filter[0], + tools = depset(ctx.files._nogo), + executable = ctx.files._nogo[0], mnemonic = "GoStaticAnalysis", progress_message = "Generating %s" % ctx.label, execution_requirements = {"supports-workers": "1"}, - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], + arguments = ["@%s" % args_file.path], ) # Build a runner that checks the filtered facts. @@ -447,7 +375,7 @@ def _nogo_test_impl(ctx): runner = ctx.actions.declare_file(ctx.label.name) runner_content = [ "#!/bin/bash", - "exec %s -check -input=%s" % (ctx.files._filter[0].short_path, findings.short_path), + "exec %s filter -text %s" % (ctx.files._nogo[0].short_path, findings.short_path), "", ] ctx.actions.write(runner, "\n".join(runner_content), is_executable = True) @@ -456,7 +384,7 @@ def _nogo_test_impl(ctx): # The runner just executes the filter again, on the # newly generated filtered findings. We still need # the filter tool as part of our runfiles, however. - runfiles = ctx.runfiles(files = ctx.files._filter + [findings]), + runfiles = ctx.runfiles(files = ctx.files._nogo + [findings]), executable = runner, ), OutputGroupInfo( # Propagate the filtered filters, for consumption by @@ -481,11 +409,14 @@ nogo_test = rule( allow_files = True, doc = "Relevant src files. This is ignored except to make the nogo_test directly affected by the files.", ), + "_nogo": attr.label( + default = "//tools/nogo:nogo", + cfg = "host", + ), "_target": attr.label( default = "//tools/nogo:target", cfg = "target", ), - "_filter": attr.label(default = "//tools/nogo/filter:filter"), }, test = True, ) @@ -504,3 +435,74 @@ def _nogo_aspect_tricorder_impl(target, ctx): nogo_aspect_tricorder = aspect( implementation = _nogo_aspect_tricorder_impl, ) + +def _nogo_facts_impl(ctx): + """Extract nogo facts.""" + + # Build a complete configuration. Note that we don't care about the import + # path, since this will generate facts only. We use ctx as the target here, + # since this will refer to ctx.files (which contains no binaries). + go_ctx, args, inputs, _ = _nogo_package_config(ctx, ctx.attr.deps) + + # Build the argument file, and the runner. + args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") + ctx.actions.write( + output = args_file, + content = "\n".join(args + [ + "render", + "-template=%s" % ctx.files.template[0].path, + "-output=%s" % ctx.outputs.output.path, + ] + [src.path for src in ctx.files.srcs]), + ) + inputs += ctx.files.template + ctx.actions.run( + inputs = inputs + ctx.files.srcs + ctx.files.template + [args_file], + outputs = [ctx.outputs.output], + tools = depset(go_ctx.runfiles.to_list() + ctx.files._nogo), + executable = ctx.files._nogo[0], + env = go_ctx.env, + mnemonic = "GoStaticAnalysis", + progress_message = "Generating %s" % ctx.label, + arguments = ["@%s" % args_file.path], + ) + + # Return the output. + return [DefaultInfo(files = depset([ctx.outputs.output]))] + +nogo_facts = go_rule( + rule, + implementation = _nogo_facts_impl, + attrs = { + "srcs": attr.label_list( + allow_files = True, + doc = "Source files to be processed.", + mandatory = True, + ), + "deps": attr.label_list( + aspects = [nogo_aspect], + doc = "Go dependencies to be analyzed.", + ), + "template": attr.label( + allow_files = True, + doc = "Template to be rendered for the output.", + mandatory = True, + ), + "output": attr.output( + doc = "Output file to be rendered.", + mandatory = True, + ), + "_nogo": attr.label( + default = "//tools/nogo:nogo", + cfg = "host", + ), + # See _nogo_aspect, above. + "_nogo_stdlib": attr.label( + default = "//tools/nogo:stdlib", + cfg = "host", + ), + "_target": attr.label( + default = "//tools/nogo:target", + cfg = "target", + ), + }, +) diff --git a/tools/nogo/facts/BUILD b/tools/nogo/facts/BUILD new file mode 100644 index 000000000..755c9aba9 --- /dev/null +++ b/tools/nogo/facts/BUILD @@ -0,0 +1,13 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "facts", + srcs = ["facts.go"], + visibility = ["//tools:__subpackages__"], + deps = [ + "@org_golang_x_tools//go/analysis:go_default_library", + "@org_golang_x_tools//go/types/objectpath:go_default_library", + ], +) diff --git a/tools/nogo/facts/facts.go b/tools/nogo/facts/facts.go new file mode 100644 index 000000000..782058b31 --- /dev/null +++ b/tools/nogo/facts/facts.go @@ -0,0 +1,356 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package facts implements alternate fact types. +package facts + +import ( + "bytes" + "encoding/gob" + "go/types" + "io" + "log" + "reflect" + "sort" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/types/objectpath" +) + +// Writer is used for fact serialization. +type Writer interface { + io.ReaderFrom + io.WriterTo +} + +// item is used for serialiation. +type item struct { + Key string + Value interface{} +} + +// writeItems is an implementation of io.WriterTo.WriteTo. +// +// This will sort the list as a side effect. +func writeItems(w io.Writer, is []item) error { + sort.Slice(is, func(i, j int) bool { + return is[i].Key < is[j].Key + }) + enc := gob.NewEncoder(w) + return enc.Encode(is) +} + +// readItems is an implementation of io.ReaderTo.ReadTo. +func readItems(r io.Reader) (is []item, err error) { + dec := gob.NewDecoder(r) + err = dec.Decode(&is) + return +} + +// Package is a set of facts about a single package. +// +// These use the types.Object as the key because this is canonical. Normally, +// this is canonical only in the context of a single types.Package. However, +// because all imports are shared across all packages, there is a single +// canonical types.Object shared among all packages being analyzed. +type Package struct { + pkg *types.Package + Objects map[types.Object][]analysis.Fact +} + +// NewPackage returns a new set of Package facts. +func NewPackage(pkg *types.Package) *Package { + return &Package{ + pkg: pkg, + Objects: make(map[types.Object][]analysis.Fact), + } +} + +// WriteTo implements io.WriterTo.WriteTo. +func (p *Package) WriteTo(w io.Writer) (int64, error) { + is := make([]item, 0, len(p.Objects)) + for obj, facts := range p.Objects { + var ( + name objectpath.Path + err error + ) + if obj != nil { + name, err = objectpath.For(obj) + } + if err != nil { + continue // Not exported, expected. + } + for _, fact := range facts { + is = append(is, item{ + Key: string(name), + Value: fact, + }) + } + } + if err := writeItems(w, is); err != nil { + return 0, err + } + return 1, nil +} + +// ReadFrom implements io.ReaderFrom.ReadFrom. +func (p *Package) ReadFrom(r io.Reader) (int64, error) { + is, err := readItems(r) + if err != nil { + return 0, err + } + for _, fi := range is { + var ( + obj types.Object + err error + ) + if fi.Key != "" { + obj, err = objectpath.Object(p.pkg, objectpath.Path(fi.Key)) + } + if err != nil { + // This could simply be a fact saved on an unexported + // object. We just suppress this error and ignore it. + continue + } + p.Objects[obj] = append(p.Objects[obj], fi.Value.(analysis.Fact)) + } + return 1, nil +} + +// Size implements worker.Sizer.Size. +func (p *Package) Size() int64 { + total := int64(0) + for _, val := range p.Objects { + total += int64(8) // 8-byte pointer. + total += int64(len(val)) * 16 // 16-bytes per object. + } + return total +} + +// ExportFact exports an object fact. +func (p Package) ExportFact(obj types.Object, ptr analysis.Fact) { + for i, v := range p.Objects[obj] { + if reflect.TypeOf(v) == reflect.TypeOf(ptr) { + // Drop this item from the list. + p.Objects[obj] = append(p.Objects[obj][:i], p.Objects[obj][i+1:]...) + break + } + } + // Append this new fact. + p.Objects[obj] = append(p.Objects[obj], ptr) +} + +// ImportFact imports an object fact. +func (p *Package) ImportFact(obj types.Object, ptr analysis.Fact) bool { + for _, v := range p.Objects[obj] { + if reflect.TypeOf(v) == reflect.TypeOf(ptr) { + // Set the value to the element saved in our facts. + reflect.ValueOf(ptr).Elem().Set(reflect.ValueOf(v).Elem()) + return true + } + } + return false +} + +// Bundle is a set of facts about different packages. This is typically +// used for the standard library, but may be used for e.g. module dependencies. +type Bundle struct { + importer types.Importer + Packages map[string]*Package +} + +// NewBundle returns a new bundle. +func NewBundle(importer types.Importer) *Bundle { + return &Bundle{ + importer: importer, + Packages: make(map[string]*Package), + } +} + +// Size implements worker.Sizer.Size. +func (b *Bundle) Size() int64 { + size := int64(0) + for filename, p := range b.Packages { + size += int64(len(filename)) + size += p.Size() + } + return size +} + +// WriteTo implements io.WriterTo.WriteTo. +func (b *Bundle) WriteTo(w io.Writer) (int64, error) { + is := make([]item, 0, len(b.Packages)) + for pkg, facts := range b.Packages { + if facts == nil { + // Some facts may be omitted for bundles, if there is + // only type information but no source information. We + // omit these completely from the serialized bundle. + continue + } + var buf bytes.Buffer + if _, err := facts.WriteTo(&buf); err != nil { + return 0, err + } + is = append(is, item{ + Key: pkg, + Value: buf.Bytes(), + }) + } + if err := writeItems(w, is); err != nil { + return 0, err + } + return 1, nil +} + +// ReadFrom implements io.ReaderFrom.ReadFrom. +func (b *Bundle) ReadFrom(r io.Reader) (int64, error) { + is, err := readItems(r) + if err != nil { + return 0, err + } + for _, fi := range is { + pkg, err := b.importer.Import(fi.Key) + if err != nil { + // There's nothing that can be done here, but we can + // report the warning at least. This is not expected. + log.Printf("WARNING: lost facts from %q: %v", fi.Key, err) + continue + } + buf := bytes.NewBuffer(fi.Value.([]byte)) + facts := NewPackage(pkg) + if _, err := facts.ReadFrom(buf); err != nil { + return 0, err + } + b.Packages[fi.Key] = facts + } + return 1, nil +} + +// Resolved is a human-readable fact format. +type Resolved map[string]interface{} + +// addRecursively adds a entry to a map recursively. +// +// Precondition: len(names) > 0. +func (r Resolved) addRecursively(names []string, value interface{}) { + start := r + for i := 0; i < len(names)-1; i++ { + m, ok := start[names[i]] + if !ok { + m = make(Resolved) + start[names[i]] = m + } else { + // This may have been used by a conflicting fact. This + // should be rare, but we ensure that the proper name + // itself is used in the scope instead of the fact. + if _, ok = m.(Resolved); !ok { + m = make(Resolved) + start[names[i]] = m + } + } + start = m.(Resolved) + } + if _, ok := start[names[len(names)-1]]; ok { + // Skip, already exists. See above. + return + } + start[names[len(names)-1]] = value +} + +// addObject adds the object with the given name. +func (r Resolved) addObject(names []string, obj types.Object, facts *Package, allFactNames map[reflect.Type]string) { + for _, fact := range facts.Objects[obj] { + v := reflect.ValueOf(fact) + typeName, ok := allFactNames[v.Type()] + if !ok { + continue + } + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + r.addRecursively(append(names, typeName), v.Interface()) + } +} + +// walkObject resolves all objects recursively. +// +// Parent should be empty or end with a period. +func (r Resolved) walkObject(parents []string, obj types.Object, facts *Package, allFactNames map[reflect.Type]string) { + switch x := obj.(type) { + case *types.TypeName: + s := append(parents, x.Name()) + r.addObject(s, obj, facts, allFactNames) + // Skip if just an alias, or if not underlying type. + if x.IsAlias() || x.Type() == nil || x.Type().Underlying() == nil { + break + } + // Recurse to fields if this is a definition. + if structType, ok := x.Type().Underlying().(*types.Struct); ok { + for i := 0; i < structType.NumFields(); i++ { + r.walkObject(s, structType.Field(i), facts, allFactNames) + } + } + case *types.Func: + // Skip if no underlying type. + if x.Type() == nil { + break + } + // Recurse to all parameters. + sig := x.Type().(*types.Signature) + s := parents + if recv := sig.Recv(); recv != nil { + s = append(s, recv.Type().String()) + } + s = append(s, x.Name()) + r.addObject(s, obj, facts, allFactNames) + if params := sig.Params(); params != nil { + for i := 0; i < params.Len(); i++ { + r.walkObject(s, params.At(i), facts, allFactNames) + } + } + if results := sig.Results(); results != nil { + for i := 0; i < results.Len(); i++ { + r.walkObject(s, results.At(i), facts, allFactNames) + } + } + default: + r.addObject(append(parents, obj.Name()), obj, facts, allFactNames) + } +} + +// walkScope recursively resolves a scope. +func (r Resolved) walkScope(parents []string, scope *types.Scope, facts *Package, allFactNames map[reflect.Type]string) { + for _, name := range scope.Names() { + r.walkObject(parents, scope.Lookup(name), facts, allFactNames) + } +} + +// Resolve resolves all object facts. +func Resolve(pkg *types.Package, localFacts *Package, allFacts *Bundle, allFactNames map[reflect.Type]string) Resolved { + // Populate the tree. Allocating this slice up front prevents + // allocation during name resolution. We allow for up to 64 names + // without allocating a new backing array. + r := make(Resolved) + names := make([]string, 0, 64) + r.walkScope(names, pkg.Scope(), localFacts, allFactNames) + for _, importPkg := range pkg.Imports() { + importFacts := allFacts.Packages[importPkg.Path()] + r.walkScope(append(names, "import", importPkg.Name()), importPkg.Scope(), importFacts, allFactNames) + } + return r +} + +func init() { + gob.Register((*item)(nil)) +} diff --git a/tools/nogo/filter/BUILD b/tools/nogo/filter/BUILD deleted file mode 100644 index e3049521e..000000000 --- a/tools/nogo/filter/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "filter", - srcs = ["main.go"], - nogo = False, - visibility = ["//visibility:public"], - deps = [ - "//tools/nogo", - "//tools/worker", - "@in_gopkg_yaml_v2//:go_default_library", - ], -) diff --git a/tools/nogo/filter/main.go b/tools/nogo/filter/main.go deleted file mode 100644 index 4a925d03c..000000000 --- a/tools/nogo/filter/main.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary filter is the filters and reports nogo findings. -package main - -import ( - "bytes" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - yaml "gopkg.in/yaml.v2" - "gvisor.dev/gvisor/tools/nogo" - "gvisor.dev/gvisor/tools/worker" -) - -type stringList []string - -func (s *stringList) String() string { - return strings.Join(*s, ",") -} - -func (s *stringList) Set(value string) error { - *s = append(*s, value) - return nil -} - -var ( - inputFiles stringList - configFiles stringList - outputFile string - showConfig bool - check bool -) - -func init() { - flag.Var(&inputFiles, "input", "findings input files (gob format)") - flag.StringVar(&outputFile, "output", "", "findings output file (json format)") - flag.Var(&configFiles, "config", "findings configuration files") - flag.BoolVar(&showConfig, "show-config", false, "dump configuration only") - flag.BoolVar(&check, "check", false, "assume input is in json format") -} - -func main() { - worker.Work(run) -} - -var ( - cachedFindings = worker.NewCache("findings") // With nogo.FindingSet. - cachedFiltered = worker.NewCache("filtered") // With nogo.FindingSet. - cachedConfigs = worker.NewCache("configs") // With nogo.Config. - cachedFullConfigs = worker.NewCache("compiled") // With nogo.Config. -) - -func loadFindings(filename string) nogo.FindingSet { - return cachedFindings.Lookup([]string{filename}, func() worker.Sizer { - r, err := os.Open(filename) - if err != nil { - log.Fatalf("unable to open input %q: %v", filename, err) - } - inputFindings, err := nogo.ExtractFindingsFrom(r, check /* json */) - if err != nil { - log.Fatalf("unable to extract findings from %s: %v", filename, err) - } - return inputFindings - }).(nogo.FindingSet) -} - -func loadConfig(filename string) *nogo.Config { - return cachedConfigs.Lookup([]string{filename}, func() worker.Sizer { - content, err := ioutil.ReadFile(filename) - if err != nil { - log.Fatalf("unable to read %s: %v", filename, err) - } - var newConfig nogo.Config // For current file. - dec := yaml.NewDecoder(bytes.NewBuffer(content)) - dec.SetStrict(true) - if err := dec.Decode(&newConfig); err != nil { - log.Fatalf("unable to decode %s: %v", filename, err) - } - if showConfig { - content, err := yaml.Marshal(&newConfig) - if err != nil { - log.Fatalf("error marshalling config: %v", err) - } - fmt.Fprintf(os.Stdout, "Loaded configuration from %s:\n%s\n", filename, string(content)) - } - return &newConfig - }).(*nogo.Config) -} - -func loadConfigs(filenames []string) *nogo.Config { - return cachedFullConfigs.Lookup(filenames, func() worker.Sizer { - config := &nogo.Config{ - Global: make(nogo.AnalyzerConfig), - Analyzers: make(map[nogo.AnalyzerName]nogo.AnalyzerConfig), - } - for _, filename := range configFiles { - config.Merge(loadConfig(filename)) - if showConfig { - mergedBytes, err := yaml.Marshal(config) - if err != nil { - log.Fatalf("error marshalling config: %v", err) - } - fmt.Fprintf(os.Stdout, "Merged configuration:\n%s\n", string(mergedBytes)) - } - } - if err := config.Compile(); err != nil { - log.Fatalf("error compiling config: %v", err) - } - return config - }).(*nogo.Config) -} - -func run([]string) int { - // Open and merge all configuations. - config := loadConfigs(configFiles) - if showConfig { - return 0 - } - - // Load and filer available findings. - var filteredFindings []nogo.Finding - for _, filename := range inputFiles { - // Note that this applies a caching strategy to the filtered - // findings, because *this is by far the most expensive part of - // evaluation*. The set of findings is large and applying the - // configuration is complex. Therefore, we segment this cache - // on each individual raw findings input file and the - // configuration files. Note that this cache is keyed on all - // the configuration files and each individual raw findings, so - // is guaranteed to be safe. This allows us to reuse the same - // filter result many times over, because e.g. all standard - // library findings will be available to all packages. - filteredFindings = append(filteredFindings, - cachedFiltered.Lookup(append(configFiles, filename), func() worker.Sizer { - inputFindings := loadFindings(filename) - filteredFindings := make(nogo.FindingSet, 0, len(inputFindings)) - for _, finding := range inputFindings { - if ok := config.ShouldReport(finding); ok { - filteredFindings = append(filteredFindings, finding) - } - } - return filteredFindings - }).(nogo.FindingSet)...) - } - - // Write the output (if required). - // - // If the outputFile is specified, then we exit here. Otherwise, - // we continue to write to stdout and treat like a test. - // - // Note that the output of the filter is always json, which is - // human readable and the format that is consumed by tricorder. - if outputFile != "" { - w, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - log.Fatalf("unable to open output file %q: %v", outputFile, err) - } - if err := nogo.WriteFindingsTo(w, filteredFindings, true /* json */); err != nil { - log.Fatalf("unable to write findings: %v", err) - } - return 0 - } - - // Treat the run as a test. - if len(filteredFindings) == 0 { - fmt.Fprintf(os.Stdout, "PASS\n") - return 0 - } - for _, finding := range filteredFindings { - fmt.Fprintf(os.Stdout, "%s\n", finding.String()) - } - return 1 -} diff --git a/tools/nogo/objdump/BUILD b/tools/nogo/flags/BUILD similarity index 64% rename from tools/nogo/objdump/BUILD rename to tools/nogo/flags/BUILD index da56efdf7..87eccd32b 100644 --- a/tools/nogo/objdump/BUILD +++ b/tools/nogo/flags/BUILD @@ -3,8 +3,8 @@ load("//tools:defs.bzl", "go_library") package(licenses = ["notice"]) go_library( - name = "objdump", - srcs = ["objdump.go"], - nogo = False, + name = "flags", + srcs = ["flags.go"], visibility = ["//tools:__subpackages__"], + deps = ["//runsc/flag"], ) diff --git a/tools/nogo/flags/flags.go b/tools/nogo/flags/flags.go new file mode 100644 index 000000000..0b85b2ec9 --- /dev/null +++ b/tools/nogo/flags/flags.go @@ -0,0 +1,143 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package flags contains globally-visible flags. +package flags + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "sync" + + "gvisor.dev/gvisor/runsc/flag" +) + +var ( + // Go location. + Go string + + // GOOS defines the GOOS for analysis. + GOOS string + + // GOARCH defines the GOARCH for analysis. + GOARCH string + + // BuildTags defines the set of build tags for analysis. Note that + // while this may also be a StringList, it is kept as an explicit + // comma-separated list in order to build the standard flag. + BuildTags string + + // ImportMap defines all binary input files. + ImportMap = StringMap{} + + // FactMap defines all fact input files. + FactMap = StringMap{} + + // Bundles define fact bundles. This is typically used to contain the + // inputs for the standard library. + Bundles StringList +) + +func init() { + flag.StringVar(&Go, "go", "go", "command used to invoke 'go tool'") + flag.StringVar(&GOOS, "GOOS", runtime.GOOS, "GOOS for analysis") + flag.StringVar(&GOARCH, "GOARCH", runtime.GOARCH, "GOARCH for analysis") + flag.StringVar(&BuildTags, "tags", "", "comma-separated build tags") + flag.Var(&ImportMap, "import", "map of import paths to binaries") + flag.Var(&FactMap, "facts", "map of import paths to facts") + flag.Var(&Bundles, "bundle", "list of fact bundles") +} + +// StringList is a list of strings. +type StringList []string + +// String implements fmt.Stringer.String. +func (s *StringList) String() string { + return strings.Join((*s), ",") +} + +// Set implements flag.Value.Set. +func (s *StringList) Set(value string) error { + (*s) = append((*s), value) + return nil +} + +// Get implements flag.Value.Get. +func (s *StringList) Get() interface{} { + return *s +} + +// StringMap is a string to string map. +type StringMap map[string]string + +// String implements fmt.Stringer.String. +func (s *StringMap) String() string { + parts := make([]string, 0, len(*s)) + for k, v := range *s { + parts = append(parts, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(parts, ",") +} + +// Get implements flag.Value.Get. +func (s *StringMap) Get() interface{} { + return *s +} + +// Set implements flag.Value.Set. +func (s *StringMap) Set(value string) error { + if (*s) == nil { + (*s) = make(map[string]string) + } + parts := strings.SplitN(value, "=", 2) + if len(parts) != 2 { + // We specify the flag as -x=y=z. This string missed the second '='. + return fmt.Errorf("invalid format: expected at least one '=' in flag value, got %q", value) + } + (*s)[parts[0]] = parts[1] + return nil +} + +var ( + envOnce sync.Once + envErr error + envMap = map[string]string{} +) + +// Env gets a Go environment value. +func Env(value string) (string, error) { + if v := os.Getenv(value); v != "" { + return v, nil + } + envOnce.Do(func() { + // Pull the go environment. + cmd := exec.Command(Go, "env", "-json") + output, err := cmd.Output() + if err != nil { + envErr = fmt.Errorf("error executing 'go env -json': %w", err) + return + } + dec := json.NewDecoder(bytes.NewBuffer(output)) + if err := dec.Decode(&envMap); err != nil { + envErr = fmt.Errorf("error decoding 'go env -json': %w", err) + return + } + }) + return envMap[value], envErr // From above. +} diff --git a/tools/nogo/main.go b/tools/nogo/main.go new file mode 100644 index 000000000..93254f920 --- /dev/null +++ b/tools/nogo/main.go @@ -0,0 +1,24 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary nogo performs static analysis. +package main + +import ( + "gvisor.dev/gvisor/tools/nogo/cli" +) + +func main() { + cli.Main() +} diff --git a/tools/nogo/nogo.go b/tools/nogo/nogo.go deleted file mode 100644 index a96cb400a..000000000 --- a/tools/nogo/nogo.go +++ /dev/null @@ -1,673 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package nogo implements binary analysis similar to bazel's nogo, -// or the unitchecker package. It exists in order to provide additional -// facilities for analysis, namely plumbing through the output from -// dumping the generated binary (to analyze actual produced code). -package nogo - -import ( - "bytes" - "encoding/gob" - "errors" - "fmt" - "go/ast" - "go/build" - "go/parser" - "go/token" - "go/types" - "io" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "reflect" - "sort" - "strings" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/internal/facts" - "golang.org/x/tools/go/gcexportdata" - "golang.org/x/tools/go/types/objectpath" - - // Special case: flags live here and change overall behavior. - "gvisor.dev/gvisor/tools/nogo/objdump" - "gvisor.dev/gvisor/tools/worker" -) - -// StdlibConfig is serialized as the configuration. -// -// This contains everything required for stdlib analysis. -type StdlibConfig struct { - Srcs []string - GOOS string - GOARCH string - BuildTags []string - ReleaseTags []string // Use build.Default if nil. -} - -// PackageConfig is serialized as the configuration. -// -// This contains everything required for single package analysis. -type PackageConfig struct { - ImportPath string - GoFiles []string - NonGoFiles []string - BuildTags []string - ReleaseTags []string // Use build.Default if nil. - GOOS string - GOARCH string - ImportMap map[string]string - FactMap map[string]string - StdlibFacts string -} - -// loader is a fact-loader function. -type loader func(string) ([]byte, error) - -// saver is a fact-saver function. -type saver func([]byte) error - -// stdlibFact is used for serialiation. -type stdlibFact struct { - Package string - Facts []byte -} - -// stdlibFacts is a set of standard library facts. -type stdlibFacts map[string][]byte - -// Size implements worker.Sizer.Size. -func (sf stdlibFacts) Size() int64 { - size := int64(0) - for filename, data := range sf { - size += int64(len(filename)) - size += int64(len(data)) - } - return size -} - -// EncodeTo serializes stdlibFacts. -func (sf stdlibFacts) EncodeTo(w io.Writer) error { - stdlibFactsSorted := make([]stdlibFact, 0, len(sf)) - for pkg, facts := range sf { - stdlibFactsSorted = append(stdlibFactsSorted, stdlibFact{ - Package: pkg, - Facts: facts, - }) - } - sort.Slice(stdlibFactsSorted, func(i, j int) bool { - return stdlibFactsSorted[i].Package < stdlibFactsSorted[j].Package - }) - enc := gob.NewEncoder(w) - if err := enc.Encode(stdlibFactsSorted); err != nil { - return err - } - return nil -} - -// DecodeFrom deserializes stdlibFacts. -func (sf stdlibFacts) DecodeFrom(r io.Reader) error { - var stdlibFactsSorted []stdlibFact - dec := gob.NewDecoder(r) - if err := dec.Decode(&stdlibFactsSorted); err != nil { - return err - } - for _, stdlibFact := range stdlibFactsSorted { - sf[stdlibFact.Package] = stdlibFact.Facts - } - return nil -} - -var ( - // cachedFacts caches by file (just byte data). - cachedFacts = worker.NewCache("facts") - - // stdlibCachedFacts caches the standard library (stdlibFacts). - stdlibCachedFacts = worker.NewCache("stdlib") -) - -// factLoader loads facts. -func (c *PackageConfig) factLoader(path string) (data []byte, err error) { - filename, ok := c.FactMap[path] - if ok { - cb := cachedFacts.Lookup([]string{filename}, func() worker.Sizer { - data, readErr := ioutil.ReadFile(filename) - if readErr != nil { - err = fmt.Errorf("error loading %q: %w", filename, readErr) - return nil - } - return worker.CacheBytes(data) - }) - if cb != nil { - return []byte(cb.(worker.CacheBytes)), err - } - return nil, err - } - cb := stdlibCachedFacts.Lookup([]string{c.StdlibFacts}, func() worker.Sizer { - r, openErr := os.Open(c.StdlibFacts) - if openErr != nil { - err = fmt.Errorf("error loading stdlib facts from %q: %w", c.StdlibFacts, openErr) - return nil - } - defer r.Close() - sf := make(stdlibFacts) - if readErr := sf.DecodeFrom(r); readErr != nil { - err = fmt.Errorf("error loading stdlib facts: %w", readErr) - return nil - } - return sf - }) - if cb != nil { - return (cb.(stdlibFacts))[path], err - } - return nil, err -} - -// shouldInclude indicates whether the file should be included. -// -// NOTE: This does only basic parsing of tags. -func (c *PackageConfig) shouldInclude(path string) (bool, error) { - ctx := build.Default - ctx.GOOS = c.GOOS - ctx.GOARCH = c.GOARCH - ctx.BuildTags = c.BuildTags - if c.ReleaseTags != nil { - ctx.ReleaseTags = c.ReleaseTags - } - return ctx.MatchFile(filepath.Dir(path), filepath.Base(path)) -} - -// importer is an implementation of go/types.Importer. -// -// This wraps a configuration, which provides the map of package names to -// files, and the facts. Note that this importer implementation will always -// pass when a given package is not available. -type importer struct { - *PackageConfig - fset *token.FileSet - cache map[string]*types.Package - lastErr error - callback func(string) error -} - -// Import implements types.Importer.Import. -func (i *importer) Import(path string) (*types.Package, error) { - if path == "unsafe" { - // Special case: go/types has pre-defined type information for - // unsafe. We ensure that this package is correct, in case any - // analyzers are specifically looking for this. - return types.Unsafe, nil - } - - // Call the internal callback. This is used to resolve loading order - // for the standard library. See checkStdlib. - if i.callback != nil { - if err := i.callback(path); err != nil { - i.lastErr = err - return nil, err - } - } - - // Check the cache. - if pkg, ok := i.cache[path]; ok && pkg.Complete() { - return pkg, nil - } - - // Actually load the data. - realPath, ok := i.ImportMap[path] - var ( - rc io.ReadCloser - err error - ) - if !ok { - // Not found in the import path. Attempt to find the package - // via the standard library. - rc, err = findStdPkg(i.GOOS, i.GOARCH, path) - } else { - // Open the file. - rc, err = os.Open(realPath) - } - if err != nil { - i.lastErr = err - return nil, err - } - defer rc.Close() - - // Load all exported data. - r, err := gcexportdata.NewReader(rc) - if err != nil { - return nil, err - } - - return gcexportdata.Read(r, i.fset, i.cache, path) -} - -// ErrSkip indicates the package should be skipped. -var ErrSkip = errors.New("skipped") - -// CheckStdlib checks the standard library. -// -// This constructs a synthetic package configuration for each library in the -// standard library sources, and call CheckPackage repeatedly. -// -// Note that not all parts of the source are expected to build. We skip obvious -// test files, and cmd files, which should not be dependencies. -func CheckStdlib(config *StdlibConfig, analyzers []*analysis.Analyzer) (allFindings FindingSet, facts []byte, err error) { - if len(config.Srcs) == 0 { - return nil, nil, nil - } - - // Ensure all paths are normalized. - for i := 0; i < len(config.Srcs); i++ { - config.Srcs[i] = path.Clean(config.Srcs[i]) - } - - // Calculate the root source directory. This is always a directory - // named 'src', of which we simply take the first we find. This is a - // bit fragile, but works for all currently known Go source - // configurations. - // - // Note that there may be extra files outside of the root source - // directory; we simply ignore those. - rootSrcPrefix := "" - for _, file := range config.Srcs { - const src = "/src/" - i := strings.Index(file, src) - if i == -1 { - // Superfluous file. - continue - } - - // Index of first character after /src/. - i += len(src) - rootSrcPrefix = file[:i] - break - } - - // Go standard library packages using Go 1.18 type parameter features. - // - // As of writing, analysis tooling is not updated to support type - // parameters and will choke on these packages. We skip these packages - // entirely for now. - // - // TODO(b/201686256): remove once tooling can handle type parameters. - usesTypeParams := map[string]struct{}{ - "constraints": struct{}{}, // golang.org/issue/45458 - "maps": struct{}{}, // golang.org/issue/47649 - "slices": struct{}{}, // golang.org/issue/45955 - } - - // Aggregate all files by directory. - packages := make(map[string]*PackageConfig) - for _, file := range config.Srcs { - if !strings.HasPrefix(file, rootSrcPrefix) { - // Superflouous file. - continue - } - - d := path.Dir(file) - if len(rootSrcPrefix) >= len(d) { - continue // Not a file. - } - pkg := d[len(rootSrcPrefix):] - - // Skip cmd packages and obvious test files: see above. - if strings.HasPrefix(pkg, "cmd/") || strings.HasSuffix(file, "_test.go") { - continue - } - - if _, ok := usesTypeParams[pkg]; ok { - log.Printf("WARNING: Skipping package %q: type param analysis not yet supported", pkg) - continue - } - - c, ok := packages[pkg] - if !ok { - c = &PackageConfig{ - ImportPath: pkg, - GOOS: config.GOOS, - GOARCH: config.GOARCH, - BuildTags: config.BuildTags, - ReleaseTags: config.ReleaseTags, - } - packages[pkg] = c - } - // Add the files appropriately. Note that they will be further - // filtered by architecture and build tags below, so this need - // not be done immediately. - if strings.HasSuffix(file, ".go") { - c.GoFiles = append(c.GoFiles, file) - } else { - c.NonGoFiles = append(c.NonGoFiles, file) - } - } - - // Closure to check a single package. - localStdlibFacts := make(stdlibFacts) - localStdlibErrs := make(map[string]error) - stdlibCachedFacts.Lookup([]string{""}, func() worker.Sizer { - return localStdlibFacts - }) - var checkOne func(pkg string) error // Recursive. - checkOne = func(pkg string) error { - // Is this already done? - if _, ok := localStdlibFacts[pkg]; ok { - return nil - } - // Did this fail previously? - if _, ok := localStdlibErrs[pkg]; ok { - return nil - } - - // Lookup the configuration. - config, ok := packages[pkg] - if !ok { - return nil // Not known. - } - - // Find the binary package, and provide to objdump. - rc, err := findStdPkg(config.GOOS, config.GOARCH, pkg) - if err != nil { - // If there's no binary for this package, it is likely - // not built with the distribution. That's fine, we can - // just skip analysis. - localStdlibErrs[pkg] = err - return nil - } - - // Provide the input. - oldReader := objdump.Reader - objdump.Reader = rc // For analysis. - defer func() { - rc.Close() - objdump.Reader = oldReader // Restore. - }() - - // Run the analysis. - findings, factData, err := CheckPackage(config, analyzers, checkOne) - if err != nil { - // If we can't analyze a package from the standard library, - // then we skip it. It will simply not have any findings. - localStdlibErrs[pkg] = err - return nil - } - localStdlibFacts[pkg] = factData - allFindings = append(allFindings, findings...) - return nil - } - - // Check all packages. - // - // Note that this may call checkOne recursively, so it's not guaranteed - // to evaluate in the order provided here. We do ensure however, that - // all packages are evaluated. - for pkg := range packages { - if err := checkOne(pkg); err != nil { - return nil, nil, err - } - } - - // Sanity check. - if len(localStdlibFacts) == 0 { - return nil, nil, fmt.Errorf("no stdlib facts found: misconfiguration?") - } - - // Write out all findings. - buf := bytes.NewBuffer(nil) - if err := localStdlibFacts.EncodeTo(buf); err != nil { - return nil, nil, fmt.Errorf("error serialized stdlib facts: %v", err) - } - - // Write out all errors. - for pkg, err := range localStdlibErrs { - log.Printf("WARNING: error while processing %v: %v", pkg, err) - } - - // Return all findings. - return allFindings, buf.Bytes(), nil -} - -// sanityCheckScope checks that all object in astTypes map to the correct -// objects in binaryTypes. Note that we don't check whether the sets are the -// same, we only care about the fidelity of objects in astTypes. -// -// When an inconsistency is identified, we record it in the astToBinaryMap. -// This allows us to dynamically replace facts and correct for the issue. The -// total number of mismatches is returned. -func sanityCheckScope(astScope *types.Scope, binaryTypes *types.Package, binaryScope *types.Scope, astToBinary map[types.Object]types.Object) error { - for _, x := range astScope.Names() { - fe := astScope.Lookup(x) - path, err := objectpath.For(fe) - if err != nil { - continue // Not an encoded object. - } - se, err := objectpath.Object(binaryTypes, path) - if err != nil { - continue // May be unused, see below. - } - if fe.Id() != se.Id() { - // These types are incompatible. This means that when - // this objectpath is loading from the binaryTypes (for - // dependencies) it will resolve to a fact for that - // type. We don't actually care about this error since - // we do the rewritten, but may as well alert. - log.Printf("WARNING: Object %s is a victim of go/issues/44195.", fe.Id()) - } - se = binaryScope.Lookup(x) - if se == nil { - // The fact may not be exported in the objectdata, if - // it is package internal. This is fine, as nothing out - // of this package can use these symbols. - continue - } - // Save the translation. - astToBinary[fe] = se - } - for i := 0; i < astScope.NumChildren(); i++ { - if err := sanityCheckScope(astScope.Child(i), binaryTypes, binaryScope, astToBinary); err != nil { - return err - } - } - return nil -} - -// sanityCheckTypes checks that two types are sane. The total number of -// mismatches is returned. -func sanityCheckTypes(astTypes, binaryTypes *types.Package, astToBinary map[types.Object]types.Object) error { - return sanityCheckScope(astTypes.Scope(), binaryTypes, binaryTypes.Scope(), astToBinary) -} - -// CheckPackage runs all given analyzers. -// -// The implementation was adapted from [1], which was in turn adpated from [2]. -// This returns a list of matching analysis issues, or an error if the analysis -// could not be completed. -// -// [1] bazelbuid/rules_go/tools/builders/nogo_main.go -// [2] golang.org/x/tools/go/checker/internal/checker -func CheckPackage(config *PackageConfig, analyzers []*analysis.Analyzer, importCallback func(string) error) (findings []Finding, factData []byte, err error) { - imp := &importer{ - PackageConfig: config, - fset: token.NewFileSet(), - cache: make(map[string]*types.Package), - callback: importCallback, - } - - // Load all source files. - var syntax []*ast.File - for _, file := range config.GoFiles { - include, err := config.shouldInclude(file) - if err != nil { - return nil, nil, fmt.Errorf("error evaluating file %q: %v", file, err) - } - if !include { - continue - } - s, err := parser.ParseFile(imp.fset, file, nil, parser.ParseComments) - if err != nil { - return nil, nil, fmt.Errorf("error parsing file %q: %v", file, err) - } - syntax = append(syntax, s) - } - - // Check type information. - typesSizes := types.SizesFor("gc", config.GOARCH) - typeConfig := types.Config{Importer: imp} - typesInfo := &types.Info{ - Types: make(map[ast.Expr]types.TypeAndValue), - Uses: make(map[*ast.Ident]types.Object), - Defs: make(map[*ast.Ident]types.Object), - Implicits: make(map[ast.Node]types.Object), - Scopes: make(map[ast.Node]*types.Scope), - Selections: make(map[*ast.SelectorExpr]*types.Selection), - } - astTypes, err := typeConfig.Check(config.ImportPath, imp.fset, syntax, typesInfo) - if err != nil && imp.lastErr != ErrSkip { - return nil, nil, fmt.Errorf("error checking types: %w", err) - } - - // Load all facts using the astTypes, although it may need reconciling - // later on. See the fact functions below. - astFacts, err := facts.Decode(astTypes, config.factLoader) - if err != nil { - return nil, nil, fmt.Errorf("error decoding facts: %w", err) - } - - // Sanity check all types and record metadata to prevent - // https://github.com/golang/go/issues/44195. - // - // This block loads the binary types, whose encoding will be well - // defined and aligned with any downstream consumers. Below in the fact - // functions for the analysis, we serialize types to both the astFacts - // and the binaryFacts if available. The binaryFacts are the final - // encoded facts in order to ensure compatibility. We keep the - // intermediate astTypes in order to allow exporting and importing - // within the local package under analysis. - var ( - astToBinary = make(map[types.Object]types.Object) - binaryFacts *facts.Set - ) - if _, ok := config.ImportMap[config.ImportPath]; ok { - binaryTypes, err := imp.Import(config.ImportPath) - if err != nil { - return nil, nil, fmt.Errorf("error loading self: %w", err) - } - if err := sanityCheckTypes(astTypes, binaryTypes, astToBinary); err != nil { - return nil, nil, fmt.Errorf("error sanity checking types: %w", err) - } - binaryFacts, err = facts.Decode(binaryTypes, config.factLoader) - if err != nil { - return nil, nil, fmt.Errorf("error decoding facts: %w", err) - } - } - - // Register fact types and establish dependencies between analyzers. - // The visit closure will execute recursively, and populate results - // will all required analysis results. - results := make(map[*analysis.Analyzer]interface{}) - var visit func(*analysis.Analyzer) error // For recursion. - visit = func(a *analysis.Analyzer) error { - if _, ok := results[a]; ok { - return nil - } - - // Run recursively for all dependencies. - for _, req := range a.Requires { - if err := visit(req); err != nil { - return err - } - } - - // Run the analysis. - localFactsFilter := make(map[reflect.Type]bool) - for _, f := range a.FactTypes { - localFactsFilter[reflect.TypeOf(f)] = true - } - p := &analysis.Pass{ - Analyzer: a, - Fset: imp.fset, - Files: syntax, - Pkg: astTypes, - TypesInfo: typesInfo, - ResultOf: results, // All results. - Report: func(d analysis.Diagnostic) { - findings = append(findings, Finding{ - Category: AnalyzerName(a.Name), - Position: imp.fset.Position(d.Pos), - Message: d.Message, - }) - }, - ImportPackageFact: astFacts.ImportPackageFact, - ExportPackageFact: func(fact analysis.Fact) { - astFacts.ExportPackageFact(fact) - if binaryFacts != nil { - binaryFacts.ExportPackageFact(fact) - } - }, - ImportObjectFact: astFacts.ImportObjectFact, - ExportObjectFact: func(obj types.Object, fact analysis.Fact) { - astFacts.ExportObjectFact(obj, fact) - // Note that if no object is recorded in - // astToBinary and binaryFacts != nil, then the - // object doesn't appear in the exported data. - // It was likely an internal object to the - // package, and there is no meaningful - // downstream consumer of the fact. - if binaryObj, ok := astToBinary[obj]; ok && binaryFacts != nil { - binaryFacts.ExportObjectFact(binaryObj, fact) - } - }, - AllPackageFacts: func() []analysis.PackageFact { return astFacts.AllPackageFacts(localFactsFilter) }, - AllObjectFacts: func() []analysis.ObjectFact { return astFacts.AllObjectFacts(localFactsFilter) }, - TypesSizes: typesSizes, - } - result, err := a.Run(p) - if err != nil { - return fmt.Errorf("error running analysis %s: %v", a, err) - } - - // Sanity check & save the result. - if got, want := reflect.TypeOf(result), a.ResultType; got != want { - return fmt.Errorf("error: analyzer %s returned a result of type %v, but declared ResultType %v", a, got, want) - } - results[a] = result - return nil // Success. - } - - // Visit all analyzers recursively. - for _, a := range analyzers { - if imp.lastErr == ErrSkip { - continue // No local analysis. - } - if err := visit(a); err != nil { - return nil, nil, err // Already has context. - } - } - - // Return all findings. Note that we have a preference to returning the - // binary facts if available, so that downstream consumers of these - // facts will find the export aligns with the internal type details. - // See the block above with the call to sanityCheckTypes. - if binaryFacts != nil { - return findings, binaryFacts.Encode(), nil - } - return findings, astFacts.Encode(), nil -} - -func init() { - gob.Register((*stdlibFact)(nil)) -} diff --git a/tools/nogo/objdump/objdump.go b/tools/nogo/objdump/objdump.go deleted file mode 100644 index 48484abf3..000000000 --- a/tools/nogo/objdump/objdump.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package objdump is a wrapper around relevant objdump flags. -package objdump - -import ( - "flag" - "fmt" - "io" - "os" - "os/exec" -) - -var ( - // Binary is the binary under analysis. - // - // See Reader, below. - binary = flag.String("binary", "", "binary under analysis") - - // Reader is the input stream. - // - // This may be set instead of Binary. - Reader io.Reader - - // objdumpTool is the tool used to dump a binary. - objdumpTool = flag.String("objdump_tool", "", "tool used to dump a binary") -) - -// LoadRaw reads the raw object output. -func LoadRaw(fn func(r io.Reader) error) error { - var r io.Reader - if *binary != "" { - f, err := os.Open(*binary) - if err != nil { - return err - } - defer f.Close() - r = f - } else if Reader != nil { - r = Reader - } else { - // We have no input stream. - return fmt.Errorf("no binary or reader provided") - } - return fn(r) -} - -// Load reads the objdump output. -func Load(fn func(r io.Reader) error) error { - var ( - args []string - stdin io.Reader - ) - if *binary != "" { - args = append(args, *binary) - } else if Reader != nil { - stdin = Reader - } else { - // We have no input stream or binary. - return fmt.Errorf("no binary or reader provided") - } - - // Construct our command. - cmd := exec.Command(*objdumpTool, args...) - cmd.Stdin = stdin - cmd.Stderr = os.Stderr - out, err := cmd.StdoutPipe() - if err != nil { - return err - } - if err := cmd.Start(); err != nil { - return err - } - - // Call the user hook. - userErr := fn(out) - - // Wait for the dump to finish. - if err := cmd.Wait(); userErr == nil && err != nil { - return err - } - - return userErr -} diff --git a/tools/rules_go_visibility.patch b/tools/rules_go_visibility.patch deleted file mode 100644 index e5bb2e3d5..000000000 --- a/tools/rules_go_visibility.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/third_party/org_golang_x_tools-gazelle.patch b/third_party/org_golang_x_tools-gazelle.patch -index 7bdacff5..2fe9ce93 100644 ---- a/third_party/org_golang_x_tools-gazelle.patch -+++ b/third_party/org_golang_x_tools-gazelle.patch -@@ -2054,7 +2054,7 @@ diff -urN b/go/analysis/internal/facts/BUILD.bazel c/go/analysis/internal/facts/ - + "imports.go", - + ], - + importpath = "golang.org/x/tools/go/analysis/internal/facts", --+ visibility = ["//go/analysis:__subpackages__"], -++ visibility = ["//visibility:public"], - + deps = [ - + "//go/analysis", - + "//go/types/objectpath", -@@ -2078,7 +2078,7 @@ diff -urN b/go/analysis/internal/facts/BUILD.bazel c/go/analysis/internal/facts/ - +alias( - + name = "go_default_library", - + actual = ":facts", --+ visibility = ["//go/analysis:__subpackages__"], -++ visibility = ["//visibility:public"], - +) - + - +go_test( diff --git a/tools/worker/BUILD b/tools/worker/BUILD index dc03ce11e..2c12fed2f 100644 --- a/tools/worker/BUILD +++ b/tools/worker/BUILD @@ -1,4 +1,5 @@ load("//tools:defs.bzl", "bazel_worker_proto", "go_library") +load("//tools/go_generics:defs.bzl", "go_template_instance") package(licenses = ["notice"]) @@ -8,12 +9,41 @@ glaze_ignore = [ "worker.go", ] +go_template_instance( + name = "lru_list", + out = "lru_list.go", + package = "worker", + prefix = "lru", + template = "//pkg/ilist:generic_list", + types = { + "Element": "*cacheEntry", + "Linker": "*cacheEntry", + }, +) + +go_template_instance( + name = "cache_list", + out = "cache_list.go", + package = "worker", + prefix = "caches", + template = "//pkg/ilist:generic_list", + types = { + "Element": "*Cache", + "Linker": "*Cache", + }, +) + go_library( name = "worker", - srcs = ["worker.go"], + srcs = [ + "worker.go", + ":cache_list", + ":lru_list", + ], visibility = ["//tools:__subpackages__"], deps = [ bazel_worker_proto, + "//runsc/flag", "@org_golang_google_protobuf//encoding/protowire:go_default_library", "@org_golang_google_protobuf//proto:go_default_library", "@org_golang_x_sys//unix:go_default_library", diff --git a/tools/worker/worker.go b/tools/worker/worker.go index 669a5f203..d0c1f2d56 100644 --- a/tools/worker/worker.go +++ b/tools/worker/worker.go @@ -21,7 +21,6 @@ package worker import ( "bufio" "bytes" - "flag" "fmt" "io" "io/ioutil" @@ -32,6 +31,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" _ "net/http/pprof" // For profiling. @@ -40,6 +40,7 @@ import ( "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" wpb "gvisor.dev/bazel/worker_protocol_go_proto" + "gvisor.dev/gvisor/runsc/flag" ) var ( @@ -54,14 +55,78 @@ var ( // This is used for cache invalidation. The key is the *absolute* path // name, and the value is the digest in the current run. inputFiles = make(map[string]string) +) - // activeCaches is the set of active caches. - activeCaches = make(map[*Cache]struct{}) +// LookupDigest returns a digest for the given file. +func LookupDigest(filename string) (string, bool) { + digest, ok := inputFiles[filename] + return digest, ok +} + +var ( + // allCaches is a global list of caches. + allCaches cachesList + + // globalMu is a globalMutex for globalLRU. + // + // Note that this has a strict lock ordering requirement. No cache locks + // may be held when acquiring this lock. + globalMu sync.Mutex + + // globalLRU is a globalLRU for all entries. + // + // Protected by globalMu. + globalLRU lruList // totalCacheUsage is the total usage of all caches. + // + // Protected by globalMu. totalCacheUsage int64 ) +// Sizer returns a size. +type Sizer interface { + Size() int64 +} + +// cacheEntry is a cache entry. +// +// The cacheEntry object is immutable, with the exception of the ready +// WaitGroup, which may be signalled. +type cacheEntry struct { + cache *Cache + key string + sizer Sizer + err error + ready sync.WaitGroup + lruEntry // in globalLRU. +} + +// Cache is a worker cache. +// +// They can be created via NewCache. +type Cache struct { + name string + mu sync.Mutex + entries map[string]*cacheEntry + size int64 + hits int64 + misses int64 + cachesEntry // in allCaches. +} + +// NewCache returns a new cache. +// +// Precondition: this must be called at init. +func NewCache(name string) *Cache { + c := &Cache{ + name: name, + entries: make(map[string]*cacheEntry), + } + allCaches.PushBack(c) + return c +} + // mustAbs returns the absolute path of a filename or dies. func mustAbs(filename string) string { abs, err := filepath.Abs(filename) @@ -71,46 +136,8 @@ func mustAbs(filename string) string { return abs } -// updateInputFiles creates an entry in inputFiles. -func updateInputFile(filename, digest string) { - inputFiles[mustAbs(filename)] = digest -} - -// Sizer returns a size. -type Sizer interface { - Size() int64 -} - -// CacheBytes is an example of a Sizer. -type CacheBytes []byte - -// Size implements Sizer.Size. -func (cb CacheBytes) Size() int64 { - return int64(len(cb)) -} - -// Cache is a worker cache. -// -// They can be created via NewCache. -type Cache struct { - name string - entries map[string]Sizer - size int64 - hits int64 - misses int64 -} - -// NewCache returns a new cache. -func NewCache(name string) *Cache { - return &Cache{ - name: name, - } -} - // Lookup looks up an entry in the cache. -// -// It is a function of the given files. -func (c *Cache) Lookup(filenames []string, generate func() Sizer) Sizer { +func (c *Cache) Lookup(filenames []string, generate func() (Sizer, error)) (Sizer, error) { digests := make([]string, 0, len(filenames)) for _, filename := range filenames { digest, ok := inputFiles[mustAbs(filename)] @@ -129,61 +156,93 @@ func (c *Cache) Lookup(filenames []string, generate func() Sizer) Sizer { return digests[i] < digests[j] }) cacheKey := strings.Join(digests, "+") - if c.entries == nil { - c.entries = make(map[string]Sizer) - activeCaches[c] = struct{}{} - } + + c.mu.Lock() entry, ok := c.entries[cacheKey] if ok { c.hits++ - return entry + c.mu.Unlock() // See ordering requirement. + if entry.sizer != nil { + globalMu.Lock() + globalLRU.Remove(entry) + globalLRU.PushBack(entry) + globalMu.Unlock() + } + entry.ready.Wait() + return entry.sizer, entry.err } // Generate a new entry. - entry = generate() c.misses++ + entry = &cacheEntry{ + cache: c, + key: cacheKey, + } + entry.ready.Add(1) c.entries[cacheKey] = entry - if entry != nil { - sz := entry.Size() - c.size += sz - totalCacheUsage += sz + c.mu.Unlock() // Unlock for generate. + entry.sizer, entry.err = generate() + entry.ready.Done() + + // Does this need to be accounted? We consider negative cache entries + // to be free, in order to avoid extra work. + if entry.sizer == nil { + return entry.sizer, entry.err } - // Check the capacity of all caches. If it greater than the maximum, we - // flush everything but still return this entry. + // Account for the size of this item. This is complex, but we may clear + // out other caches based on the globalLRU. Only items with non-zero + // size are added here. This routine is the reason for the locking + // order requirement on globalMu and must be respected. + globalMu.Lock() + globalLRU.PushBack(entry) + totalCacheUsage += entry.sizer.Size() if totalCacheUsage > *maximumCacheUsage { - for entry, _ := range activeCaches { - // Drop all entries. - entry.size = 0 - entry.entries = nil - } - totalCacheUsage = 0 // Reset. - } + for entry := globalLRU.Front(); entry != nil && totalCacheUsage > *maximumCacheUsage; entry = globalLRU.Front() { + sz := entry.sizer.Size() - return entry + // Remove from its cache. + entry.cache.mu.Lock() + delete(entry.cache.entries, entry.key) + entry.cache.size -= sz + entry.cache.mu.Unlock() + + // Remove from the global list. + globalLRU.Remove(entry) + totalCacheUsage -= sz + } + } + globalMu.Unlock() + + // Return the value. + return entry.sizer, entry.err } // allCacheStats returns stats for all caches. func allCacheStats() string { - var sb strings.Builder - for entry, _ := range activeCaches { - ratio := float64(entry.hits) / float64(entry.hits+entry.misses) + var ( + sb strings.Builder + count int + ) + for c := allCaches.Front(); c != nil; c = c.Next() { + c.mu.Lock() + if len(c.entries) == 0 { + c.mu.Unlock() + continue // Not active. + } + count++ // At least one active cache. + ratio := float64(c.hits) / float64(c.hits+c.misses) fmt.Fprintf(&sb, "% 10s: count: % 5d size: % 10d hits: % 7d misses: % 7d ratio: %2.2f\n", - entry.name, len(entry.entries), entry.size, entry.hits, entry.misses, ratio) + c.name, len(c.entries), c.size, c.hits, c.misses, ratio) + c.mu.Unlock() } - if len(activeCaches) > 0 { + if count > 0 { fmt.Fprintf(&sb, "total: % 10d\n", totalCacheUsage) } return sb.String() } -// LookupDigest returns a digest for the given file. -func LookupDigest(filename string) (string, bool) { - digest, ok := inputFiles[filename] - return digest, ok -} - // Work invokes the main function. func Work(run func([]string) int) { flag.CommandLine.Parse(os.Args[1:]) @@ -197,6 +256,15 @@ func Work(run func([]string) int) { } // Pull arguments from the file. args = strings.Split(string(content), "\n") + for i := 0; i < len(args); { + if args[i] == "" { + // Remove empty arguments. + copy(args[i:], args[i+1:]) + args = args[:len(args)-1] + continue + } + i++ // Visit next. + } flag.CommandLine.Parse(args) args = flag.CommandLine.Args() } @@ -276,7 +344,7 @@ func Work(run func([]string) int) { // Flush relevant caches. inputFiles = make(map[string]string) for _, input := range wreq.GetInputs() { - updateInputFile(input.GetPath(), string(input.GetDigest())) + inputFiles[mustAbs(input.GetPath())] = string(input.GetDigest()) } // Prepare logging. diff --git a/vdso/BUILD b/vdso/BUILD index c70bb8218..89ff98592 100644 --- a/vdso/BUILD +++ b/vdso/BUILD @@ -7,6 +7,8 @@ load("//tools:defs.bzl", "cc_flags_supplier", "cc_toolchain", "select_arch", "vd package(licenses = ["notice"]) +exports_files(["check_vdso.py"]) + genrule( name = "vdso", srcs = [ @@ -53,13 +55,7 @@ genrule( ) + "-o $(location vdso.so) " + "$(location vdso.cc) " + - "$(location vdso_time.cc) " + - "&& $(location :check_vdso) " + - "--check-data " + - "--vdso $(location vdso.so) ", - exec_tools = [ - ":check_vdso", - ], + "$(location vdso_time.cc)", features = ["-pie"], toolchains = [ cc_toolchain, @@ -73,9 +69,14 @@ cc_flags_supplier( features = ["-pie"], ) -py_binary( - name = "check_vdso", +py_test( + name = "vdso_test", srcs = ["check_vdso.py"], + args = [ + "--check-data", + "--vdso=$(location :vdso)", + ], + data = [":vdso"], + main = "check_vdso.py", python_version = "PY3", - visibility = ["//:sandbox"], )