diff --git a/WORKSPACE b/WORKSPACE index f0c0466c7..b654932e3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -51,11 +51,12 @@ http_archive( "//tools:rules_go_symbols.patch", # Allow for patching of the go_sdk. "//tools:rules_go_sdk.patch", + "//tools:rules_go_facts.patch", ], - sha256 = "7c76d6236b28ff695aa28cf35f95de317a9472fd1fb14ac797c9bf684f09b37c", + sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.44.2/rules_go-v0.44.2.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.44.2/rules_go-v0.44.2.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", ], ) @@ -95,7 +96,7 @@ go_download_sdk( # time on our continuous integration. patch = "//tools:go_types_memoize.patch", patch_strip = 2, - version = "1.21.6", + version = "1.22.0", ) gazelle_dependencies() @@ -113,8 +114,8 @@ go_repository( go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=", - version = "v0.16.0", + sum = "h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=", + version = "v0.17.0", ) go_repository( diff --git a/go.mod b/go.mod index d58cfccb1..ac82a2aae 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 golang.org/x/mod v0.14.0 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.17.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.16.1 google.golang.org/protobuf v1.32.0 diff --git a/go.sum b/go.sum index 154307be8..fa28ff61a 100644 --- a/go.sum +++ b/go.sum @@ -352,6 +352,8 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= diff --git a/tools/rules_go_facts.patch b/tools/rules_go_facts.patch new file mode 100644 index 000000000..fe8c3e710 --- /dev/null +++ b/tools/rules_go_facts.patch @@ -0,0 +1,873 @@ +commit 0a21a547f4ebf1d26d79854512cc2e0f1e1e4e90 +Author: Andrei Vagin +Date: Thu Feb 15 22:30:22 2024 -0800 + + Revert "Emit nogo facts into a separate archive (#3789)" + + This reverts commit 30099a6add3c43706b4eec82b773b78310874935. + +diff --git a/go/private/actions/archive.bzl b/go/private/actions/archive.bzl +index 42bf2039..a4e737ee 100644 +--- a/go/private/actions/archive.bzl ++++ b/go/private/actions/archive.bzl +@@ -58,13 +58,9 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d + pre_ext += _recompile_suffix + out_lib = go.declare_file(go, name = source.library.name, ext = pre_ext + ".a") + +- # store export information for compiling dependent packages separately ++ # store __.PKGDEF and nogo facts in .x + out_export = go.declare_file(go, name = source.library.name, ext = pre_ext + ".x") + out_cgo_export_h = None # set if cgo used in c-shared or c-archive mode +- out_facts = None +- nogo = go.get_nogo(go) +- if nogo: +- out_facts = go.declare_file(go, name = source.library.name, ext = pre_ext + ".facts") + + direct = [get_archive(dep) for dep in source.deps] + runfiles = source.runfiles +@@ -109,8 +105,6 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d + archives = direct, + out_lib = out_lib, + out_export = out_export, +- out_facts = out_facts, +- nogo = nogo, + out_cgo_export_h = out_cgo_export_h, + gc_goopts = source.gc_goopts, + cgo = True, +@@ -135,8 +129,6 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d + archives = direct, + out_lib = out_lib, + out_export = out_export, +- out_facts = out_facts, +- nogo = nogo, + gc_goopts = source.gc_goopts, + cgo = False, + testfilter = testfilter, +@@ -181,7 +173,6 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d + # Information needed by dependents + file = out_lib, + export_file = out_export, +- facts_file = out_facts, + data_files = as_tuple(data_files), + _cgo_deps = as_tuple(cgo_deps), + ) +diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl +index 10fa6970..48adb910 100644 +--- a/go/private/actions/compilepkg.bzl ++++ b/go/private/actions/compilepkg.bzl +@@ -28,18 +28,6 @@ def _archive(v): + v.data.export_file.path if v.data.export_file else v.data.file.path, + ) + +-def _facts(v): +- facts_file = v.data.facts_file +- if not facts_file: +- return None +- importpaths = [v.data.importpath] +- importpaths.extend(v.data.importpath_aliases) +- return "{}={}={}".format( +- ":".join(importpaths), +- v.data.importmap, +- facts_file.path, +- ) +- + def _embedroot_arg(src): + return src.root.path + +@@ -67,8 +55,6 @@ def emit_compilepkg( + clinkopts = [], + out_lib = None, + out_export = None, +- out_facts = None, +- nogo = None, + out_cgo_export_h = None, + gc_goopts = [], + testfilter = None, # TODO: remove when test action compiles packages +@@ -78,8 +64,6 @@ def emit_compilepkg( + fail("sources is a required parameter") + if out_lib == None: + fail("out_lib is a required parameter") +- if bool(nogo) != bool(out_facts): +- fail("nogo must be specified if and only if out_facts is specified") + + inputs = (sources + embedsrcs + [go.package_list] + + [archive.data.export_file for archive in archives] + +@@ -124,13 +108,10 @@ def emit_compilepkg( + args.add("-p", importmap) + args.add("-package_list", go.package_list) + +- args.add("-lo", out_lib) +- args.add("-o", out_export) ++ args.add("-o", out_lib) ++ args.add("-x", out_export) ++ nogo = go.get_nogo(go) + if nogo: +- args.add_all(archives, before_each = "-facts", map_each = _facts) +- inputs.extend([archive.data.facts_file for archive in archives if archive.data.facts_file]) +- args.add("-out_facts", out_facts) +- outputs.append(out_facts) + args.add("-nogo", nogo) + inputs.append(nogo) + if out_cgo_export_h: +diff --git a/go/providers.rst b/go/providers.rst +index a2361ac1..dccc0e1e 100644 +--- a/go/providers.rst ++++ b/go/providers.rst +@@ -260,15 +260,7 @@ rule. Instead, it's referenced in the ``data`` field of GoArchive_. + +--------------------------------+-----------------------------------------------------------------+ + | :param:`file` | :type:`File` | + +--------------------------------+-----------------------------------------------------------------+ +-| The archive file for the linker produced when this library is compiled. | +-+--------------------------------+-----------------------------------------------------------------+ +-| :param:`export_file` | :type:`File` | +-+--------------------------------+-----------------------------------------------------------------+ +-| The archive file for compilation of dependent libraries produced when this library is compiled. | +-+--------------------------------+-----------------------------------------------------------------+ +-| :param:`facts_file` | :type:`File` | +-+--------------------------------+-----------------------------------------------------------------+ +-| The serialized facts for this library produced when nogo ran for this library. | ++| The archive file produced when this library is compiled. | + +--------------------------------+-----------------------------------------------------------------+ + | :param:`srcs` | :type:`tuple of File` | + +--------------------------------+-----------------------------------------------------------------+ +diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel +index d327a3af..1b44a15c 100644 +--- a/go/tools/builders/BUILD.bazel ++++ b/go/tools/builders/BUILD.bazel +@@ -76,6 +76,7 @@ filegroup( + "generate_test_main.go", + "importcfg.go", + "link.go", ++ "pack.go", + "read.go", + "replicate.go", + "stdlib.go", +@@ -96,6 +97,7 @@ go_source( + "nogo_typeparams_go117.go", + "nogo_typeparams_go118.go", + "nolint.go", ++ "pack.go", + ], + # //go/tools/builders:nogo_srcs is considered a different target by + # Bazel's visibility check than +diff --git a/go/tools/builders/ar.go b/go/tools/builders/ar.go +index d2de6b96..2f4b36c8 100644 +--- a/go/tools/builders/ar.go ++++ b/go/tools/builders/ar.go +@@ -23,18 +23,6 @@ import ( + "strings" + ) + +-const ( +- // arHeader appears at the beginning of archives created by "ar" and +- // "go tool pack" on all platforms. +- arHeader = "!\n" +- +- // entryLength is the size in bytes of the metadata preceding each file +- // in an archive. +- entryLength = 60 +-) +- +-var zeroBytes = []byte("0 ") +- + type header struct { + NameRaw [16]byte + ModTimeRaw [12]byte +diff --git a/go/tools/builders/cgo2.go b/go/tools/builders/cgo2.go +index 80043e46..fc2876a9 100644 +--- a/go/tools/builders/cgo2.go ++++ b/go/tools/builders/cgo2.go +@@ -23,11 +23,9 @@ package main + import ( + "bytes" + "fmt" +- "io" + "io/ioutil" + "os" + "path/filepath" +- "runtime" + "strings" + ) + +@@ -397,34 +395,3 @@ func (e cgoError) Error() string { + fmt.Fprintf(b, "Ensure that 'cgo = True' is set and the C/C++ toolchain is configured.") + return b.String() + } +- +-func copyFile(inPath, outPath string) error { +- inFile, err := os.Open(inPath) +- if err != nil { +- return err +- } +- defer inFile.Close() +- outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) +- if err != nil { +- return err +- } +- defer outFile.Close() +- _, err = io.Copy(outFile, inFile) +- return err +-} +- +-func linkFile(inPath, outPath string) error { +- inPath, err := filepath.Abs(inPath) +- if err != nil { +- return err +- } +- return os.Symlink(inPath, outPath) +-} +- +-func copyOrLinkFile(inPath, outPath string) error { +- if runtime.GOOS == "windows" { +- return copyFile(inPath, outPath) +- } else { +- return linkFile(inPath, outPath) +- } +-} +diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go +index b909fa86..46cae3c0 100644 +--- a/go/tools/builders/compilepkg.go ++++ b/go/tools/builders/compilepkg.go +@@ -50,9 +50,9 @@ func compilePkg(args []string) error { + fs := flag.NewFlagSet("GoCompilePkg", flag.ExitOnError) + goenv := envFlags(fs) + var unfilteredSrcs, coverSrcs, embedSrcs, embedLookupDirs, embedRoots, recompileInternalDeps multiFlag +- var deps, facts archiveMultiFlag ++ var deps archiveMultiFlag + var importPath, packagePath, nogoPath, packageListPath, coverMode string +- var outLinkobjPath, outInterfacePath, outFactsPath, cgoExportHPath string ++ var outPath, outFactsPath, cgoExportHPath string + var testFilter string + var gcFlags, asmFlags, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags quoteMultiFlag + var coverFormat string +@@ -63,7 +63,6 @@ func compilePkg(args []string) error { + fs.Var(&embedLookupDirs, "embedlookupdir", "Root-relative paths to directories relative to which //go:embed directives are resolved") + fs.Var(&embedRoots, "embedroot", "Bazel output root under which a file passed via -embedsrc resides") + fs.Var(&deps, "arc", "Import path, package path, and file name of a direct dependency, separated by '='") +- fs.Var(&facts, "facts", "Import path, package path, and file name of a direct dependency's nogo facts file, separated by '='") + fs.StringVar(&importPath, "importpath", "", "The import path of the package being compiled. Not passed to the compiler, but may be displayed in debug data.") + fs.StringVar(&packagePath, "p", "", "The package path (importmap) of the package being compiled") + fs.Var(&gcFlags, "gcflags", "Go compiler flags") +@@ -77,9 +76,8 @@ func compilePkg(args []string) error { + fs.StringVar(&nogoPath, "nogo", "", "The nogo binary. If unset, nogo will not be run.") + fs.StringVar(&packageListPath, "package_list", "", "The file containing the list of standard library packages") + fs.StringVar(&coverMode, "cover_mode", "", "The coverage mode to use. Empty if coverage instrumentation should not be added.") +- fs.StringVar(&outLinkobjPath, "lo", "", "The full output archive file required by the linker") +- fs.StringVar(&outInterfacePath, "o", "", "The export-only output archive required to compile dependent packages") +- fs.StringVar(&outFactsPath, "out_facts", "", "The file to emit serialized nogo facts to (must be set if -nogo is set") ++ fs.StringVar(&outPath, "o", "", "The output archive file to write compiled code") ++ fs.StringVar(&outFactsPath, "x", "", "The output archive file to write export data and nogo facts") + fs.StringVar(&cgoExportHPath, "cgoexport", "", "The _cgo_exports.h file to write") + fs.StringVar(&testFilter, "testfilter", "off", "Controls test package filtering") + fs.StringVar(&coverFormat, "cover_format", "", "Emit source file paths in coverage instrumentation suitable for the specified coverage format") +@@ -96,7 +94,7 @@ func compilePkg(args []string) error { + } + cgoEnabled := os.Getenv("CGO_ENABLED") == "1" + cc := os.Getenv("CC") +- outLinkobjPath = abs(outLinkobjPath) ++ outPath = abs(outPath) + for i := range unfilteredSrcs { + unfilteredSrcs[i] = abs(unfilteredSrcs[i]) + } +@@ -144,7 +142,6 @@ func compilePkg(args []string) error { + packagePath, + srcs, + deps, +- facts, + coverMode, + coverSrcs, + embedSrcs, +@@ -162,8 +159,7 @@ func compilePkg(args []string) error { + ldFlags, + nogoPath, + packageListPath, +- outLinkobjPath, +- outInterfacePath, ++ outPath, + outFactsPath, + cgoExportHPath, + coverFormat, +@@ -177,7 +173,6 @@ func compileArchive( + packagePath string, + srcs archiveSrcs, + deps []archive, +- facts []archive, + coverMode string, + coverSrcs []string, + embedSrcs []string, +@@ -195,9 +190,8 @@ func compileArchive( + ldFlags []string, + nogoPath string, + packageListPath string, +- outLinkObj string, +- outInterfacePath string, +- outFactsPath string, ++ outPath string, ++ outXPath string, + cgoExportHPath string, + coverFormat string, + recompileInternalDeps []string, +@@ -215,7 +209,7 @@ func compileArchive( + // Otherwise, GoPack will complain if we try to add assembly or cgo objects. + // A truly empty archive does not include any references to source file paths, which + // ensures hermeticity even though the temp file path is random. +- emptyGoFile, err := os.CreateTemp(filepath.Dir(outLinkObj), "*.go") ++ emptyGoFile, err := os.CreateTemp(filepath.Dir(outPath), "*.go") + if err != nil { + return err + } +@@ -406,7 +400,7 @@ func compileArchive( + } + + // Build an importcfg file for the compiler. +- importcfgPath, err := buildImportcfgFileForCompile(imports, goenv.installSuffix, filepath.Dir(outLinkObj)) ++ importcfgPath, err := buildImportcfgFileForCompile(imports, goenv.installSuffix, filepath.Dir(outPath)) + if err != nil { + return err + } +@@ -449,11 +443,12 @@ func compileArchive( + + // Run nogo concurrently. + var nogoChan chan error +- if nogoPath != "" { ++ outFactsPath := filepath.Join(workDir, nogoFact) ++ if nogoPath != "" && len(goSrcsNogo) > 0 { + ctx, cancel := context.WithCancel(context.Background()) + nogoChan = make(chan error) + go func() { +- nogoChan <- runNogo(ctx, workDir, nogoPath, goSrcsNogo, facts, packagePath, importcfgPath, outFactsPath) ++ nogoChan <- runNogo(ctx, workDir, nogoPath, goSrcsNogo, deps, packagePath, importcfgPath, outFactsPath) + }() + defer func() { + if nogoChan != nil { +@@ -483,7 +478,7 @@ func compileArchive( + } + + // Compile the filtered .go files. +- if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, pgoprofile, outLinkObj, outInterfacePath); err != nil { ++ if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, pgoprofile, outPath); err != nil { + return err + } + +@@ -517,25 +512,44 @@ func compileArchive( + // Pack .o files into the archive. These may come from cgo generated code, + // cgo dependencies (cdeps), or assembly. + if len(objFiles) > 0 { +- if err := appendToArchive(goenv, outLinkObj, objFiles); err != nil { ++ if err := appendFiles(goenv, outPath, objFiles); err != nil { + return err + } + } + + // Check results from nogo. ++ nogoStatus := nogoNotRun + if nogoChan != nil { + err := <-nogoChan + nogoChan = nil // no cancellation needed + if err != nil { +- // TODO: Move nogo into a separate action so we don't fail the compilation here. ++ nogoStatus = nogoFailed ++ // TODO: should we still create the .x file without nogo facts in this case? + return err + } ++ nogoStatus = nogoSucceeded ++ } ++ ++ // Extract the export data file and pack it in an .x archive together with the ++ // nogo facts file (if there is one). This allows compile actions to depend ++ // on .x files only, so we don't need to recompile a package when one of its ++ // imports changes in a way that doesn't affect export data. ++ // TODO(golang/go#33820): After Go 1.16 is the minimum supported version, ++ // use -linkobj to tell the compiler to create separate .a and .x files for ++ // compiled code and export data. Before that version, the linker needed ++ // export data in the .a file when building a plugin. To work around that, ++ // we copy the export data into .x ourselves. ++ if err = extractFileFromArchive(outPath, workDir, pkgDef); err != nil { ++ return err + } +- +- return nil ++ pkgDefPath := filepath.Join(workDir, pkgDef) ++ if nogoStatus == nogoSucceeded { ++ return appendFiles(goenv, outXPath, []string{pkgDefPath, outFactsPath}) ++ } ++ return appendFiles(goenv, outXPath, []string{pkgDefPath}) + } + +-func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, pgoprofile, outLinkobjPath, outInterfacePath string) error { ++func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, pgoprofile string, outPath string) error { + args := goenv.goTool("compile") + args = append(args, "-p", packagePath, "-importcfg", importcfgPath, "-pack") + if embedcfgPath != "" { +@@ -551,24 +565,19 @@ func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPa + args = append(args, "-pgoprofile", pgoprofile) + } + args = append(args, gcFlags...) +- args = append(args, "-o", outInterfacePath) +- args = append(args, "-linkobj", outLinkobjPath) ++ args = append(args, "-o", outPath) + args = append(args, "--") + args = append(args, srcs...) + absArgs(args, []string{"-I", "-o", "-trimpath", "-importcfg"}) + return goenv.runCommand(args) + } + +-func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string, facts []archive, packagePath, importcfgPath, outFactsPath string) error { +- if len(srcs) == 0 { +- // emit_compilepkg expects a nogo facts file, even if it's empty. +- return os.WriteFile(outFactsPath, nil, 0o666) +- } ++func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string, deps []archive, packagePath, importcfgPath, outFactsPath string) error { + args := []string{nogoPath} + args = append(args, "-p", packagePath) + args = append(args, "-importcfg", importcfgPath) +- for _, fact := range facts { +- args = append(args, "-fact", fmt.Sprintf("%s=%s", fact.importPath, fact.file)) ++ for _, dep := range deps { ++ args = append(args, "-fact", fmt.Sprintf("%s=%s", dep.importPath, dep.file)) + } + args = append(args, "-x", outFactsPath) + args = append(args, srcs...) +@@ -598,13 +607,6 @@ func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string + return nil + } + +-func appendToArchive(goenv *env, outPath string, objFiles []string) error { +- // Use abs to work around long path issues on Windows. +- args := goenv.goTool("pack", "r", abs(outPath)) +- args = append(args, objFiles...) +- return goenv.runCommand(args) +-} +- + func createTrimPath(gcFlags []string, path string) string { + for _, flag := range gcFlags { + if strings.HasPrefix(flag, "-trimpath=") { +diff --git a/go/tools/builders/nogo_main.go b/go/tools/builders/nogo_main.go +index 23acdef0..17ff5314 100644 +--- a/go/tools/builders/nogo_main.go ++++ b/go/tools/builders/nogo_main.go +@@ -610,8 +610,8 @@ func (i *importer) Import(path string) (*types.Package, error) { + } + + func (i *importer) readFacts(pkgPath string) ([]byte, error) { +- facts := i.factMap[pkgPath] +- if facts == "" { ++ archive := i.factMap[pkgPath] ++ if archive == "" { + // Packages that were not built with the nogo toolchain will not be + // analyzed, so there's no opportunity to store facts. This includes + // packages in the standard library and packages built with go_tool_library, +@@ -621,7 +621,18 @@ func (i *importer) readFacts(pkgPath string) ([]byte, error) { + // fmt.Printf accepts a format string. + return nil, nil + } +- return os.ReadFile(facts) ++ factReader, err := readFileInArchive(nogoFact, archive) ++ if os.IsNotExist(err) { ++ // Packages that were not built with the nogo toolchain will not be ++ // analyzed, so there's no opportunity to store facts. This includes ++ // packages in the standard library and packages built with go_tool_library, ++ // such as coverdata. ++ return nil, nil ++ } else if err != nil { ++ return nil, err ++ } ++ defer factReader.Close() ++ return ioutil.ReadAll(factReader) + } + + type factMultiFlag map[string]string +diff --git a/go/tools/builders/pack.go b/go/tools/builders/pack.go +new file mode 100644 +index 00000000..ddbb1930 +--- /dev/null ++++ b/go/tools/builders/pack.go +@@ -0,0 +1,388 @@ ++// Copyright 2017 The Bazel Authors. All rights reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package main ++ ++import ( ++ "bufio" ++ "bytes" ++ "errors" ++ "fmt" ++ "io" ++ "io/ioutil" ++ "os" ++ "path/filepath" ++ "runtime" ++ "strconv" ++ "strings" ++) ++ ++func copyFile(inPath, outPath string) error { ++ inFile, err := os.Open(inPath) ++ if err != nil { ++ return err ++ } ++ defer inFile.Close() ++ outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) ++ if err != nil { ++ return err ++ } ++ defer outFile.Close() ++ _, err = io.Copy(outFile, inFile) ++ return err ++} ++ ++func linkFile(inPath, outPath string) error { ++ inPath, err := filepath.Abs(inPath) ++ if err != nil { ++ return err ++ } ++ return os.Symlink(inPath, outPath) ++} ++ ++func copyOrLinkFile(inPath, outPath string) error { ++ if runtime.GOOS == "windows" { ++ return copyFile(inPath, outPath) ++ } else { ++ return linkFile(inPath, outPath) ++ } ++} ++ ++const ( ++ // arHeader appears at the beginning of archives created by "ar" and ++ // "go tool pack" on all platforms. ++ arHeader = "!\n" ++ ++ // entryLength is the size in bytes of the metadata preceding each file ++ // in an archive. ++ entryLength = 60 ++ ++ // pkgDef is the name of the export data file within an archive ++ pkgDef = "__.PKGDEF" ++ ++ // nogoFact is the name of the nogo fact file ++ nogoFact = "nogo.out" ++) ++ ++var zeroBytes = []byte("0 ") ++ ++type bufioReaderWithCloser struct { ++ // bufio.Reader is needed to skip bytes in archives ++ *bufio.Reader ++ io.Closer ++} ++ ++func extractFiles(archive, dir string, names map[string]struct{}) (files []string, err error) { ++ rc, err := openArchive(archive) ++ if err != nil { ++ return nil, err ++ } ++ defer rc.Close() ++ ++ var nameData []byte ++ bufReader := rc.Reader ++ for { ++ name, size, err := readMetadata(bufReader, &nameData) ++ if err == io.EOF { ++ return files, nil ++ } ++ if err != nil { ++ return nil, err ++ } ++ if !isObjectFile(name) { ++ if err := skipFile(bufReader, size); err != nil { ++ return nil, err ++ } ++ continue ++ } ++ name, err = simpleName(name, names) ++ if err != nil { ++ return nil, err ++ } ++ name = filepath.Join(dir, name) ++ if err := extractFile(bufReader, name, size); err != nil { ++ return nil, err ++ } ++ files = append(files, name) ++ } ++} ++ ++func openArchive(archive string) (bufioReaderWithCloser, error) { ++ f, err := os.Open(archive) ++ if err != nil { ++ return bufioReaderWithCloser{}, err ++ } ++ r := bufio.NewReader(f) ++ header := make([]byte, len(arHeader)) ++ if _, err := io.ReadFull(r, header); err != nil || string(header) != arHeader { ++ f.Close() ++ return bufioReaderWithCloser{}, fmt.Errorf("%s: bad header", archive) ++ } ++ return bufioReaderWithCloser{r, f}, nil ++} ++ ++// readMetadata reads the relevant fields of an entry. Before calling, ++// r must be positioned at the beginning of an entry. Afterward, r will ++// be positioned at the beginning of the file data. io.EOF is returned if ++// there are no more files in the archive. ++// ++// Both BSD and GNU / SysV naming conventions are supported. ++func readMetadata(r *bufio.Reader, nameData *[]byte) (name string, size int64, err error) { ++retry: ++ // Each file is preceded by a 60-byte header that contains its metadata. ++ // We only care about two fields, name and size. Other fields (mtime, ++ // owner, group, mode) are ignored because they don't affect compilation. ++ var entry [entryLength]byte ++ if _, err := io.ReadFull(r, entry[:]); err != nil { ++ return "", 0, err ++ } ++ ++ sizeField := strings.TrimSpace(string(entry[48:58])) ++ size, err = strconv.ParseInt(sizeField, 10, 64) ++ if err != nil { ++ return "", 0, err ++ } ++ ++ nameField := strings.TrimRight(string(entry[:16]), " ") ++ switch { ++ case strings.HasPrefix(nameField, "#1/"): ++ // BSD-style name. The number of bytes in the name is written here in ++ // ASCII, right-padded with spaces. The actual name is stored at the ++ // beginning of the file data, left-padded with NUL bytes. ++ nameField = nameField[len("#1/"):] ++ nameLen, err := strconv.ParseInt(nameField, 10, 64) ++ if err != nil { ++ return "", 0, err ++ } ++ nameBuf := make([]byte, nameLen) ++ if _, err := io.ReadFull(r, nameBuf); err != nil { ++ return "", 0, err ++ } ++ name = strings.TrimRight(string(nameBuf), "\x00") ++ size -= nameLen ++ ++ case nameField == "//": ++ // GNU / SysV-style name data. This is a fake file that contains names ++ // for files with long names. We read this into nameData, then read ++ // the next entry. ++ *nameData = make([]byte, size) ++ if _, err := io.ReadFull(r, *nameData); err != nil { ++ return "", 0, err ++ } ++ if size%2 != 0 { ++ // Files are aligned at 2-byte offsets. Discard the padding byte if the ++ // size was odd. ++ if _, err := r.ReadByte(); err != nil { ++ return "", 0, err ++ } ++ } ++ goto retry ++ ++ case nameField == "/": ++ // GNU / SysV-style symbol lookup table. Skip. ++ if err := skipFile(r, size); err != nil { ++ return "", 0, err ++ } ++ goto retry ++ ++ case strings.HasPrefix(nameField, "/"): ++ // GNU / SysV-style long file name. The number that follows the slash is ++ // an offset into the name data that should have been read earlier. ++ // The file name ends with a slash. ++ nameField = nameField[1:] ++ nameOffset, err := strconv.Atoi(nameField) ++ if err != nil { ++ return "", 0, err ++ } ++ if nameData == nil || nameOffset < 0 || nameOffset >= len(*nameData) { ++ return "", 0, fmt.Errorf("invalid name length: %d", nameOffset) ++ } ++ i := bytes.IndexByte((*nameData)[nameOffset:], '/') ++ if i < 0 { ++ return "", 0, errors.New("file name does not end with '/'") ++ } ++ name = string((*nameData)[nameOffset : nameOffset+i]) ++ ++ case strings.HasSuffix(nameField, "/"): ++ // GNU / SysV-style short file name. ++ name = nameField[:len(nameField)-1] ++ ++ default: ++ // Common format name. ++ name = nameField ++ } ++ ++ return name, size, err ++} ++ ++// extractFile reads size bytes from r and writes them to a new file, name. ++func extractFile(r *bufio.Reader, name string, size int64) error { ++ w, err := os.Create(name) ++ if err != nil { ++ return err ++ } ++ defer w.Close() ++ _, err = io.CopyN(w, r, size) ++ if err != nil { ++ return err ++ } ++ if size%2 != 0 { ++ // Files are aligned at 2-byte offsets. Discard the padding byte if the ++ // size was odd. ++ if _, err := r.ReadByte(); err != nil { ++ return err ++ } ++ } ++ return nil ++} ++ ++func skipFile(r *bufio.Reader, size int64) error { ++ if size%2 != 0 { ++ // Files are aligned at 2-byte offsets. Discard the padding byte if the ++ // size was odd. ++ size += 1 ++ } ++ _, err := r.Discard(int(size)) ++ return err ++} ++ ++func isObjectFile(name string) bool { ++ return strings.HasSuffix(name, ".o") ++} ++ ++// simpleName returns a file name which is at most 15 characters ++// and doesn't conflict with other names. If it is not possible to choose ++// such a name, simpleName will truncate the given name to 15 characters. ++// The original file extension will be preserved. ++func simpleName(name string, names map[string]struct{}) (string, error) { ++ if _, ok := names[name]; !ok && len(name) < 16 { ++ names[name] = struct{}{} ++ return name, nil ++ } ++ var stem, ext string ++ if i := strings.LastIndexByte(name, '.'); i < 0 { ++ stem = name ++ } else { ++ stem = strings.Replace(name[:i], ".", "_", -1) ++ ext = name[i:] ++ } ++ for n := 0; n < len(names)+1; n++ { ++ ns := strconv.Itoa(n) ++ stemLen := 15 - len(ext) - len(ns) ++ if stemLen < 0 { ++ break ++ } ++ if stemLen > len(stem) { ++ stemLen = len(stem) ++ } ++ candidate := stem[:stemLen] + ns + ext ++ if _, ok := names[candidate]; !ok { ++ names[candidate] = struct{}{} ++ return candidate, nil ++ } ++ } ++ return "", fmt.Errorf("cannot shorten file name: %q", name) ++} ++ ++func appendFiles(goenv *env, archive string, files []string) error { ++ archive = abs(archive) // required for long filenames on Windows. ++ ++ // Create an empty archive if one doesn't already exist. ++ // In Go 1.16, 'go tool pack r' reports an error if the archive doesn't exist. ++ // 'go tool pack c' copies export data in addition to creating the archive, ++ // so we don't want to use that directly. ++ _, err := os.Stat(archive) ++ if err != nil && !os.IsNotExist(err) { ++ return err ++ } ++ if os.IsNotExist(err) { ++ if err := ioutil.WriteFile(archive, []byte(arHeader), 0666); err != nil { ++ return err ++ } ++ } ++ ++ // Append files to the archive. ++ // TODO(jayconrod): copy cmd/internal/archive and use that instead of ++ // shelling out to cmd/pack. ++ args := goenv.goTool("pack", "r", archive) ++ args = append(args, files...) ++ return goenv.runCommand(args) ++} ++ ++type readWithCloser struct { ++ io.Reader ++ io.Closer ++} ++ ++func readFileInArchive(fileName, archive string) (io.ReadCloser, error) { ++ rc, err := openArchive(archive) ++ if err != nil { ++ return nil, err ++ } ++ var nameData []byte ++ bufReader := rc.Reader ++ for err == nil { ++ // avoid shadowing err in the loop it can be returned correctly in the end ++ var ( ++ name string ++ size int64 ++ ) ++ name, size, err = readMetadata(bufReader, &nameData) ++ if err != nil { ++ break ++ } ++ if name == fileName { ++ return readWithCloser{ ++ Reader: io.LimitReader(rc, size), ++ Closer: rc, ++ }, nil ++ } ++ err = skipFile(bufReader, size) ++ } ++ if err == io.EOF { ++ err = os.ErrNotExist ++ } ++ rc.Close() ++ return nil, err ++} ++ ++func extractFileFromArchive(archive, dir, name string) (err error) { ++ archiveReader, err := readFileInArchive(name, archive) ++ if err != nil { ++ return fmt.Errorf("error reading %s from %s: %v", name, archive, err) ++ } ++ defer func() { ++ e := archiveReader.Close() ++ if e != nil && err == nil { ++ err = fmt.Errorf("error closing %q: %v", archive, e) ++ } ++ }() ++ outPath := filepath.Join(dir, pkgDef) ++ outFile, err := os.Create(outPath) ++ if err != nil { ++ return fmt.Errorf("error creating %s: %v", outPath, err) ++ } ++ defer func() { ++ e := outFile.Close() ++ if e != nil && err == nil { ++ err = fmt.Errorf("error closing %q: %v", outPath, e) ++ } ++ }() ++ if size, err := io.Copy(outFile, archiveReader); err != nil { ++ return fmt.Errorf("error writing %s: %v", outPath, err) ++ } else if size == 0 { ++ return fmt.Errorf("%s is empty in %s", name, archive) ++ } ++ return err ++}