[dev.boringcrypto] all: merge master into dev.boringcrypto

Change-Id: I0f610a900fcd5575ca12b34bc74fa63c2146b10b

1 files changed, 479 insertions, 356 deletions

diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go
index 49bd98216c..12bf0e1f7a 100644
--- a/src/cmd/go/internal/load/pkg.go
+++ b/src/cmd/go/internal/load/pkg.go
@@ -7,6 +7,7 @@ package load
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"go/build"
 	"go/token"
@@ -14,6 +15,7 @@ import (
 	"os"
 	pathpkg "path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -23,6 +25,7 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/modinfo"
+	"cmd/go/internal/par"
 	"cmd/go/internal/search"
 	"cmd/go/internal/str"
 )
@@ -32,14 +35,14 @@ var (
 	ModInit func()
 
 	// module hooks; nil if module use is disabled
-	ModBinDir            func() string                                       // return effective bin directory
-	ModLookup            func(path string) (dir, realPath string, err error) // lookup effective meaning of import
-	ModPackageModuleInfo func(path string) *modinfo.ModulePublic             // return module info for Package struct
-	ModImportPaths       func(args []string) []*search.Match                 // expand import paths
-	ModPackageBuildInfo  func(main string, deps []string) string             // return module info to embed in binary
-	ModInfoProg          func(info string) []byte                            // wrap module info in .go code for binary
-	ModImportFromFiles   func([]string)                                      // update go.mod to add modules for imports in these files
-	ModDirImportPath     func(string) string                                 // return effective import path for directory
+	ModBinDir            func() string                                                                            // return effective bin directory
+	ModLookup            func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import
+	ModPackageModuleInfo func(path string) *modinfo.ModulePublic                                                  // return module info for Package struct
+	ModImportPaths       func(args []string) []*search.Match                                                      // expand import paths
+	ModPackageBuildInfo  func(main string, deps []string) string                                                  // return module info to embed in binary
+	ModInfoProg          func(info string) []byte                                                                 // wrap module info in .go code for binary
+	ModImportFromFiles   func([]string)                                                                           // update go.mod to add modules for imports in these files
+	ModDirImportPath     func(string) string                                                                      // return effective import path for directory
 )
 
 var IgnoreImports bool // control whether we ignore imports in packages
@@ -174,7 +177,8 @@ type PackageInternal struct {
 	OmitDebug         bool                 // tell linker not to write debug information
 	GobinSubdir       bool                 // install target would be subdir of GOBIN
 	BuildInfo         string               // add this info to package main
-	TestmainGo        *[]byte              // content for _testmain.go
+	TestinginitGo     []byte               // content for _testinginit.go
+	TestmainGo        []byte               // content for _testmain.go
 
 	Asmflags   []string // -asmflags for this package
 	Gcflags    []string // -gcflags for this package
@@ -362,38 +366,56 @@ func (sp *ImportStack) shorterThan(t []string) bool {
 	return false // they are equal
 }
 
-// packageCache is a lookup cache for loadPackage,
+// packageCache is a lookup cache for LoadImport,
 // so that if we look up a package multiple times
 // we return the same pointer each time.
 var packageCache = map[string]*Package{}
 
+// ClearPackageCache clears the in-memory package cache and the preload caches.
+// It is only for use by GOPATH-based "go get".
+// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function.
 func ClearPackageCache() {
 	for name := range packageCache {
 		delete(packageCache, name)
 	}
+	resolvedImportCache.Clear()
+	packageDataCache.Clear()
 }
 
+// ClearPackageCachePartial clears packages with the given import paths from the
+// in-memory package cache and the preload caches. It is only for use by
+// GOPATH-based "go get".
+// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function.
 func ClearPackageCachePartial(args []string) {
+	shouldDelete := make(map[string]bool)
 	for _, arg := range args {
-		p := packageCache[arg]
-		if p != nil {
-			delete(packageCache, p.Dir)
-			delete(packageCache, p.ImportPath)
+		shouldDelete[arg] = true
+		if p := packageCache[arg]; p != nil {
+			delete(packageCache, arg)
 		}
 	}
+	resolvedImportCache.DeleteIf(func(key interface{}) bool {
+		return shouldDelete[key.(importSpec).path]
+	})
+	packageDataCache.DeleteIf(func(key interface{}) bool {
+		return shouldDelete[key.(string)]
+	})
 }
 
-// ReloadPackageNoFlags is like LoadPackageNoFlags but makes sure
+// ReloadPackageNoFlags is like LoadImport but makes sure
 // not to use the package cache.
 // It is only for use by GOPATH-based "go get".
 // TODO(rsc): When GOPATH-based "go get" is removed, delete this function.
 func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package {
 	p := packageCache[arg]
 	if p != nil {
-		delete(packageCache, p.Dir)
-		delete(packageCache, p.ImportPath)
+		delete(packageCache, arg)
+		resolvedImportCache.DeleteIf(func(key interface{}) bool {
+			return key.(importSpec).path == p.ImportPath
+		})
+		packageDataCache.Delete(p.ImportPath)
 	}
-	return LoadPackageNoFlags(arg, stk)
+	return LoadImport(arg, base.Cwd, nil, stk, nil, 0)
 }
 
 // dirToImportPath returns the pseudo-import path we use for a package
@@ -446,6 +468,10 @@ const (
 // this package, as part of a bigger load operation, and by GOPATH-based "go get".
 // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function.
 func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+	return loadImport(nil, path, srcDir, parent, stk, importPos, mode)
+}
+
+func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
 	if path == "" {
 		panic("LoadImport called with empty package path")
 	}
@@ -453,125 +479,51 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 	stk.Push(path)
 	defer stk.Pop()
 
-	if strings.HasPrefix(path, "mod/") {
-		// Paths beginning with "mod/" might accidentally
-		// look in the module cache directory tree in $GOPATH/pkg/mod/.
-		// This prefix is owned by the Go core for possible use in the
-		// standard library (since it does not begin with a domain name),
-		// so it's OK to disallow entirely.
-		return &Package{
-			PackagePublic: PackagePublic{
-				ImportPath: path,
-				Error: &PackageError{
-					ImportStack: stk.Copy(),
-					Err:         fmt.Sprintf("disallowed import path %q", path),
-				},
-			},
-		}
+	var parentPath, parentRoot string
+	parentIsStd := false
+	if parent != nil {
+		parentPath = parent.ImportPath
+		parentRoot = parent.Root
+		parentIsStd = parent.Standard
 	}
-
-	if strings.Contains(path, "@") {
-		var text string
-		if cfg.ModulesEnabled {
-			text = "can only use path@version syntax with 'go get'"
-		} else {
-			text = "cannot use path@version syntax in GOPATH mode"
-		}
+	bp, loaded, err := loadPackageData(path, parentPath, srcDir, parentRoot, parentIsStd, mode)
+	if loaded && pre != nil && !IgnoreImports {
+		pre.preloadImports(bp.Imports, bp)
+	}
+	if bp == nil {
 		return &Package{
 			PackagePublic: PackagePublic{
 				ImportPath: path,
 				Error: &PackageError{
 					ImportStack: stk.Copy(),
-					Err:         text,
+					Err:         err.Error(),
 				},
 			},
 		}
 	}
 
-	parentPath := ""
-	if parent != nil {
-		parentPath = parent.ImportPath
-	}
-
-	// Determine canonical identifier for this package.
-	// For a local import the identifier is the pseudo-import path
-	// we create from the full directory to the package.
-	// Otherwise it is the usual import path.
-	// For vendored imports, it is the expanded form.
-	importPath := path
-	origPath := path
-	isLocal := build.IsLocalImport(path)
-	var modDir string
-	var modErr error
-	if isLocal {
-		importPath = dirToImportPath(filepath.Join(srcDir, path))
-	} else if cfg.ModulesEnabled {
-		var p string
-		modDir, p, modErr = ModLookup(path)
-		if modErr == nil {
-			importPath = p
-		}
-	} else if mode&ResolveImport != 0 {
-		// We do our own path resolution, because we want to
-		// find out the key to use in packageCache without the
-		// overhead of repeated calls to buildContext.Import.
-		// The code is also needed in a few other places anyway.
-		path = ResolveImportPath(parent, path)
-		importPath = path
-	} else if mode&ResolveModule != 0 {
-		path = ModuleImportPath(parent, path)
-		importPath = path
-	}
-
+	importPath := bp.ImportPath
 	p := packageCache[importPath]
 	if p != nil {
 		p = reusePackage(p, stk)
 	} else {
 		p = new(Package)
-		p.Internal.Local = isLocal
+		p.Internal.Local = build.IsLocalImport(path)
 		p.ImportPath = importPath
 		packageCache[importPath] = p
 
 		// Load package.
-		// Import always returns bp != nil, even if an error occurs,
+		// loadPackageData may return bp != nil even if an error occurs,
 		// in order to return partial information.
-		var bp *build.Package
-		var err error
-		if modDir != "" {
-			bp, err = cfg.BuildContext.ImportDir(modDir, 0)
-		} else if modErr != nil {
-			bp = new(build.Package)
-			err = fmt.Errorf("unknown import path %q: %v", importPath, modErr)
-		} else if cfg.ModulesEnabled && path != "unsafe" {
-			bp = new(build.Package)
-			err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", importPath)
-		} else {
-			buildMode := build.ImportComment
-			if mode&ResolveImport == 0 || path != origPath {
-				// Not vendoring, or we already found the vendored path.
-				buildMode |= build.IgnoreVendor
-			}
-			bp, err = cfg.BuildContext.Import(path, srcDir, buildMode)
-		}
-		bp.ImportPath = importPath
-		if cfg.GOBIN != "" {
-			bp.BinDir = cfg.GOBIN
-		} else if cfg.ModulesEnabled {
-			bp.BinDir = ModBinDir()
-		}
-		if modDir == "" && err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path &&
-			!strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") {
-			err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment)
-		}
 		p.load(stk, bp, err)
 		if p.Error != nil && p.Error.Pos == "" {
 			p = setErrorPos(p, importPos)
 		}
 
-		if modDir == "" && origPath != cleanImport(origPath) {
+		if !cfg.ModulesEnabled && path != cleanImport(path) {
 			p.Error = &PackageError{
 				ImportStack: stk.Copy(),
-				Err:         fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)),
+				Err:         fmt.Sprintf("non-canonical import path: %q should be %q", path, pathpkg.Clean(path)),
 			}
 			p.Incomplete = true
 		}
@@ -582,7 +534,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 		return setErrorPos(perr, importPos)
 	}
 	if mode&ResolveImport != 0 {
-		if perr := disallowVendor(srcDir, parent, parentPath, origPath, p, stk); perr != p {
+		if perr := disallowVendor(srcDir, parent, parentPath, path, p, stk); perr != p {
 			return setErrorPos(perr, importPos)
 		}
 	}
@@ -598,9 +550,13 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 
 	if p.Internal.Local && parent != nil && !parent.Internal.Local {
 		perr := *p
+		errMsg := fmt.Sprintf("local import %q in non-local package", path)
+		if path == "." {
+			errMsg = "cannot import current directory"
+		}
 		perr.Error = &PackageError{
 			ImportStack: stk.Copy(),
-			Err:         fmt.Sprintf("local import %q in non-local package", path),
+			Err:         errMsg,
 		}
 		return setErrorPos(&perr, importPos)
 	}
@@ -617,6 +573,244 @@ func setErrorPos(p *Package, importPos []token.Position) *Package {
 	return p
 }
 
+// loadPackageData loads information needed to construct a *Package. The result
+// is cached, and later calls to loadPackageData for the same package will return
+// the same data.
+//
+// loadPackageData returns a non-nil package even if err is non-nil unless
+// the package path is malformed (for example, the path contains "mod/" or "@").
+//
+// loadPackageData returns a boolean, loaded, which is true if this is the
+// first time the package was loaded. Callers may preload imports in this case.
+func loadPackageData(path, parentPath, parentDir, parentRoot string, parentIsStd bool, mode int) (bp *build.Package, loaded bool, err error) {
+	if path == "" {
+		panic("loadPackageData called with empty package path")
+	}
+
+	if strings.HasPrefix(path, "mod/") {
+		// Paths beginning with "mod/" might accidentally
+		// look in the module cache directory tree in $GOPATH/pkg/mod/.
+		// This prefix is owned by the Go core for possible use in the
+		// standard library (since it does not begin with a domain name),
+		// so it's OK to disallow entirely.
+		return nil, false, fmt.Errorf("disallowed import path %q", path)
+	}
+
+	if strings.Contains(path, "@") {
+		if cfg.ModulesEnabled {
+			return nil, false, errors.New("can only use path@version syntax with 'go get'")
+		} else {
+			return nil, false, errors.New("cannot use path@version syntax in GOPATH mode")
+		}
+	}
+
+	// Determine canonical package path and directory.
+	// For a local import the identifier is the pseudo-import path
+	// we create from the full directory to the package.
+	// Otherwise it is the usual import path.
+	// For vendored imports, it is the expanded form.
+	importKey := importSpec{
+		path:        path,
+		parentPath:  parentPath,
+		parentDir:   parentDir,
+		parentRoot:  parentRoot,
+		parentIsStd: parentIsStd,
+		mode:        mode,
+	}
+	r := resolvedImportCache.Do(importKey, func() interface{} {
+		var r resolvedImport
+		if build.IsLocalImport(path) {
+			r.dir = filepath.Join(parentDir, path)
+			r.path = dirToImportPath(r.dir)
+		} else if cfg.ModulesEnabled {
+			r.dir, r.path, r.err = ModLookup(parentPath, parentIsStd, path)
+		} else if mode&ResolveImport != 0 {
+			// We do our own path resolution, because we want to
+			// find out the key to use in packageCache without the
+			// overhead of repeated calls to buildContext.Import.
+			// The code is also needed in a few other places anyway.
+			r.path = resolveImportPath(path, parentPath, parentDir, parentRoot, parentIsStd)
+		} else if mode&ResolveModule != 0 {
+			r.path = moduleImportPath(path, parentPath, parentDir, parentRoot)
+		}
+		if r.path == "" {
+			r.path = path
+		}
+		return r
+	}).(resolvedImport)
+	// Invariant: r.path is set to the resolved import path. If the path cannot
+	// be resolved, r.path is set to path, the source import path.
+	// r.path is never empty.
+
+	// Load the package from its directory. If we already found the package's
+	// directory when resolving its import path, use that.
+	data := packageDataCache.Do(r.path, func() interface{} {
+		loaded = true
+		var data packageData
+		if r.dir != "" {
+			var buildMode build.ImportMode
+			if !cfg.ModulesEnabled {
+				buildMode = build.ImportComment
+			}
+			data.p, data.err = cfg.BuildContext.ImportDir(r.dir, buildMode)
+		} else if r.err != nil {
+			data.p = new(build.Package)
+			data.err = fmt.Errorf("unknown import path %q: %v", r.path, r.err)
+		} else if cfg.ModulesEnabled && path != "unsafe" {
+			data.p = new(build.Package)
+			data.err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", r.path)
+		} else {
+			buildMode := build.ImportComment
+			if mode&ResolveImport == 0 || r.path != path {
+				// Not vendoring, or we already found the vendored path.
+				buildMode |= build.IgnoreVendor
+			}
+			data.p, data.err = cfg.BuildContext.Import(r.path, parentDir, buildMode)
+		}
+		data.p.ImportPath = r.path
+		if cfg.GOBIN != "" {
+			data.p.BinDir = cfg.GOBIN
+		} else if cfg.ModulesEnabled && !data.p.Goroot {
+			data.p.BinDir = ModBinDir()
+		}
+		if !cfg.ModulesEnabled && data.err == nil &&
+			data.p.ImportComment != "" && data.p.ImportComment != path &&
+			!strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") {
+			data.err = fmt.Errorf("code in directory %s expects import %q", data.p.Dir, data.p.ImportComment)
+		}
+		return data
+	}).(packageData)
+
+	return data.p, loaded, data.err
+}
+
+// importSpec describes an import declaration in source code. It is used as a
+// cache key for resolvedImportCache.
+type importSpec struct {
+	path                              string
+	parentPath, parentDir, parentRoot string
+	parentIsStd                       bool
+	mode                              int
+}
+
+// resolvedImport holds a canonical identifier for a package. It may also contain
+// a path to the package's directory and an error if one occurred. resolvedImport
+// is the value type in resolvedImportCache.
+type resolvedImport struct {
+	path, dir string
+	err       error
+}
+
+// packageData holds information loaded from a package. It is the value type
+// in packageDataCache.
+type packageData struct {
+	p   *build.Package
+	err error
+}
+
+// resolvedImportCache maps import strings (importSpec) to canonical package names
+// (resolvedImport).
+var resolvedImportCache par.Cache
+
+// packageDataCache maps canonical package names (string) to package metadata
+// (packageData).
+var packageDataCache par.Cache
+
+// preloadWorkerCount is the number of concurrent goroutines that can load
+// packages. Experimentally, there are diminishing returns with more than
+// 4 workers. This was measured on the following machines.
+//
+// * MacBookPro with a 4-core Intel Core i7 CPU
+// * Linux workstation with 6-core Intel Xeon CPU
+// * Linux workstation with 24-core Intel Xeon CPU
+//
+// It is very likely (though not confirmed) that this workload is limited
+// by memory bandwidth. We don't have a good way to determine the number of
+// workers that would saturate the bus though, so runtime.GOMAXPROCS
+// seems like a reasonable default.
+var preloadWorkerCount = runtime.GOMAXPROCS(0)
+
+// preload holds state for managing concurrent preloading of package data.
+//
+// A preload should be created with newPreload before loading a large
+// package graph. flush must be called when package loading is complete
+// to ensure preload goroutines are no longer active. This is necessary
+// because of global mutable state that cannot safely be read and written
+// concurrently. In particular, packageDataCache may be cleared by "go get"
+// in GOPATH mode, and modload.loaded (accessed via ModLookup) may be
+// modified by modload.ImportPaths (ModImportPaths).
+type preload struct {
+	cancel chan struct{}
+	sema   chan struct{}
+}
+
+// newPreload creates a new preloader. flush must be called later to avoid
+// accessing global state while it is being modified.
+func newPreload() *preload {
+	pre := &preload{
+		cancel: make(chan struct{}),
+		sema:   make(chan struct{}, preloadWorkerCount),
+	}
+	return pre
+}
+
+// preloadMatches loads data for package paths matched by patterns.
+// When preloadMatches returns, some packages may not be loaded yet, but
+// loadPackageData and loadImport are always safe to call.
+func (pre *preload) preloadMatches(matches []*search.Match) {
+	for _, m := range matches {
+		for _, pkg := range m.Pkgs {
+			select {
+			case <-pre.cancel:
+				return
+			case pre.sema <- struct{}{}:
+				go func(pkg string) {
+					mode := 0 // don't use vendoring or module import resolution
+					bp, loaded, err := loadPackageData(pkg, "", base.Cwd, "", false, mode)
+					<-pre.sema
+					if bp != nil && loaded && err == nil && !IgnoreImports {
+						pre.preloadImports(bp.Imports, bp)
+					}
+				}(pkg)
+			}
+		}
+	}
+}
+
+// preloadImports queues a list of imports for preloading.
+// When preloadImports returns, some packages may not be loaded yet,
+// but loadPackageData and loadImport are always safe to call.
+func (pre *preload) preloadImports(imports []string, parent *build.Package) {
+	parentIsStd := parent.Goroot && parent.ImportPath != "" && search.IsStandardImportPath(parent.ImportPath)
+	for _, path := range imports {
+		if path == "C" || path == "unsafe" {
+			continue
+		}
+		select {
+		case <-pre.cancel:
+			return
+		case pre.sema <- struct{}{}:
+			go func(path string) {
+				bp, loaded, err := loadPackageData(path, parent.ImportPath, parent.Dir, parent.Root, parentIsStd, ResolveImport)
+				<-pre.sema
+				if bp != nil && loaded && err == nil && !IgnoreImports {
+					pre.preloadImports(bp.Imports, bp)
+				}
+			}(path)
+		}
+	}
+}
+
+// flush stops pending preload operations. flush blocks until preload calls to
+// loadPackageData have completed. The preloader will not make any new calls
+// to loadPackageData.
+func (pre *preload) flush() {
+	close(pre.cancel)
+	for i := 0; i < preloadWorkerCount; i++ {
+		pre.sema <- struct{}{}
+	}
+}
+
 func cleanImport(path string) string {
 	orig := path
 	path = pathpkg.Clean(path)
@@ -626,18 +820,13 @@ func cleanImport(path string) string {
 	return path
 }
 
-var isDirCache = map[string]bool{}
+var isDirCache par.Cache
 
 func isDir(path string) bool {
-	result, ok := isDirCache[path]
-	if ok {
-		return result
-	}
-
-	fi, err := os.Stat(path)
-	result = err == nil && fi.IsDir()
-	isDirCache[path] = result
-	return result
+	return isDirCache.Do(path, func() interface{} {
+		fi, err := os.Stat(path)
+		return err == nil && fi.IsDir()
+	}).(bool)
 }
 
 // ResolveImportPath returns the true meaning of path when it appears in parent.
@@ -646,31 +835,44 @@ func isDir(path string) bool {
 // If vendor expansion doesn't trigger, then the path is also subject to
 // Go 1.11 module legacy conversion (golang.org/issue/25069).
 func ResolveImportPath(parent *Package, path string) (found string) {
+	var parentPath, parentDir, parentRoot string
+	parentIsStd := false
+	if parent != nil {
+		parentPath = parent.ImportPath
+		parentDir = parent.Dir
+		parentRoot = parent.Root
+		parentIsStd = parent.Standard
+	}
+	return resolveImportPath(path, parentPath, parentDir, parentRoot, parentIsStd)
+}
+
+func resolveImportPath(path, parentPath, parentDir, parentRoot string, parentIsStd bool) (found string) {
 	if cfg.ModulesEnabled {
-		if _, p, e := ModLookup(path); e == nil {
+		if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil {
 			return p
 		}
 		return path
 	}
-	found = VendoredImportPath(parent, path)
+	found = vendoredImportPath(path, parentPath, parentDir, parentRoot)
 	if found != path {
 		return found
 	}
-	return ModuleImportPath(parent, path)
+	return moduleImportPath(path, parentPath, parentDir, parentRoot)
 }
 
 // dirAndRoot returns the source directory and workspace root
 // for the package p, guaranteeing that root is a path prefix of dir.
-func dirAndRoot(p *Package) (dir, root string) {
-	dir = filepath.Clean(p.Dir)
-	root = filepath.Join(p.Root, "src")
-	if !str.HasFilePathPrefix(dir, root) || p.ImportPath != "command-line-arguments" && filepath.Join(root, p.ImportPath) != dir {
+func dirAndRoot(path string, dir, root string) (string, string) {
+	origDir, origRoot := dir, root
+	dir = filepath.Clean(dir)
+	root = filepath.Join(root, "src")
+	if !str.HasFilePathPrefix(dir, root) || path != "command-line-arguments" && filepath.Join(root, path) != dir {
 		// Look for symlinks before reporting error.
 		dir = expandPath(dir)
 		root = expandPath(root)
 	}
 
-	if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || p.ImportPath != "command-line-arguments" && !p.Internal.Local && filepath.Join(root, p.ImportPath) != dir {
+	if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || path != "command-line-arguments" && !build.IsLocalImport(path) && filepath.Join(root, path) != dir {
 		base.Fatalf("unexpected directory layout:\n"+
 			"	import path: %s\n"+
 			"	root: %s\n"+
@@ -678,27 +880,27 @@ func dirAndRoot(p *Package) (dir, root string) {
 			"	expand root: %s\n"+
 			"	expand dir: %s\n"+
 			"	separator: %s",
-			p.ImportPath,
-			filepath.Join(p.Root, "src"),
-			filepath.Clean(p.Dir),
-			root,
-			dir,
+			path,
+			filepath.Join(origRoot, "src"),
+			filepath.Clean(origDir),
+			origRoot,
+			origDir,
 			string(filepath.Separator))
 	}
 
 	return dir, root
 }
 
-// VendoredImportPath returns the vendor-expansion of path when it appears in parent.
+// vendoredImportPath returns the vendor-expansion of path when it appears in parent.
 // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path,
 // x/vendor/path, vendor/path, or else stay path if none of those exist.
-// VendoredImportPath returns the expanded path or, if no expansion is found, the original.
-func VendoredImportPath(parent *Package, path string) (found string) {
-	if parent == nil || parent.Root == "" {
+// vendoredImportPath returns the expanded path or, if no expansion is found, the original.
+func vendoredImportPath(path, parentPath, parentDir, parentRoot string) (found string) {
+	if parentRoot == "" {
 		return path
 	}
 
-	dir, root := dirAndRoot(parent)
+	dir, root := dirAndRoot(parentPath, parentDir, parentRoot)
 
 	vpath := "vendor/" + path
 	for i := len(dir); i >= len(root); i-- {
@@ -714,7 +916,7 @@ func VendoredImportPath(parent *Package, path string) (found string) {
 		}
 		targ := filepath.Join(dir[:i], vpath)
 		if isDir(targ) && hasGoFiles(targ) {
-			importPath := parent.ImportPath
+			importPath := parentPath
 			if importPath == "command-line-arguments" {
 				// If parent.ImportPath is 'command-line-arguments'.
 				// set to relative directory to root (also chopped root directory)
@@ -744,54 +946,48 @@ func VendoredImportPath(parent *Package, path string) (found string) {
 
 var (
 	modulePrefix   = []byte("\nmodule ")
-	goModPathCache = make(map[string]string)
+	goModPathCache par.Cache
 )
 
 // goModPath returns the module path in the go.mod in dir, if any.
 func goModPath(dir string) (path string) {
-	path, ok := goModPathCache[dir]
-	if ok {
-		return path
-	}
-	defer func() {
-		goModPathCache[dir] = path
-	}()
-
-	data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
-	if err != nil {
-		return ""
-	}
-	var i int
-	if bytes.HasPrefix(data, modulePrefix[1:]) {
-		i = 0
-	} else {
-		i = bytes.Index(data, modulePrefix)
-		if i < 0 {
+	return goModPathCache.Do(dir, func() interface{} {
+		data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
+		if err != nil {
 			return ""
 		}
-		i++
-	}
-	line := data[i:]
+		var i int
+		if bytes.HasPrefix(data, modulePrefix[1:]) {
+			i = 0
+		} else {
+			i = bytes.Index(data, modulePrefix)
+			if i < 0 {
+				return ""
+			}
+			i++
+		}
+		line := data[i:]
 
-	// Cut line at \n, drop trailing \r if present.
-	if j := bytes.IndexByte(line, '\n'); j >= 0 {
-		line = line[:j]
-	}
-	if line[len(line)-1] == '\r' {
-		line = line[:len(line)-1]
-	}
-	line = line[len("module "):]
+		// Cut line at \n, drop trailing \r if present.
+		if j := bytes.IndexByte(line, '\n'); j >= 0 {
+			line = line[:j]
+		}
+		if line[len(line)-1] == '\r' {
+			line = line[:len(line)-1]
+		}
+		line = line[len("module "):]
 
-	// If quoted, unquote.
-	path = strings.TrimSpace(string(line))
-	if path != "" && path[0] == '"' {
-		s, err := strconv.Unquote(path)
-		if err != nil {
-			return ""
+		// If quoted, unquote.
+		path = strings.TrimSpace(string(line))
+		if path != "" && path[0] == '"' {
+			s, err := strconv.Unquote(path)
+			if err != nil {
+				return ""
+			}
+			path = s
 		}
-		path = s
-	}
-	return path
+		return path
+	}).(string)
 }
 
 // findVersionElement returns the slice indices of the final version element /vN in path.
@@ -800,7 +996,7 @@ func findVersionElement(path string) (i, j int) {
 	j = len(path)
 	for i = len(path) - 1; i >= 0; i-- {
 		if path[i] == '/' {
-			if isVersionElement(path[i:j]) {
+			if isVersionElement(path[i+1 : j]) {
 				return i, j
 			}
 			j = i
@@ -812,10 +1008,10 @@ func findVersionElement(path string) (i, j int) {
 // isVersionElement reports whether s is a well-formed path version element:
 // v2, v3, v10, etc, but not v0, v05, v1.
 func isVersionElement(s string) bool {
-	if len(s) < 3 || s[0] != '/' || s[1] != 'v' || s[2] == '0' || s[2] == '1' && len(s) == 3 {
+	if len(s) < 2 || s[0] != 'v' || s[1] == '0' || s[1] == '1' && len(s) == 2 {
 		return false
 	}
-	for i := 2; i < len(s); i++ {
+	for i := 1; i < len(s); i++ {
 		if s[i] < '0' || '9' < s[i] {
 			return false
 		}
@@ -823,7 +1019,7 @@ func isVersionElement(s string) bool {
 	return true
 }
 
-// ModuleImportPath translates import paths found in go modules
+// moduleImportPath translates import paths found in go modules
 // back down to paths that can be resolved in ordinary builds.
 //
 // Define “new” code as code with a go.mod file in the same directory
@@ -831,8 +1027,8 @@ func isVersionElement(s string) bool {
 // x/y/v2/z does not exist and x/y/go.mod says “module x/y/v2”,
 // then go build will read the import as x/y/z instead.
 // See golang.org/issue/25069.
-func ModuleImportPath(parent *Package, path string) (found string) {
-	if parent == nil || parent.Root == "" {
+func moduleImportPath(path, parentPath, parentDir, parentRoot string) (found string) {
+	if parentRoot == "" {
 		return path
 	}
 
@@ -844,7 +1040,7 @@ func ModuleImportPath(parent *Package, path string) (found string) {
 		return path
 	}
 
-	dir, root := dirAndRoot(parent)
+	dir, root := dirAndRoot(parentPath, parentDir, parentRoot)
 
 	// Consider dir and parents, up to and including root.
 	for i := len(dir); i >= len(root); i-- {
@@ -967,6 +1163,13 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p *
 		return p
 	}
 
+	// The sort package depends on internal/reflectlite, but during bootstrap
+	// the path rewriting causes the normal internal checks to fail.
+	// Instead, just ignore the internal rules during bootstrap.
+	if p.Standard && strings.HasPrefix(importerPath, "bootstrap/") {
+		return p
+	}
+
 	// The stack includes p.ImportPath.
 	// If that's the only thing on the stack, we started
 	// with a name given on the command line, not an
@@ -1184,6 +1387,26 @@ var cgoSyscallExclude = map[string]bool{
 
 var foldPath = make(map[string]string)
 
+// DefaultExecName returns the default executable name
+// for a package with the import path importPath.
+//
+// The default executable name is the last element of the import path.
+// In module-aware mode, an additional rule is used on import paths
+// consisting of two or more path elements. If the last element is
+// a vN path element specifying the major version, then the
+// second last element of the import path is used instead.
+func DefaultExecName(importPath string) string {
+	_, elem := pathpkg.Split(importPath)
+	if cfg.ModulesEnabled {
+		// If this is example.com/mycmd/v2, it's more useful to
+		// install it as mycmd than as v2. See golang.org/issue/24667.
+		if elem != importPath && isVersionElement(elem) {
+			_, elem = pathpkg.Split(pathpkg.Dir(importPath))
+		}
+	}
+	return elem
+}
+
 // load populates p using information from bp, err, which should
 // be the result of calling build.Context.Import.
 func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
@@ -1224,38 +1447,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 			p.Error = &PackageError{Err: e}
 			return
 		}
-		_, elem := filepath.Split(p.Dir)
-		if cfg.ModulesEnabled {
-			// NOTE(rsc): Using p.ImportPath instead of p.Dir
-			// makes sure we install a package in the root of a
-			// cached module directory as that package name
-			// not name@v1.2.3.
-			// Using p.ImportPath instead of p.Dir
-			// is probably correct all the time,
-			// even for non-module-enabled code,
-			// but I'm not brave enough to change the
-			// non-module behavior this late in the
-			// release cycle. Maybe for Go 1.12.
-			// See golang.org/issue/26869.
-			_, elem = pathpkg.Split(p.ImportPath)
-
-			// If this is example.com/mycmd/v2, it's more useful to install it as mycmd than as v2.
-			// See golang.org/issue/24667.
-			isVersion := func(v string) bool {
-				if len(v) < 2 || v[0] != 'v' || v[1] < '1' || '9' < v[1] {
-					return false
-				}
-				for i := 2; i < len(v); i++ {
-					if c := v[i]; c < '0' || '9' < c {
-						return false
-					}
-				}
-				return true
-			}
-			if isVersion(elem) {
-				_, elem = pathpkg.Split(pathpkg.Dir(p.ImportPath))
-			}
-		}
+		elem := DefaultExecName(p.ImportPath)
 		full := cfg.BuildContext.GOOS + "_" + cfg.BuildContext.GOARCH + "/" + elem
 		if cfg.BuildContext.GOOS != base.ToolGOOS || cfg.BuildContext.GOARCH != base.ToolGOARCH {
 			// Install cross-compiled binaries to subdirectories of bin.
@@ -1407,16 +1599,6 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 			continue
 		}
 		p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
-		if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil {
-			p.Error = &PackageError{
-				ImportStack: stk.Copy(),
-				Err:         fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath),
-			}
-			pos := p.Internal.Build.ImportPos[path]
-			if len(pos) > 0 {
-				p.Error.Pos = pos[0].String()
-			}
-		}
 
 		path = p1.ImportPath
 		importPaths[i] = path
@@ -1430,41 +1612,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 		}
 	}
 	p.Internal.Imports = imports
-
-	deps := make(map[string]*Package)
-	var q []*Package
-	q = append(q, imports...)
-	for i := 0; i < len(q); i++ {
-		p1 := q[i]
-		path := p1.ImportPath
-		// The same import path could produce an error or not,
-		// depending on what tries to import it.
-		// Prefer to record entries with errors, so we can report them.
-		p0 := deps[path]
-		if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) {
-			deps[path] = p1
-			for _, p2 := range p1.Internal.Imports {
-				if deps[p2.ImportPath] != p2 {
-					q = append(q, p2)
-				}
-			}
-		}
-	}
-
-	p.Deps = make([]string, 0, len(deps))
-	for dep := range deps {
-		p.Deps = append(p.Deps, dep)
-	}
-	sort.Strings(p.Deps)
-	for _, dep := range p.Deps {
-		p1 := deps[dep]
-		if p1 == nil {
-			panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath)
-		}
-		if p1.Error != nil {
-			p.DepsErrors = append(p.DepsErrors, p1.Error)
-		}
-	}
+	p.collectDeps()
 
 	// unsafe is a fake package.
 	if p.Standard && (p.ImportPath == "unsafe" || cfg.BuildContext.Compiler == "gccgo") {
@@ -1534,6 +1682,48 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 	}
 }
 
+// collectDeps populates p.Deps and p.DepsErrors by iterating over
+// p.Internal.Imports.
+//
+// TODO(jayconrod): collectDeps iterates over transitive imports for every
+// package. We should only need to visit direct imports.
+func (p *Package) collectDeps() {
+	deps := make(map[string]*Package)
+	var q []*Package
+	q = append(q, p.Internal.Imports...)
+	for i := 0; i < len(q); i++ {
+		p1 := q[i]
+		path := p1.ImportPath
+		// The same import path could produce an error or not,
+		// depending on what tries to import it.
+		// Prefer to record entries with errors, so we can report them.
+		p0 := deps[path]
+		if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) {
+			deps[path] = p1
+			for _, p2 := range p1.Internal.Imports {
+				if deps[p2.ImportPath] != p2 {
+					q = append(q, p2)
+				}
+			}
+		}
+	}
+
+	p.Deps = make([]string, 0, len(deps))
+	for dep := range deps {
+		p.Deps = append(p.Deps, dep)
+	}
+	sort.Strings(p.Deps)
+	for _, dep := range p.Deps {
+		p1 := deps[dep]
+		if p1 == nil {
+			panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath)
+		}
+		if p1.Error != nil {
+			p.DepsErrors = append(p.DepsErrors, p1.Error)
+		}
+	}
+}
+
 // SafeArg reports whether arg is a "safe" command-line argument,
 // meaning that when it appears in a command-line, it probably
 // doesn't have some special meaning other than its own name.
@@ -1720,99 +1910,17 @@ func TestPackageList(roots []*Package) []*Package {
 	return all
 }
 
-var cmdCache = map[string]*Package{}
-
-func ClearCmdCache() {
-	for name := range cmdCache {
-		delete(cmdCache, name)
-	}
-}
-
-// LoadPackage loads the package named by arg.
-func LoadPackage(arg string, stk *ImportStack) *Package {
-	p := loadPackage(arg, stk)
+// LoadImportWithFlags loads the package with the given import path and
+// sets tool flags on that package. This function is useful loading implicit
+// dependencies (like sync/atomic for coverage).
+// TODO(jayconrod): delete this function and set flags automatically
+// in LoadImport instead.
+func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+	p := LoadImport(path, srcDir, parent, stk, importPos, mode)
 	setToolFlags(p)
 	return p
 }
 
-// LoadPackageNoFlags is like LoadPackage
-// but does not guarantee that the build tool flags are set in the result.
-// It is only for use by GOPATH-based "go get"
-// and is only appropriate for preliminary loading of packages.
-// A real load using LoadPackage or (more likely)
-// Packages, PackageAndErrors, or PackagesForBuild
-// must be done before passing the package to any build
-// steps, so that the tool flags can be set properly.
-// TODO(rsc): When GOPATH-based "go get" is removed, delete this function.
-func LoadPackageNoFlags(arg string, stk *ImportStack) *Package {
-	return loadPackage(arg, stk)
-}
-
-// loadPackage is like loadImport but is used for command-line arguments,
-// not for paths found in import statements. In addition to ordinary import paths,
-// loadPackage accepts pseudo-paths beginning with cmd/ to denote commands
-// in the Go command directory, as well as paths to those directories.
-func loadPackage(arg string, stk *ImportStack) *Package {
-	if arg == "" {
-		panic("loadPackage called with empty package path")
-	}
-	if build.IsLocalImport(arg) {
-		dir := arg
-		if !filepath.IsAbs(dir) {
-			if abs, err := filepath.Abs(dir); err == nil {
-				// interpret relative to current directory
-				dir = abs
-			}
-		}
-		if sub, ok := hasSubdir(cfg.GOROOTsrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") {
-			arg = sub
-		}
-	}
-	if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") {
-		if p := cmdCache[arg]; p != nil {
-			return p
-		}
-		stk.Push(arg)
-		defer stk.Pop()
-
-		bp, err := cfg.BuildContext.ImportDir(filepath.Join(cfg.GOROOTsrc, arg), 0)
-		bp.ImportPath = arg
-		bp.Goroot = true
-		bp.BinDir = cfg.GOROOTbin
-		bp.Root = cfg.GOROOT
-		bp.SrcRoot = cfg.GOROOTsrc
-		p := new(Package)
-		cmdCache[arg] = p
-		p.load(stk, bp, err)
-		if p.Error == nil && p.Name != "main" {
-			p.Error = &PackageError{
-				ImportStack: stk.Copy(),
-				Err:         fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir),
-			}
-		}
-		return p
-	}
-
-	// Wasn't a command; must be a package.
-	// If it is a local import path but names a standard package,
-	// we treat it as if the user specified the standard package.
-	// This lets you run go test ./ioutil in package io and be
-	// referring to io/ioutil rather than a hypothetical import of
-	// "./ioutil".
-	if build.IsLocalImport(arg) || filepath.IsAbs(arg) {
-		dir := arg
-		if !filepath.IsAbs(arg) {
-			dir = filepath.Join(base.Cwd, arg)
-		}
-		bp, _ := cfg.BuildContext.ImportDir(dir, build.FindOnly)
-		if bp.ImportPath != "" && bp.ImportPath != "." {
-			arg = bp.ImportPath
-		}
-	}
-
-	return LoadImport(arg, base.Cwd, nil, stk, nil, 0)
-}
-
 // Packages returns the packages named by the
 // command line arguments 'args'. If a named package
 // cannot be loaded at all (for example, if the directory does not exist),
@@ -1838,8 +1946,12 @@ func Packages(args []string) []*Package {
 // cannot be loaded at all.
 // The packages that fail to load will have p.Error != nil.
 func PackagesAndErrors(patterns []string) []*Package {
-	if len(patterns) > 0 && strings.HasSuffix(patterns[0], ".go") {
-		return []*Package{GoFilesPackage(patterns)}
+	if len(patterns) > 0 {
+		for _, p := range patterns {
+			if strings.HasSuffix(p, ".go") {
+				return []*Package{GoFilesPackage(patterns)}
+			}
+		}
 	}
 
 	matches := ImportPaths(patterns)
@@ -1849,12 +1961,16 @@ func PackagesAndErrors(patterns []string) []*Package {
 		seenPkg = make(map[*Package]bool)
 	)
 
+	pre := newPreload()
+	defer pre.flush()
+	pre.preloadMatches(matches)
+
 	for _, m := range matches {
 		for _, pkg := range m.Pkgs {
 			if pkg == "" {
 				panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern))
 			}
-			p := loadPackage(pkg, &stk)
+			p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0)
 			p.Match = append(p.Match, m.Pattern)
 			p.Internal.CmdlinePkg = true
 			if m.Literal {
@@ -1947,7 +2063,14 @@ func GoFilesPackage(gofiles []string) *Package {
 
 	for _, f := range gofiles {
 		if !strings.HasSuffix(f, ".go") {
-			base.Fatalf("named files must be .go files")
+			pkg := new(Package)
+			pkg.Internal.Local = true
+			pkg.Internal.CmdlineFiles = true
+			pkg.Name = f
+			pkg.Error = &PackageError{
+				Err: fmt.Sprintf("named files must be .go files: %s", pkg.Name),
+			}
+			return pkg
 		}
 	}