diff options
Diffstat (limited to 'src/cmd/go/internal')
51 files changed, 2612 insertions, 1353 deletions
diff --git a/src/cmd/go/internal/base/flag.go b/src/cmd/go/internal/base/flag.go index 6727196816..c97c744520 100644 --- a/src/cmd/go/internal/base/flag.go +++ b/src/cmd/go/internal/base/flag.go @@ -28,13 +28,42 @@ func (v *StringsFlag) String() string { return "<StringsFlag>" } +// explicitStringFlag is like a regular string flag, but it also tracks whether +// the string was set explicitly to a non-empty value. +type explicitStringFlag struct { + value *string + explicit *bool +} + +func (f explicitStringFlag) String() string { + if f.value == nil { + return "" + } + return *f.value +} + +func (f explicitStringFlag) Set(v string) error { + *f.value = v + if v != "" { + *f.explicit = true + } + return nil +} + // AddBuildFlagsNX adds the -n and -x build flags to the flag set. func AddBuildFlagsNX(flags *flag.FlagSet) { flags.BoolVar(&cfg.BuildN, "n", false, "") flags.BoolVar(&cfg.BuildX, "x", false, "") } -// AddLoadFlags adds the -mod build flag to the flag set. -func AddLoadFlags(flags *flag.FlagSet) { - flags.StringVar(&cfg.BuildMod, "mod", "", "") +// AddModFlag adds the -mod build flag to the flag set. +func AddModFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") +} + +// AddModCommonFlags adds the module-related flags common to build commands +// and 'go mod' subcommands. +func AddModCommonFlags(flags *flag.FlagSet) { + flags.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") + flags.StringVar(&cfg.ModFile, "modfile", "", "") } diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index f9bbcd9180..9bf1db73ef 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -27,7 +27,8 @@ var ( BuildBuildmode string // -buildmode flag BuildContext = defaultContext() BuildMod string // -mod flag - BuildModReason string // reason -mod flag is set, if set by default + BuildModExplicit bool // whether -mod was set explicitly + BuildModReason string // reason -mod was set, if set by default BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag @@ -48,6 +49,8 @@ var ( ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag + Insecure bool // -insecure flag + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index f96cff429c..b0c1c59b40 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -23,7 +23,8 @@ import ( func init() { base.AddBuildFlagsNX(&CmdFmt.Flag) - base.AddLoadFlags(&CmdFmt.Flag) + base.AddModFlag(&CmdFmt.Flag) + base.AddModCommonFlags(&CmdFmt.Flag) } var CmdFmt = &base.Command{ diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index e5bacadaa3..3f7a66384a 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -18,8 +18,11 @@ import ( "cmd/go/internal/load" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/vcs" "cmd/go/internal/web" "cmd/go/internal/work" + + "golang.org/x/mod/module" ) var CmdGet = &base.Command{ @@ -41,7 +44,10 @@ The -fix flag instructs get to run the fix tool on the downloaded packages before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. +custom domains using insecure schemes such as HTTP. Use with caution. The +GOINSECURE environment variable is usually a better alternative, since it +provides control over which modules may be retrieved using an insecure scheme. +See 'go help environment' for details. The -t flag instructs get to also download the packages required to build the tests for the specified packages. @@ -103,14 +109,12 @@ var ( getT = CmdGet.Flag.Bool("t", false, "") getU = CmdGet.Flag.Bool("u", false, "") getFix = CmdGet.Flag.Bool("fix", false, "") - - Insecure bool ) func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") } func runGet(ctx context.Context, cmd *base.Command, args []string) { @@ -403,17 +407,12 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // to make the first copy of or update a copy of the given package. func downloadPackage(p *load.Package) error { var ( - vcs *vcsCmd + vcsCmd *vcs.Cmd repo, rootPath string err error blindRepo bool // set if the repo has unusual configuration ) - security := web.SecureOnly - if Insecure { - security = web.Insecure - } - // p can be either a real package, or a pseudo-package whose “import path” is // actually a wildcard pattern. // Trim the path at the element containing the first wildcard, @@ -427,22 +426,26 @@ func downloadPackage(p *load.Package) error { } importPrefix = importPrefix[:slash] } - if err := CheckImportPath(importPrefix); err != nil { + if err := module.CheckImportPath(importPrefix); err != nil { return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err) } + security := web.SecureOnly + if cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) { + security = web.Insecure + } if p.Internal.Build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. - vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot) + vcsCmd, rootPath, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot) if err != nil { return err } repo = "<local>" // should be unused; make distinctive // Double-check where it came from. - if *getU && vcs.remoteRepo != nil { + if *getU && vcsCmd.RemoteRepo != nil { dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - remote, err := vcs.remoteRepo(vcs, dir) + remote, err := vcsCmd.RemoteRepo(vcsCmd, dir) if err != nil { // Proceed anyway. The package is present; we likely just don't understand // the repo configuration (e.g. unusual remote protocol). @@ -450,10 +453,10 @@ func downloadPackage(p *load.Package) error { } repo = remote if !*getF && err == nil { - if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil { + if rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security); err == nil { repo := rr.Repo - if rr.vcs.resolveRepo != nil { - resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) + if rr.VCS.ResolveRepo != nil { + resolved, err := rr.VCS.ResolveRepo(rr.VCS, dir, repo) if err == nil { repo = resolved } @@ -467,13 +470,13 @@ func downloadPackage(p *load.Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security) if err != nil { return err } - vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root + vcsCmd, repo, rootPath = rr.VCS, rr.Repo, rr.Root } - if !blindRepo && !vcs.isSecure(repo) && !Insecure { + if !blindRepo && !vcsCmd.IsSecure(repo) && security != web.Insecure { return fmt.Errorf("cannot download, %v uses insecure protocol", repo) } @@ -496,7 +499,7 @@ func downloadPackage(p *load.Package) error { } root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil { + if err := vcs.CheckNested(vcsCmd, root, p.Internal.Build.SrcRoot); err != nil { return err } @@ -512,7 +515,7 @@ func downloadPackage(p *load.Package) error { // Check that this is an appropriate place for the repo to be checked out. // The target directory must either not exist or have a repo checked out already. - meta := filepath.Join(root, "."+vcs.cmd) + meta := filepath.Join(root, "."+vcsCmd.Cmd) if _, err := os.Stat(meta); err != nil { // Metadata file or directory does not exist. Prepare to checkout new copy. // Some version control tools require the target directory not to exist. @@ -533,12 +536,12 @@ func downloadPackage(p *load.Package) error { fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root) } - if err = vcs.create(root, repo); err != nil { + if err = vcsCmd.Create(root, repo); err != nil { return err } } else { // Metadata directory does exist; download incremental updates. - if err = vcs.download(root); err != nil { + if err = vcsCmd.Download(root); err != nil { return err } } @@ -547,12 +550,12 @@ func downloadPackage(p *load.Package) error { // Do not show tag sync in -n; it's noise more than anything, // and since we're not running commands, no tag will be found. // But avoid printing nothing. - fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) + fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcsCmd.Cmd) return nil } // Select and sync to appropriate version of the repository. - tags, err := vcs.tags(root) + tags, err := vcsCmd.Tags(root) if err != nil { return err } @@ -560,7 +563,7 @@ func downloadPackage(p *load.Package) error { if i := strings.Index(vers, " "); i >= 0 { vers = vers[:i] } - if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { + if err := vcsCmd.TagSync(root, selectTag(vers, tags)); err != nil { return err } diff --git a/src/cmd/go/internal/get/path.go b/src/cmd/go/internal/get/path.go deleted file mode 100644 index ce2e0cdd70..0000000000 --- a/src/cmd/go/internal/get/path.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package get - -import ( - "fmt" - "strings" - "unicode" - "unicode/utf8" -) - -// The following functions are copied verbatim from golang.org/x/mod/module/module.go, -// with a change to additionally reject Windows short-names, -// and one to accept arbitrary letters (golang.org/issue/29101). -// -// TODO(bcmills): After the call site for this function is backported, -// consolidate this back down to a single copy. -// -// NOTE: DO NOT MERGE THESE UNTIL WE DECIDE ABOUT ARBITRARY LETTERS IN MODULE MODE. - -// CheckImportPath checks that an import path is valid. -func CheckImportPath(path string) error { - if err := checkPath(path, false); err != nil { - return fmt.Errorf("malformed import path %q: %v", path, err) - } - return nil -} - -// checkPath checks that a general path is valid. -// It returns an error describing why but not mentioning path. -// Because these checks apply to both module paths and import paths, -// the caller is expected to add the "malformed ___ path %q: " prefix. -// fileName indicates whether the final element of the path is a file name -// (as opposed to a directory name). -func checkPath(path string, fileName bool) error { - if !utf8.ValidString(path) { - return fmt.Errorf("invalid UTF-8") - } - if path == "" { - return fmt.Errorf("empty string") - } - if path[0] == '-' { - return fmt.Errorf("leading dash") - } - if strings.Contains(path, "//") { - return fmt.Errorf("double slash") - } - if path[len(path)-1] == '/' { - return fmt.Errorf("trailing slash") - } - elemStart := 0 - for i, r := range path { - if r == '/' { - if err := checkElem(path[elemStart:i], fileName); err != nil { - return err - } - elemStart = i + 1 - } - } - if err := checkElem(path[elemStart:], fileName); err != nil { - return err - } - return nil -} - -// checkElem checks whether an individual path element is valid. -// fileName indicates whether the element is a file name (not a directory name). -func checkElem(elem string, fileName bool) error { - if elem == "" { - return fmt.Errorf("empty path element") - } - if strings.Count(elem, ".") == len(elem) { - return fmt.Errorf("invalid path element %q", elem) - } - if elem[0] == '.' && !fileName { - return fmt.Errorf("leading dot in path element") - } - if elem[len(elem)-1] == '.' { - return fmt.Errorf("trailing dot in path element") - } - - charOK := pathOK - if fileName { - charOK = fileNameOK - } - for _, r := range elem { - if !charOK(r) { - return fmt.Errorf("invalid char %q", r) - } - } - - // Windows disallows a bunch of path elements, sadly. - // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file - short := elem - if i := strings.Index(short, "."); i >= 0 { - short = short[:i] - } - for _, bad := range badWindowsNames { - if strings.EqualFold(bad, short) { - return fmt.Errorf("disallowed path element %q", elem) - } - } - - // Reject path components that look like Windows short-names. - // Those usually end in a tilde followed by one or more ASCII digits. - if tilde := strings.LastIndexByte(short, '~'); tilde >= 0 && tilde < len(short)-1 { - suffix := short[tilde+1:] - suffixIsDigits := true - for _, r := range suffix { - if r < '0' || r > '9' { - suffixIsDigits = false - break - } - } - if suffixIsDigits { - return fmt.Errorf("trailing tilde and digits in path element") - } - } - - return nil -} - -// pathOK reports whether r can appear in an import path element. -// -// NOTE: This function DIVERGES from module mode pathOK by accepting Unicode letters. -func pathOK(r rune) bool { - if r < utf8.RuneSelf { - return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' || - '0' <= r && r <= '9' || - 'A' <= r && r <= 'Z' || - 'a' <= r && r <= 'z' - } - return unicode.IsLetter(r) -} - -// fileNameOK reports whether r can appear in a file name. -// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters. -// If we expand the set of allowed characters here, we have to -// work harder at detecting potential case-folding and normalization collisions. -// See note about "safe encoding" below. -func fileNameOK(r rune) bool { - if r < utf8.RuneSelf { - // Entire set of ASCII punctuation, from which we remove characters: - // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~ - // We disallow some shell special characters: " ' * < > ? ` | - // (Note that some of those are disallowed by the Windows file system as well.) - // We also disallow path separators / : and \ (fileNameOK is only called on path element characters). - // We allow spaces (U+0020) in file names. - const allowed = "!#$%&()+,-.=@[]^_{}~ " - if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' { - return true - } - for i := 0; i < len(allowed); i++ { - if rune(allowed[i]) == r { - return true - } - } - return false - } - // It may be OK to add more ASCII punctuation here, but only carefully. - // For example Windows disallows < > \, and macOS disallows :, so we must not allow those. - return unicode.IsLetter(r) -} - -// badWindowsNames are the reserved file path elements on Windows. -// See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file -var badWindowsNames = []string{ - "CON", - "PRN", - "AUX", - "NUL", - "COM1", - "COM2", - "COM3", - "COM4", - "COM5", - "COM6", - "COM7", - "COM8", - "COM9", - "LPT1", - "LPT2", - "LPT3", - "LPT4", - "LPT5", - "LPT6", - "LPT7", - "LPT8", - "LPT9", -} diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index e68c39f392..23500dd9d8 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "os" "sort" @@ -215,6 +216,7 @@ applied to a Go struct, but now a Module struct: Dir string // directory holding files for this module, if any GoMod string // path to go.mod file used when loading this module, if any GoVersion string // go version used in module + Retracted string // retraction information, if any (with -retracted or -u) Error *ModuleError // error loading module } @@ -246,14 +248,16 @@ the replaced source code.) The -u flag adds information about available upgrades. When the latest version of a given module is newer than the current one, list -u sets the Module's Update field -to information about the newer module. +to information about the newer module. list -u will also set +the module's Retracted field if the current version is retracted. The Module's String method indicates an available upgrade by formatting the newer version in brackets after the current version. +If a version is retracted, the string "(retracted)" will follow it. For example, 'go list -m -u all' might print: my/main/module golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text - rsc.io/pdf v0.1.1 [v0.1.2] + rsc.io/pdf v0.1.1 (retracted) [v0.1.2] (For tools, 'go list -m -u -json all' may be more convenient to parse.) @@ -263,6 +267,14 @@ to semantic versioning, earliest to latest. The flag also changes the default output format to display the module path followed by the space-separated version list. +The -retracted flag causes list to report information about retracted +module versions. When -retracted is used with -f or -json, the Retracted +field will be set to a string explaining why the version was retracted. +The string is taken from comments on the retract directive in the +module's go.mod file. When -retracted is used with -versions, retracted +versions are listed together with unretracted versions. The -retracted +flag may be used with or without -m. + The arguments to list -m are interpreted as a list of modules, not packages. The main module is the module containing the current directory. The active modules are the main module and its dependencies. @@ -296,17 +308,18 @@ func init() { } var ( - listCompiled = CmdList.Flag.Bool("compiled", false, "") - listDeps = CmdList.Flag.Bool("deps", false, "") - listE = CmdList.Flag.Bool("e", false, "") - listExport = CmdList.Flag.Bool("export", false, "") - listFmt = CmdList.Flag.String("f", "", "") - listFind = CmdList.Flag.Bool("find", false, "") - listJson = CmdList.Flag.Bool("json", false, "") - listM = CmdList.Flag.Bool("m", false, "") - listU = CmdList.Flag.Bool("u", false, "") - listTest = CmdList.Flag.Bool("test", false, "") - listVersions = CmdList.Flag.Bool("versions", false, "") + listCompiled = CmdList.Flag.Bool("compiled", false, "") + listDeps = CmdList.Flag.Bool("deps", false, "") + listE = CmdList.Flag.Bool("e", false, "") + listExport = CmdList.Flag.Bool("export", false, "") + listFmt = CmdList.Flag.String("f", "", "") + listFind = CmdList.Flag.Bool("find", false, "") + listJson = CmdList.Flag.Bool("json", false, "") + listM = CmdList.Flag.Bool("m", false, "") + listRetracted = CmdList.Flag.Bool("retracted", false, "") + listTest = CmdList.Flag.Bool("test", false, "") + listU = CmdList.Flag.Bool("u", false, "") + listVersions = CmdList.Flag.Bool("versions", false, "") ) var nl = []byte{'\n'} @@ -367,6 +380,16 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } + modload.Init() + if *listRetracted { + if cfg.BuildMod == "vendor" { + base.Fatalf("go list -retracted cannot be used when vendoring is enabled") + } + if !modload.Enabled() { + base.Fatalf("go list -retracted can only be used in module-aware mode") + } + } + if *listM { // Module mode. if *listCompiled { @@ -414,9 +437,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } - modload.LoadBuildList(ctx) - - mods := modload.ListModules(ctx, args, *listU, *listVersions) + mods := modload.ListModules(ctx, args, *listU, *listVersions, *listRetracted) if !*listE { for _, m := range mods { if m.Error != nil { @@ -522,7 +543,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { // Note that -deps is applied after -test, // so that you only get descriptions of tests for the things named // explicitly on the command line, not for all dependencies. - pkgs = load.PackageList(pkgs) + pkgs = loadPackageList(pkgs) } // Do we need to run a build to gather information? @@ -557,7 +578,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { if *listTest { all := pkgs if !*listDeps { - all = load.PackageList(pkgs) + all = loadPackageList(pkgs) } // Update import paths to distinguish the real package p // from p recompiled for q.test. @@ -607,6 +628,55 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } + // TODO(golang.org/issue/40676): This mechanism could be extended to support + // -u without -m. + if *listRetracted { + // Load retractions for modules that provide packages that will be printed. + // TODO(golang.org/issue/40775): Packages from the same module refer to + // distinct ModulePublic instance. It would be nice if they could all point + // to the same instance. This would require additional global state in + // modload.loaded, so that should be refactored first. For now, we update + // all instances. + modToArg := make(map[*modinfo.ModulePublic]string) + argToMods := make(map[string][]*modinfo.ModulePublic) + var args []string + addModule := func(mod *modinfo.ModulePublic) { + if mod.Version == "" { + return + } + arg := fmt.Sprintf("%s@%s", mod.Path, mod.Version) + if argToMods[arg] == nil { + args = append(args, arg) + } + argToMods[arg] = append(argToMods[arg], mod) + modToArg[mod] = arg + } + for _, p := range pkgs { + if p.Module == nil { + continue + } + addModule(p.Module) + if p.Module.Replace != nil { + addModule(p.Module.Replace) + } + } + + if len(args) > 0 { + listU := false + listVersions := false + rmods := modload.ListModules(ctx, args, listU, listVersions, *listRetracted) + for i, arg := range args { + rmod := rmods[i] + for _, mod := range argToMods[arg] { + mod.Retracted = rmod.Retracted + if rmod.Error != nil && mod.Error == nil { + mod.Error = rmod.Error + } + } + } + } + } + // Record non-identity import mappings in p.ImportMap. for _, p := range pkgs { for i, srcPath := range p.Internal.RawImports { @@ -625,6 +695,23 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } +// loadPackageList is like load.PackageList, but prints error messages and exits +// with nonzero status if listE is not set and any package in the expanded list +// has errors. +func loadPackageList(roots []*load.Package) []*load.Package { + pkgs := load.PackageList(roots) + + if !*listE { + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("%v", pkg.Error) + } + } + } + + return pkgs +} + // TrackingWriter tracks the last byte written on every write so // we can avoid printing a newline if one was already written or // if there is no output at all. diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index a0e275095b..e0f13323df 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -191,6 +191,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p GoFiles: p.XTestGoFiles, Imports: p.XTestImports, ForTest: p.ImportPath, + Module: p.Module, Error: pxtestErr, }, Internal: PackageInternal{ @@ -222,6 +223,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p ImportPath: p.ImportPath + ".test", Root: p.Root, Imports: str.StringList(TestMainDeps), + Module: p.Module, }, Internal: PackageInternal{ Build: &build.Package{Name: "main"}, diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index d4c161fca1..6227fd9f33 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -12,9 +12,8 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modload" "cmd/go/internal/modfetch" - "cmd/go/internal/work" + "cmd/go/internal/modload" "golang.org/x/mod/module" ) @@ -64,7 +63,7 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") - work.AddModCommonFlags(cmdDownload) + base.AddModCommonFlags(&cmdDownload.Flag) } type moduleJSON struct { @@ -136,9 +135,10 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { var mods []*moduleJSON listU := false listVersions := false + listRetractions := false type token struct{} sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + for _, info := range modload.ListModules(ctx, args, listU, listVersions, listRetractions) { if info.Replace != nil { info = info.Replace } @@ -187,4 +187,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } base.ExitIfErrors() } + + // Update go.mod and especially go.sum if needed. + modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go index a81c25270f..03a774b824 100644 --- a/src/cmd/go/internal/modcmd/edit.go +++ b/src/cmd/go/internal/modcmd/edit.go @@ -19,7 +19,6 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -68,9 +67,14 @@ The -dropreplace=old[@v] flag drops a replacement of the given module path and version pair. If the @v is omitted, a replacement without a version on the left side is dropped. +The -retract=version and -dropretract=version flags add and drop a +retraction on the given version. The version may be a single version +like "v1.2.3" or a closed interval like "[v1.1.0-v1.1.9]". Note that +-retract=version is a no-op if that retraction already exists. + The -require, -droprequire, -exclude, -dropexclude, -replace, -and -dropreplace editing flags may be repeated, and the changes -are applied in the order given. +-dropreplace, -retract, and -dropretract editing flags may be repeated, +and the changes are applied in the order given. The -go=version flag sets the expected Go language version. @@ -104,6 +108,15 @@ writing it back to go.mod. The JSON output corresponds to these Go types: New Module } + type Retract struct { + Low string + High string + Rationale string + } + +Retract entries representing a single version (not an interval) will have +the "Low" and "High" fields set to the same value. + Note that this only describes the go.mod file itself, not other modules referred to indirectly. For the full set of modules available to a build, use 'go list -m -json all'. @@ -137,8 +150,10 @@ func init() { cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "") + cmdEdit.Flag.Var(flagFunc(flagRetract), "retract", "") + cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "") - work.AddModCommonFlags(cmdEdit) + base.AddModCommonFlags(&cmdEdit.Flag) base.AddBuildFlagsNX(&cmdEdit.Flag) } @@ -252,12 +267,7 @@ func parsePathVersion(flag, arg string) (path, version string) { base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err) } - // We don't call modfile.CheckPathVersion, because that insists - // on versions being in semver form, but here we want to allow - // versions like "master" or "1234abcdef", which the go command will resolve - // the next time it runs (or during -fix). - // Even so, we need to make sure the version is a valid token. - if modfile.MustQuote(version) { + if !allowedVersionArg(version) { base.Fatalf("go mod: -%s=%s: invalid version %q", flag, arg, version) } @@ -289,12 +299,48 @@ func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version return path, version, fmt.Errorf("invalid %s path: %v", adj, err) } } - if path != arg && modfile.MustQuote(version) { + if path != arg && !allowedVersionArg(version) { return path, version, fmt.Errorf("invalid %s version: %q", adj, version) } return path, version, nil } +// parseVersionInterval parses a single version like "v1.2.3" or a closed +// interval like "[v1.2.3,v1.4.5]". Note that a single version has the same +// representation as an interval with equal upper and lower bounds: both +// Low and High are set. +func parseVersionInterval(arg string) (modfile.VersionInterval, error) { + if !strings.HasPrefix(arg, "[") { + if !allowedVersionArg(arg) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version: %q", arg) + } + return modfile.VersionInterval{Low: arg, High: arg}, nil + } + if !strings.HasSuffix(arg, "]") { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + s := arg[1 : len(arg)-1] + i := strings.Index(s, ",") + if i < 0 { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + low := strings.TrimSpace(s[:i]) + high := strings.TrimSpace(s[i+1:]) + if !allowedVersionArg(low) || !allowedVersionArg(high) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + return modfile.VersionInterval{Low: low, High: high}, nil +} + +// allowedVersionArg returns whether a token may be used as a version in go.mod. +// We don't call modfile.CheckPathVersion, because that insists on versions +// being in semver form, but here we want to allow versions like "master" or +// "1234abcdef", which the go command will resolve the next time it runs (or +// during -fix). Even so, we need to make sure the version is a valid token. +func allowedVersionArg(arg string) bool { + return !modfile.MustQuote(arg) +} + // flagRequire implements the -require flag. func flagRequire(arg string) { path, version := parsePathVersion("require", arg) @@ -377,6 +423,32 @@ func flagDropReplace(arg string) { }) } +// flagRetract implements the -retract flag. +func flagRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.AddRetract(vi, ""); err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + }) +} + +// flagDropRetract implements the -dropretract flag. +func flagDropRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.DropRetract(vi); err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + }) +} + // fileJSON is the -json output data structure. type fileJSON struct { Module module.Version @@ -384,6 +456,7 @@ type fileJSON struct { Require []requireJSON Exclude []module.Version Replace []replaceJSON + Retract []retractJSON } type requireJSON struct { @@ -397,6 +470,12 @@ type replaceJSON struct { New module.Version } +type retractJSON struct { + Low string `json:",omitempty"` + High string `json:",omitempty"` + Rationale string `json:",omitempty"` +} + // editPrintJSON prints the -json output. func editPrintJSON(modFile *modfile.File) { var f fileJSON @@ -415,6 +494,9 @@ func editPrintJSON(modFile *modfile.File) { for _, r := range modFile.Replace { f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) } + for _, r := range modFile.Retract { + f.Retract = append(f.Retract, retractJSON{r.Low, r.High, r.Rationale}) + } data, err := json.MarshalIndent(&f, "", "\t") if err != nil { base.Fatalf("go: internal error: %v", err) diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 6da12b9cab..a149b65605 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -33,7 +32,7 @@ path@version, except for the main module, which has no @version suffix. } func init() { - work.AddModCommonFlags(cmdGraph) + base.AddModCommonFlags(&cmdGraph.Flag) } func runGraph(ctx context.Context, cmd *base.Command, args []string) { @@ -48,7 +47,7 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go: cannot find main module; see 'go help modules'") } } - modload.LoadBuildList(ctx) + modload.LoadAllModules(ctx) reqs := modload.MinReqs() format := func(m module.Version) string { diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index b6cffd332d..21b235653e 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -9,7 +9,6 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/modload" - "cmd/go/internal/work" "context" "os" "strings" @@ -30,7 +29,7 @@ To override this guess, supply the module path as an argument. } func init() { - work.AddModCommonFlags(cmdInit) + base.AddModCommonFlags(&cmdInit.Flag) } func runInit(ctx context.Context, cmd *base.Command, args []string) { diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index c7c53d7c0c..30df674ef6 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -10,7 +10,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/work" "context" ) @@ -32,7 +31,7 @@ to standard error. func init() { cmdTidy.Run = runTidy // break init cycle cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdTidy) + base.AddModCommonFlags(&cmdTidy.Flag) } func runTidy(ctx context.Context, cmd *base.Command, args []string) { @@ -40,6 +39,18 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } + // Tidy aims to make 'go test' reproducible for any package in 'all', so we + // need to include test dependencies. For modules that specify go 1.15 or + // earlier this is a no-op (because 'all' saturates transitive test + // dependencies). + // + // However, with lazy loading (go 1.16+) 'all' includes only the packages that + // are transitively imported by the main module, not the test dependencies of + // those packages. In order to make 'go test' reproducible for the packages + // that are in 'all' but outside of the main module, we must explicitly + // request that their test dependencies be included. + modload.LoadTests = true + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index e5353b5c7f..91d2509452 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -19,7 +19,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -41,7 +40,7 @@ modules and packages to standard error. func init() { cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdVendor) + base.AddModCommonFlags(&cmdVendor.Flag) } func runVendor(ctx context.Context, cmd *base.Command, args []string) { @@ -77,7 +76,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } var buf bytes.Buffer - for _, m := range modload.BuildList()[1:] { + for _, m := range modload.LoadedModules()[1:] { if pkgs := modpkgs[m]; len(pkgs) > 0 || isExplicit[m] { line := moduleLine(m, modload.Replacement(m)) buf.WriteString(line) diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 73ab714d10..7700588bde 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -17,7 +17,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -38,7 +37,7 @@ non-zero status. } func init() { - work.AddModCommonFlags(cmdVerify) + base.AddModCommonFlags(&cmdVerify.Flag) } func runVerify(ctx context.Context, cmd *base.Command, args []string) { @@ -60,7 +59,7 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) { sem := make(chan token, runtime.GOMAXPROCS(0)) // Use a slice of result channels, so that the output is deterministic. - mods := modload.LoadBuildList(ctx)[1:] + mods := modload.LoadAllModules(ctx)[1:] errsChans := make([]<-chan []error, len(mods)) for i, mod := range mods { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index da33fff89e..8454fdfec6 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -11,7 +11,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -58,23 +57,26 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle - work.AddModCommonFlags(cmdWhy) + base.AddModCommonFlags(&cmdWhy.Flag) } func runWhy(ctx context.Context, cmd *base.Command, args []string) { loadALL := modload.LoadALL if *whyVendor { loadALL = modload.LoadVendor + } else { + modload.LoadTests = true } if *whyM { listU := false listVersions := false + listRetractions := false for _, arg := range args { if strings.Contains(arg, "@") { base.Fatalf("go mod why: module query not allowed") } } - mods := modload.ListModules(ctx, args, listU, listVersions) + mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) byModule := make(map[module.Version][]string) for _, path := range loadALL(ctx) { m := modload.PackageModule(path) diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e29eb0a942..01d8f007ac 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -503,6 +503,9 @@ func checkGoMod(path, version string, data []byte) error { } // checkModSum checks that the recorded checksum for mod is h. +// +// mod.Version may have the additional suffix "/go.mod" to request the checksum +// for the module's go.mod file only. func checkModSum(mod module.Version, h string) error { // We lock goSum when manipulating it, // but we arrange to release the lock when calling checkSumDB, @@ -579,9 +582,16 @@ func addModSumLocked(mod module.Version, h string) { // checkSumDB checks the mod, h pair against the Go checksum database. // It calls base.Fatalf if the hash is to be rejected. func checkSumDB(mod module.Version, h string) error { + modWithoutSuffix := mod + noun := "module" + if strings.HasSuffix(mod.Version, "/go.mod") { + noun = "go.mod" + modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod") + } + db, lines, err := lookupSumDB(mod) if err != nil { - return module.VersionError(mod, fmt.Errorf("verifying module: %v", err)) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err)) } have := mod.Path + " " + mod.Version + " " + h @@ -591,7 +601,7 @@ func checkSumDB(mod module.Version, h string) error { return nil } if strings.HasPrefix(line, prefix) { - return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):])) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):])) } } return nil diff --git a/src/cmd/go/internal/modfetch/insecure.go b/src/cmd/go/internal/modfetch/insecure.go index b692669cba..012d05f29d 100644 --- a/src/cmd/go/internal/modfetch/insecure.go +++ b/src/cmd/go/internal/modfetch/insecure.go @@ -6,12 +6,11 @@ package modfetch import ( "cmd/go/internal/cfg" - "cmd/go/internal/get" "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) + return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 34f805d58a..eed4dd4258 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -13,9 +13,9 @@ import ( "time" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" + "cmd/go/internal/vcs" web "cmd/go/internal/web" "golang.org/x/mod/module" @@ -261,13 +261,13 @@ func lookupDirect(path string) (Repo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. return nil, notExistError{err: err} } - if rr.VCS == "mod" { + if rr.VCS.Name == "mod" { // Fetch module from proxy with base URL rr.Repo. return newProxyRepo(rr.Repo, path) } @@ -279,8 +279,8 @@ func lookupDirect(path string) (Repo, error) { return newCodeRepo(code, rr.Root, path) } -func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { - code, err := codehost.NewRepo(rr.VCS, rr.Repo) +func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { + code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) if err != nil { if _, ok := err.(*codehost.VCSError); ok { return nil, err @@ -306,7 +306,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security) if err != nil { return nil, nil, err } diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 783c4a433b..47a2571531 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -22,7 +22,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/lockedfile" "cmd/go/internal/web" @@ -33,7 +32,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index ee9757912b..829cfe055a 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -17,7 +17,7 @@ import ( "sync" "cmd/go/internal/base" - "cmd/go/internal/get" + "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/load" "cmd/go/internal/modload" @@ -181,7 +181,7 @@ var ( getM = CmdGet.Flag.Bool("m", false, "") getT = CmdGet.Flag.Bool("t", false, "") getU upgradeFlag - // -insecure is get.Insecure + // -insecure is cfg.Insecure // -v is cfg.BuildV ) @@ -206,7 +206,7 @@ func (v *upgradeFlag) String() string { return "" } func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&get.Insecure, "insecure", get.Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") CmdGet.Flag.Var(&getU, "u", "") } @@ -278,7 +278,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } modload.LoadTests = *getT - buildList := modload.LoadBuildList(ctx) + buildList := modload.LoadAllModules(ctx) buildList = buildList[:len(buildList):len(buildList)] // copy on append versionByPath := make(map[string]string) for _, m := range buildList { @@ -290,7 +290,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // what was requested. modload.DisallowWriteGoMod() - // Allow looking up modules for import paths outside of a module. + // Allow looking up modules for import paths when outside of a module. // 'go get' is expected to do this, unlike other commands. modload.AllowMissingModuleImports() @@ -599,7 +599,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { base.ExitIfErrors() // Stop if no changes have been made to the build list. - buildList = modload.BuildList() + buildList = modload.LoadedModules() eq := len(buildList) == len(prevBuildList) for i := 0; eq && i < len(buildList); i++ { eq = buildList[i] == prevBuildList[i] @@ -617,7 +617,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Handle downgrades. var down []module.Version - for _, m := range modload.BuildList() { + for _, m := range modload.LoadedModules() { q := byPath[m.Path] if q != nil && semver.Compare(m.Version, q.m.Version) > 0 { down = append(down, module.Version{Path: m.Path, Version: q.m.Version}) @@ -628,6 +628,10 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if err != nil { base.Fatalf("go: %v", err) } + + // TODO(bcmills) What should happen here under lazy loading? + // Downgrading may intentionally violate the lazy-loading invariants. + modload.SetBuildList(buildList) modload.ReloadBuildList() // note: does not update go.mod base.ExitIfErrors() @@ -637,7 +641,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { var lostUpgrades []*query if len(down) > 0 { versionByPath = make(map[string]string) - for _, m := range modload.BuildList() { + for _, m := range modload.LoadedModules() { versionByPath[m.Path] = m.Version } for _, q := range byPath { @@ -702,6 +706,15 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Everything succeeded. Update go.mod. modload.AllowWriteGoMod() modload.WriteGoMod() + modload.DisallowWriteGoMod() + + // Report warnings if any retracted versions are in the build list. + // This must be done after writing go.mod to avoid spurious '// indirect' + // comments. These functions read and write global state. + // TODO(golang.org/issue/40775): ListModules resets modload.loader, which + // contains information about direct dependencies that WriteGoMod uses. + // Refactor to avoid these kinds of global side effects. + reportRetractions(ctx) // If -d was specified, we're done after the module work. // We've already downloaded modules by loading packages above. @@ -804,6 +817,14 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc base.Fatalf("go get: internal error: prevM may be set if and only if forceModulePath is set") } + // If vers is a query like "latest", we should ignore retracted and excluded + // versions. If vers refers to a specific version or commit like "v1.0.0" + // or "master", we should only ignore excluded versions. + allowed := modload.CheckAllowed + if modload.IsRevisionQuery(vers) { + allowed = modload.CheckExclusions + } + // If the query must be a module path, try only that module path. if forceModulePath { if path == modload.Target.Path { @@ -812,7 +833,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc } } - info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +859,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -864,190 +885,41 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc return m, nil } -// An upgrader adapts an underlying mvs.Reqs to apply an -// upgrade policy to a list of targets and their dependencies. -type upgrader struct { - mvs.Reqs - - // cmdline maps a module path to a query made for that module at a - // specific target version. Each query corresponds to a module - // matched by a command line argument. - cmdline map[string]*query - - // upgrade is a set of modules providing dependencies of packages - // matched by command line arguments. If -u or -u=patch is set, - // these modules are upgraded accordingly. - upgrade map[string]bool -} - -// newUpgrader creates an upgrader. cmdline contains queries made at -// specific versions for modules matched by command line arguments. pkgs -// is the set of packages matched by command line arguments. If -u or -u=patch -// is set, modules providing dependencies of pkgs are upgraded accordingly. -func newUpgrader(cmdline map[string]*query, pkgs map[string]bool) *upgrader { - u := &upgrader{ - Reqs: modload.Reqs(), - cmdline: cmdline, - } - if getU != "" { - u.upgrade = make(map[string]bool) - - // Traverse package import graph. - // Initialize work queue with root packages. - seen := make(map[string]bool) - var work []string - add := func(path string) { - if !seen[path] { - seen[path] = true - work = append(work, path) - } - } - for pkg := range pkgs { - add(pkg) +// reportRetractions prints warnings if any modules in the build list are +// retracted. +func reportRetractions(ctx context.Context) { + // Query for retractions of modules in the build list. + // Use modload.ListModules, since that provides information in the same format + // as 'go list -m'. Don't query for "all", since that's not allowed outside a + // module. + buildList := modload.LoadedModules() + args := make([]string, 0, len(buildList)) + for _, m := range buildList { + if m.Version == "" { + // main module or dummy target module + continue } - for len(work) > 0 { - pkg := work[0] - work = work[1:] - m := modload.PackageModule(pkg) - u.upgrade[m.Path] = true - - // testImports is empty unless test imports were actually loaded, - // i.e., -t was set or "all" was one of the arguments. - imports, testImports := modload.PackageImports(pkg) - for _, imp := range imports { - add(imp) - } - for _, imp := range testImports { - add(imp) + args = append(args, m.Path+"@"+m.Version) + } + listU := false + listVersions := false + listRetractions := true + mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) + retractPath := "" + for _, mod := range mods { + if len(mod.Retracted) > 0 { + if retractPath == "" { + retractPath = mod.Path + } else { + retractPath = "<module>" } + rationale := modload.ShortRetractionRationale(mod.Retracted[0]) + logOncef("go: warning: %s@%s is retracted: %s", mod.Path, mod.Version, rationale) } } - return u -} - -// Required returns the requirement list for m. -// For the main module, we override requirements with the modules named -// one the command line, and we include new requirements. Otherwise, -// we defer to u.Reqs. -func (u *upgrader) Required(m module.Version) ([]module.Version, error) { - rs, err := u.Reqs.Required(m) - if err != nil { - return nil, err - } - if m != modload.Target { - return rs, nil - } - - overridden := make(map[string]bool) - for i, m := range rs { - if q := u.cmdline[m.Path]; q != nil && q.m.Version != "none" { - rs[i] = q.m - overridden[q.m.Path] = true - } - } - for _, q := range u.cmdline { - if !overridden[q.m.Path] && q.m.Path != modload.Target.Path && q.m.Version != "none" { - rs = append(rs, q.m) - } - } - return rs, nil -} - -// Upgrade returns the desired upgrade for m. -// -// If m was requested at a specific version on the command line, then -// Upgrade returns that version. -// -// If -u is set and m provides a dependency of a package matched by -// command line arguments, then Upgrade may provider a newer tagged version. -// If m is a tagged version, then Upgrade will return the latest tagged -// version (with the same minor version number if -u=patch). -// If m is a pseudo-version, then Upgrade returns the latest tagged version -// only if that version has a time-stamp newer than m. This special case -// prevents accidental downgrades when already using a pseudo-version -// newer than the latest tagged version. -// -// If none of the above cases apply, then Upgrade returns m. -func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { - // Allow pkg@vers on the command line to override the upgrade choice v. - // If q's version is < m.Version, then we're going to downgrade anyway, - // and it's cleaner to avoid moving back and forth and picking up - // extraneous other newer dependencies. - // If q's version is > m.Version, then we're going to upgrade past - // m.Version anyway, and again it's cleaner to avoid moving back and forth - // picking up extraneous other newer dependencies. - if q := u.cmdline[m.Path]; q != nil { - return q.m, nil - } - - if !u.upgrade[m.Path] { - // Not involved in upgrade. Leave alone. - return m, nil - } - - // Run query required by upgrade semantics. - // Note that Query "latest" is not the same as using repo.Latest, - // which may return a pseudoversion for the latest commit. - // Query "latest" returns the newest tagged version or the newest - // prerelease version if there are no non-prereleases, or repo.Latest - // if there aren't any tagged versions. - // If we're querying "upgrade" or "patch", Query will compare the current - // version against the chosen version and will return the current version - // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) - if err != nil { - // Report error but return m, to let version selection continue. - // (Reporting the error will fail the command at the next base.ExitIfErrors.) - - // Special case: if the error is for m.Version itself and m.Version has a - // replacement, then keep it and don't report the error: the fact that the - // version is invalid is likely the reason it was replaced to begin with. - var vErr *module.InvalidVersionError - if errors.As(err, &vErr) && vErr.Version == m.Version && modload.Replacement(m).Path != "" { - return m, nil - } - - // Special case: if the error is "no matching versions" then don't - // even report the error. Because Query does not consider pseudo-versions, - // it may happen that we have a pseudo-version but during -u=patch - // the query v0.0 matches no versions (not even the one we're using). - var noMatch *modload.NoMatchingVersionError - if !errors.As(err, &noMatch) { - base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) - } - return m, nil - } - - if info.Version != m.Version { - logOncef("go: %s %s => %s", m.Path, getU, info.Version) - } - return module.Version{Path: m.Path, Version: info.Version}, nil -} - -// buildListForLostUpgrade returns the build list for the module graph -// rooted at lost. Unlike mvs.BuildList, the target module (lost) is not -// treated specially. The returned build list may contain a newer version -// of lost. -// -// buildListForLostUpgrade is used after a downgrade has removed a module -// requested at a specific version. This helps us understand the requirements -// implied by each downgrade. -func buildListForLostUpgrade(lost module.Version, reqs mvs.Reqs) ([]module.Version, error) { - return mvs.BuildList(lostUpgradeRoot, &lostUpgradeReqs{Reqs: reqs, lost: lost}) -} - -var lostUpgradeRoot = module.Version{Path: "lost-upgrade-root", Version: ""} - -type lostUpgradeReqs struct { - mvs.Reqs - lost module.Version -} - -func (r *lostUpgradeReqs) Required(mod module.Version) ([]module.Version, error) { - if mod == lostUpgradeRoot { - return []module.Version{r.lost}, nil + if modload.HasModRoot() && retractPath != "" { + logOncef("go: run 'go get %s@latest' to switch to the latest unretracted version", retractPath) } - return r.Reqs.Required(mod) } var loggedLines sync.Map diff --git a/src/cmd/go/internal/modget/mvs.go b/src/cmd/go/internal/modget/mvs.go new file mode 100644 index 0000000000..19fffd2947 --- /dev/null +++ b/src/cmd/go/internal/modget/mvs.go @@ -0,0 +1,202 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modget + +import ( + "context" + "errors" + + "cmd/go/internal/base" + "cmd/go/internal/modload" + "cmd/go/internal/mvs" + + "golang.org/x/mod/module" +) + +// An upgrader adapts an underlying mvs.Reqs to apply an +// upgrade policy to a list of targets and their dependencies. +type upgrader struct { + mvs.Reqs + + // cmdline maps a module path to a query made for that module at a + // specific target version. Each query corresponds to a module + // matched by a command line argument. + cmdline map[string]*query + + // upgrade is a set of modules providing dependencies of packages + // matched by command line arguments. If -u or -u=patch is set, + // these modules are upgraded accordingly. + upgrade map[string]bool +} + +// newUpgrader creates an upgrader. cmdline contains queries made at +// specific versions for modules matched by command line arguments. pkgs +// is the set of packages matched by command line arguments. If -u or -u=patch +// is set, modules providing dependencies of pkgs are upgraded accordingly. +func newUpgrader(cmdline map[string]*query, pkgs map[string]bool) *upgrader { + u := &upgrader{ + Reqs: modload.Reqs(), + cmdline: cmdline, + } + if getU != "" { + u.upgrade = make(map[string]bool) + + // Traverse package import graph. + // Initialize work queue with root packages. + seen := make(map[string]bool) + var work []string + add := func(path string) { + if !seen[path] { + seen[path] = true + work = append(work, path) + } + } + for pkg := range pkgs { + add(pkg) + } + for len(work) > 0 { + pkg := work[0] + work = work[1:] + m := modload.PackageModule(pkg) + u.upgrade[m.Path] = true + + // testImports is empty unless test imports were actually loaded, + // i.e., -t was set or "all" was one of the arguments. + imports, testImports := modload.PackageImports(pkg) + for _, imp := range imports { + add(imp) + } + for _, imp := range testImports { + add(imp) + } + } + } + return u +} + +// Required returns the requirement list for m. +// For the main module, we override requirements with the modules named +// one the command line, and we include new requirements. Otherwise, +// we defer to u.Reqs. +func (u *upgrader) Required(m module.Version) ([]module.Version, error) { + rs, err := u.Reqs.Required(m) + if err != nil { + return nil, err + } + if m != modload.Target { + return rs, nil + } + + overridden := make(map[string]bool) + for i, m := range rs { + if q := u.cmdline[m.Path]; q != nil && q.m.Version != "none" { + rs[i] = q.m + overridden[q.m.Path] = true + } + } + for _, q := range u.cmdline { + if !overridden[q.m.Path] && q.m.Path != modload.Target.Path && q.m.Version != "none" { + rs = append(rs, q.m) + } + } + return rs, nil +} + +// Upgrade returns the desired upgrade for m. +// +// If m was requested at a specific version on the command line, then +// Upgrade returns that version. +// +// If -u is set and m provides a dependency of a package matched by +// command line arguments, then Upgrade may provider a newer tagged version. +// If m is a tagged version, then Upgrade will return the latest tagged +// version (with the same minor version number if -u=patch). +// If m is a pseudo-version, then Upgrade returns the latest tagged version +// only if that version has a time-stamp newer than m. This special case +// prevents accidental downgrades when already using a pseudo-version +// newer than the latest tagged version. +// +// If none of the above cases apply, then Upgrade returns m. +func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { + // Allow pkg@vers on the command line to override the upgrade choice v. + // If q's version is < m.Version, then we're going to downgrade anyway, + // and it's cleaner to avoid moving back and forth and picking up + // extraneous other newer dependencies. + // If q's version is > m.Version, then we're going to upgrade past + // m.Version anyway, and again it's cleaner to avoid moving back and forth + // picking up extraneous other newer dependencies. + if q := u.cmdline[m.Path]; q != nil { + return q.m, nil + } + + if !u.upgrade[m.Path] { + // Not involved in upgrade. Leave alone. + return m, nil + } + + // Run query required by upgrade semantics. + // Note that Query "latest" is not the same as using repo.Latest, + // which may return a pseudoversion for the latest commit. + // Query "latest" returns the newest tagged version or the newest + // prerelease version if there are no non-prereleases, or repo.Latest + // if there aren't any tagged versions. + // If we're querying "upgrade" or "patch", Query will compare the current + // version against the chosen version and will return the current version + // if it is newer. + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.CheckAllowed) + if err != nil { + // Report error but return m, to let version selection continue. + // (Reporting the error will fail the command at the next base.ExitIfErrors.) + + // Special case: if the error is for m.Version itself and m.Version has a + // replacement, then keep it and don't report the error: the fact that the + // version is invalid is likely the reason it was replaced to begin with. + var vErr *module.InvalidVersionError + if errors.As(err, &vErr) && vErr.Version == m.Version && modload.Replacement(m).Path != "" { + return m, nil + } + + // Special case: if the error is "no matching versions" then don't + // even report the error. Because Query does not consider pseudo-versions, + // it may happen that we have a pseudo-version but during -u=patch + // the query v0.0 matches no versions (not even the one we're using). + var noMatch *modload.NoMatchingVersionError + if !errors.As(err, &noMatch) { + base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) + } + return m, nil + } + + if info.Version != m.Version { + logOncef("go: %s %s => %s", m.Path, getU, info.Version) + } + return module.Version{Path: m.Path, Version: info.Version}, nil +} + +// buildListForLostUpgrade returns the build list for the module graph +// rooted at lost. Unlike mvs.BuildList, the target module (lost) is not +// treated specially. The returned build list may contain a newer version +// of lost. +// +// buildListForLostUpgrade is used after a downgrade has removed a module +// requested at a specific version. This helps us understand the requirements +// implied by each downgrade. +func buildListForLostUpgrade(lost module.Version, reqs mvs.Reqs) ([]module.Version, error) { + return mvs.BuildList(lostUpgradeRoot, &lostUpgradeReqs{Reqs: reqs, lost: lost}) +} + +var lostUpgradeRoot = module.Version{Path: "lost-upgrade-root", Version: ""} + +type lostUpgradeReqs struct { + mvs.Reqs + lost module.Version +} + +func (r *lostUpgradeReqs) Required(mod module.Version) ([]module.Version, error) { + if mod == lostUpgradeRoot { + return []module.Version{r.lost}, nil + } + return r.Reqs.Required(mod) +} diff --git a/src/cmd/go/internal/modinfo/info.go b/src/cmd/go/internal/modinfo/info.go index 07248d1a61..897be56397 100644 --- a/src/cmd/go/internal/modinfo/info.go +++ b/src/cmd/go/internal/modinfo/info.go @@ -21,6 +21,7 @@ type ModulePublic struct { Dir string `json:",omitempty"` // directory holding local copy of files, if any GoMod string `json:",omitempty"` // path to go.mod file describing module, if any GoVersion string `json:",omitempty"` // go version used in module + Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u) Error *ModuleError `json:",omitempty"` // error loading module } @@ -30,18 +31,26 @@ type ModuleError struct { func (m *ModulePublic) String() string { s := m.Path + versionString := func(mm *ModulePublic) string { + v := mm.Version + if len(mm.Retracted) == 0 { + return v + } + return v + " (retracted)" + } + if m.Version != "" { - s += " " + m.Version + s += " " + versionString(m) if m.Update != nil { - s += " [" + m.Update.Version + "]" + s += " [" + versionString(m.Update) + "]" } } if m.Replace != nil { s += " => " + m.Replace.Path if m.Replace.Version != "" { - s += " " + m.Replace.Version + s += " " + versionString(m.Replace) if m.Replace.Update != nil { - s += " [" + m.Replace.Update.Version + "]" + s += " [" + versionString(m.Replace.Update) + "]" } } } diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index a101681a1f..9ca6230500 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/hex" + "errors" "fmt" "internal/goroot" "os" @@ -58,7 +59,9 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(context.TODO(), m, true) + fromBuildList := true + listRetracted := false + return moduleInfo(context.TODO(), m, fromBuildList, listRetracted) } func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { @@ -66,13 +69,17 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { return nil } + listRetracted := false if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) + m := module.Version{Path: path[:i], Version: path[i+1:]} + fromBuildList := false + return moduleInfo(ctx, m, fromBuildList, listRetracted) } - for _, m := range BuildList() { + for _, m := range LoadedModules() { if m.Path == path { - return moduleInfo(ctx, m, true) + fromBuildList := true + return moduleInfo(ctx, m, fromBuildList, listRetracted) } } @@ -90,7 +97,7 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { return } - if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -100,11 +107,37 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { } // addVersions fills in m.Versions with the list of known versions. -func addVersions(m *modinfo.ModulePublic) { - m.Versions, _ = versions(m.Path) +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + m.Versions, _ = versions(ctx, m.Path, allowed) } -func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := checkRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var rerr *retractedError + if errors.As(err, &rerr) { + if len(rerr.rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = rerr.rationale + } + } else if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } +} + +func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -126,12 +159,14 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi Version: m.Version, Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], } - if loaded != nil { - info.GoVersion = loaded.goVersion[m.Path] + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) } // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { + mod := module.Version{Path: m.Path, Version: m.Version} + if m.Version != "" { if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} @@ -140,7 +175,6 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi m.Time = &q.Time } - mod := module.Version{Path: m.Path, Version: m.Version} gomod, err := modfetch.CachePath(mod, "mod") if err == nil { if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { @@ -151,10 +185,22 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi if err == nil { m.Dir = dir } + + if listRetracted { + addRetraction(ctx, m) + } + } + + if m.GoVersion == "" { + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" { + m.GoVersion = summary.goVersionV[1:] + } } } if !fromBuildList { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. completeFromModCache(info) // Will set m.Error in vendor mode. return info } @@ -178,9 +224,11 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi // worth the cost, and we're going to overwrite the GoMod and Dir from the // replacement anyway. See https://golang.org/issue/27859. info.Replace = &modinfo.ModulePublic{ - Path: r.Path, - Version: r.Version, - GoVersion: info.GoVersion, + Path: r.Path, + Version: r.Version, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) } if r.Version == "" { if filepath.IsAbs(r.Path) { @@ -194,7 +242,9 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi completeFromModCache(info.Replace) info.Dir = info.Replace.Dir info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted } + info.GoVersion = info.Replace.GoVersion return info } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go new file mode 100644 index 0000000000..581a1b944a --- /dev/null +++ b/src/cmd/go/internal/modload/buildlist.go @@ -0,0 +1,122 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/mvs" + "context" + "fmt" + "os" + + "golang.org/x/mod/module" +) + +// buildList is the list of modules to use for building packages. +// It is initialized by calling ImportPaths, ImportFromFiles, +// LoadALL, or LoadBuildList, each of which uses loaded.load. +// +// Ideally, exactly ONE of those functions would be called, +// and exactly once. Most of the time, that's true. +// During "go get" it may not be. TODO(rsc): Figure out if +// that restriction can be established, or else document why not. +// +var buildList []module.Version + +// LoadAllModules loads and returns the list of modules matching the "all" +// module pattern, starting with the Target module and in a deterministic +// (stable) order, without loading any packages. +// +// Modules are loaded automatically (and lazily) in ImportPaths: +// LoadAllModules need only be called if ImportPaths is not, +// typically in commands that care about modules but no particular package. +// +// The caller must not modify the returned list. +func LoadAllModules(ctx context.Context) []module.Version { + InitMod(ctx) + ReloadBuildList() + WriteGoMod() + return buildList +} + +// LoadedModules returns the list of module requirements loaded or set by a +// previous call (typically LoadAllModules or ImportPaths), starting with the +// Target module and in a deterministic (stable) order. +// +// The caller must not modify the returned list. +func LoadedModules() []module.Version { + return buildList +} + +// SetBuildList sets the module build list. +// The caller is responsible for ensuring that the list is valid. +// SetBuildList does not retain a reference to the original list. +func SetBuildList(list []module.Version) { + buildList = append([]module.Version{}, list...) +} + +// ReloadBuildList resets the state of loaded packages, then loads and returns +// the build list set in SetBuildList. +func ReloadBuildList() []module.Version { + loaded = loadFromRoots(loaderParams{ + tags: imports.Tags(), + listRoots: func() []string { return nil }, + allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. + }) + return buildList +} + +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding Load* or +// ImportPaths* call. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { + firstPath := make(map[module.Version]string, len(buildList)) + for _, mod := range buildList { + src := mod + if rep := Replacement(mod); rep.Path != "" { + src = rep + } + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + } + } + base.ExitIfErrors() +} diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index d80206b194..37f23d967f 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -432,15 +432,17 @@ verb followed by arguments. For example: require new/thing/v2 v2.3.4 exclude old/thing v1.2.3 replace bad/thing v1.4.5 => good/thing v1.4.5 + retract v1.5.6 The verbs are module, to define the module path; go, to set the expected language version; require, to require a particular module at a given version or later; - exclude, to exclude a particular module version from use; and - replace, to replace a module version with a different module version. + exclude, to exclude a particular module version from use; + replace, to replace a module version with a different module version; and + retract, to indicate a previously released version should not be used. Exclude and replace apply only in the main module's go.mod and are ignored -in dependencies. See https://research.swtch.com/vgo-mvs for details. +in dependencies. See https://golang.org/ref/mod for details. The leading verb can be factored out of adjacent lines to create a block, like in Go imports: diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 5c51a79124..10b1e7f4b8 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -26,6 +26,8 @@ import ( "golang.org/x/mod/semver" ) +var errImportMissing = errors.New("import missing") + type ImportMissingError struct { Path string Module module.Version @@ -48,6 +50,11 @@ func (e *ImportMissingError) Error() string { } return "cannot find module providing package " + e.Path } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) } @@ -100,18 +107,39 @@ func (e *AmbiguousImportError) Error() string { var _ load.ImportPathError = &AmbiguousImportError{} -// Import finds the module and directory in the build list -// containing the package with the given import path. -// The answer must be unique: Import returns an error -// if multiple modules attempt to provide the same package. -// Import can return a module with an empty m.Path, for packages in the standard library. -// Import can return an empty directory string, for fake packages like "C" and "unsafe". +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +var _ load.ImportPathError = &invalidImportError{} + +// importFromBuildList finds the module and directory in the build list +// containing the package with the given import path. The answer must be unique: +// importFromBuildList returns an error if multiple modules attempt to provide +// the same package. +// +// importFromBuildList can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromBuildList can return an empty directory string, for fake packages +// like "C" and "unsafe". // // If the package cannot be found in the current build list, -// Import returns an ImportMissingError as the error. -// If Import can identify a module that could be added to supply the package, -// the ImportMissingError records that module. -func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { +// importFromBuildList returns errImportMissing as the error. +func importFromBuildList(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -190,29 +218,25 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - // Look up module containing the package, for addition to the build list. - // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing. - if cfg.BuildMod == "readonly" { - var queryErr error - if !pathIsStd { - if cfg.BuildModReason == "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } - } - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr} - } + return module.Version{}, "", errImportMissing +} + +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +func queryImport(ctx context.Context, path string) (module.Version, error) { + pathIsStd := search.IsStandardImportPath(path) + if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, "", &ImportMissingError{ + return module.Version{}, &ImportMissingError{ Path: path, QueryErr: errors.New("working directory is not part of a module"), } } // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each module. - // (golang.org/issue/26241) + // To avoid spurious remote fetches, next try the latest replacement for each + // module (golang.org/issue/26241). This should give a useful message + // in -mod=readonly, and it will allow us to add a requirement with -mod=mod. if modFile != nil { latest := map[string]string{} // path -> version for _, r := range modFile.Replace { @@ -226,7 +250,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err } } - mods = make([]module.Version, 0, len(latest)) + mods := make([]module.Version, 0, len(latest)) for p, v := range latest { // If the replacement didn't specify a version, synthesize a // pseudo-version with an appropriate major version and a timestamp below @@ -252,19 +276,19 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. - return module.Version{}, "", err + return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, "", err + return m, err } else if ok { - return m, "", &ImportMissingError{Path: path, Module: m} + return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, // so it can only exist if in a replaced module, // and we know from the above loop that it is not. - return module.Version{}, "", &PackageNotInModuleError{ + return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], Query: "latest", Pattern: path, @@ -273,6 +297,11 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err } } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, &invalidImportError{importPath: path, err: err} + } + if pathIsStd { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has @@ -281,25 +310,39 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // QueryPackage cannot possibly find a module containing this package. // // Instead of trying QueryPackage, report an ImportMissingError immediately. - return module.Version{}, "", &ImportMissingError{Path: path} + return module.Version{}, &ImportMissingError{Path: path} + } + + if cfg.BuildMod == "readonly" { + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} } + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(ctx, path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", CheckAllowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever // low-level error QueryPackage produced. - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: err} + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} } else { - return module.Version{}, "", err + return module.Version{}, err } } - m = candidates[0].Mod - newMissingVersion := "" - for _, c := range candidates { + + candidate0MissingVersion := "" + for i, c := range candidates { cm := c.Mod + canAdd := true for _, bm := range buildList { if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { // QueryPackage proposed that we add module cm to provide the package, @@ -310,13 +353,22 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // version (e.g., v1.0.0) of a module, but we have a newer version // of the same module in the build list (e.g., v1.0.1-beta), and // the package is not present there. - m = cm - newMissingVersion = bm.Version + canAdd = false + if i == 0 { + candidate0MissingVersion = bm.Version + } break } } + if canAdd { + return cm, nil + } + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, } - return m, "", &ImportMissingError{Path: path, Module: m, newMissingVersion: newMissingVersion} } // maybeInModule reports whether, syntactically, diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index 47ce89a084..22d5b82e21 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -10,15 +10,20 @@ import ( "regexp" "strings" "testing" + + "golang.org/x/mod/module" ) var importTests = []struct { path string + m module.Version err string }{ { path: "golang.org/x/net/context", - err: "missing module for import: golang.org/x/net@.* provides golang.org/x/net/context", + m: module.Version{ + Path: "golang.org/x/net", + }, }, { path: "golang.org/x/net", @@ -26,15 +31,23 @@ var importTests = []struct { }, { path: "golang.org/x/text", - err: "missing module for import: golang.org/x/text@.* provides golang.org/x/text", + m: module.Version{ + Path: "golang.org/x/text", + }, }, { path: "github.com/rsc/quote/buggy", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote/buggy", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "github.com/rsc/quote", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "golang.org/x/foo/bar", @@ -42,7 +55,7 @@ var importTests = []struct { }, } -func TestImport(t *testing.T) { +func TestQueryImport(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") defer func(old bool) { @@ -55,12 +68,23 @@ func TestImport(t *testing.T) { for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(ctx, tt.path) - if err == nil { - t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) + m, err := queryImport(ctx, tt.path) + + if tt.err == "" { + if err != nil { + t.Fatalf("queryImport(_, %q): %v", tt.path, err) + } + } else { + if err == nil { + t.Fatalf("queryImport(_, %q) = %v, nil; expected error", tt.path, m) + } + if !regexp.MustCompile(tt.err).MatchString(err.Error()) { + t.Fatalf("queryImport(_, %q): error %q, want error matching %#q", tt.path, err, tt.err) + } } - if !regexp.MustCompile(tt.err).MatchString(err.Error()) { - t.Fatalf("Import(%q): error %q, want error matching %#q", tt.path, err, tt.err) + + if m.Path != tt.m.Path || (tt.m.Version != "" && m.Version != tt.m.Version) { + t.Errorf("queryImport(_, %q) = %v, _; want %v", tt.path, m, tt.m) } }) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 93027c44c4..1f50dcb11c 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -368,13 +368,9 @@ func InitMod(ctx context.Context) { modFile = f index = indexModFile(data, f, fixed) - if len(f.Syntax.Stmt) == 0 || f.Module == nil { - // Empty mod file. Must add module path. - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - f.AddModuleStmt(path) + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod.\n\tRun 'go mod edit -module=example.com/mod' to specify the module path.") } if len(f.Syntax.Stmt) == 1 && f.Module != nil { @@ -383,14 +379,61 @@ func InitMod(ctx context.Context) { legacyModInit() } - modFileToBuildList() + if err := checkModulePathLax(f.Module.Mod.Path); err != nil { + base.Fatalf("go: %v", err) + } + setDefaultBuildMod() + modFileToBuildList() if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() } } +// checkModulePathLax checks that the path meets some minimum requirements +// to avoid confusing users or the module cache. The requirements are weaker +// than those of module.CheckPath to allow room for weakening module path +// requirements in the future, but strong enough to help users avoid significant +// problems. +func checkModulePathLax(p string) error { + // TODO(matloob): Replace calls of this function in this CL with calls + // to module.CheckImportPath once it's been laxened, if it becomes laxened. + // See golang.org/issue/29101 for a discussion about whether to make CheckImportPath + // more lax or more strict. + + errorf := func(format string, args ...interface{}) error { + return fmt.Errorf("invalid module path %q: %s", p, fmt.Sprintf(format, args...)) + } + + // Disallow shell characters " ' * < > ? ` | to avoid triggering bugs + // with file systems and subcommands. Disallow file path separators : and \ + // because path separators other than / will confuse the module cache. + // See fileNameOK in golang.org/x/mod/module/module.go. + shellChars := "`" + `\"'*<>?|` + fsChars := `\:` + if i := strings.IndexAny(p, shellChars); i >= 0 { + return errorf("contains disallowed shell character %q", p[i]) + } + if i := strings.IndexAny(p, fsChars); i >= 0 { + return errorf("contains disallowed path separator character %q", p[i]) + } + + // Ensure path.IsAbs and build.IsLocalImport are false, and that the path is + // invariant under path.Clean, also to avoid confusing the module cache. + if path.IsAbs(p) { + return errorf("is an absolute path") + } + if build.IsLocalImport(p) { + return errorf("is a local import path") + } + if path.Clean(p) != p { + return errorf("is not clean") + } + + return nil +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, @@ -459,7 +502,15 @@ func modFileToBuildList() { list := []module.Version{Target} for _, r := range modFile.Require { - list = append(list, r.Mod) + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + } else { + list = append(list, r.Mod) + } } buildList = list } @@ -467,46 +518,49 @@ func modFileToBuildList() { // setDefaultBuildMod sets a default value for cfg.BuildMod // if it is currently empty. func setDefaultBuildMod() { - if cfg.BuildMod != "" { + if cfg.BuildModExplicit { // Don't override an explicit '-mod=' argument. return } - cfg.BuildMod = "mod" + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // Don't set -mod implicitly for commands whose purpose is to - // manipulate the build list. + // 'get' and 'go mod' commands may update go.mod automatically. + // TODO(jayconrod): should this narrower? Should 'go mod download' or + // 'go mod graph' update go.mod by default? + cfg.BuildMod = "mod" return } if modRoot == "" { + cfg.BuildMod = "mod" return } if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { modGo := "unspecified" - if index.goVersion != "" { - if semver.Compare("v"+index.goVersion, "v1.14") >= 0 { + if index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { // The Go version is at least 1.14, and a vendor directory exists. // Set -mod=vendor by default. cfg.BuildMod = "vendor" cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." return } else { - modGo = index.goVersion + modGo = index.goVersionV[1:] } } - // Since a vendor directory exists, we have a non-trivial reason for - // choosing -mod=mod, although it probably won't be used for anything. - // Record the reason anyway for consistency. - // It may be overridden if we switch to mod=readonly below. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) } p := ModFilePath() if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { cfg.BuildMod = "readonly" cfg.BuildModReason = "go.mod file is read-only." + return } + cfg.BuildMod = "mod" } func legacyModInit() { @@ -674,16 +728,35 @@ func findModulePath(dir string) (string, error) { } // Look for path in GOPATH. + var badPathErr error for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { if gpdir == "" { continue } if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { - return filepath.ToSlash(rel), nil + path := filepath.ToSlash(rel) + // TODO(matloob): replace this with module.CheckImportPath + // once it's been laxened. + // Only checkModulePathLax here. There are some unpublishable + // module names that are compatible with checkModulePathLax + // but they already work in GOPATH so don't break users + // trying to do a build with modules. gorelease will alert users + // publishing their modules to fix their paths. + if err := checkModulePathLax(path); err != nil { + badPathErr = err + break + } + return path, nil } } - msg := `cannot determine module path for source directory %s (outside GOPATH, module path must be specified) + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) Example usage: 'go mod init example.com/m' to initialize a v0 or v1 module @@ -691,7 +764,7 @@ Example usage: Run 'go help mod init' for more information. ` - return "", fmt.Errorf(msg, dir) + return "", fmt.Errorf(msg, dir, reason) } var ( @@ -787,19 +860,16 @@ func WriteGoMod() { // prefer to report a dirty go.mod over a dirty go.sum if cfg.BuildModReason != "" { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) - } else { + } else if cfg.BuildModExplicit { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") } } - // Always update go.sum, even if we didn't change go.mod: we may have - // downloaded modules that we didn't have before. - modfetch.WriteGoSum(keepSums()) - if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). - // Ignore any intervening edits. + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + modfetch.WriteGoSum(keepSums(true)) return } @@ -810,6 +880,9 @@ func WriteGoMod() { defer func() { // At this point we have determined to make the go.mod file on disk equal to new. index = indexModFile(new, modFile, false) + + // Update go.sum after releasing the side lock and refreshing the index. + modfetch.WriteGoSum(keepSums(true)) }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -850,7 +923,10 @@ func WriteGoMod() { // the last load function like ImportPaths, LoadALL, etc.). It also contains // entries for go.mod files needed for MVS (the version of these entries // ends with "/go.mod"). -func keepSums() map[module.Version]bool { +// +// If addDirect is true, the set also includes sums for modules directly +// required by go.mod, as represented by the index, with replacements applied. +func keepSums(addDirect bool) map[module.Version]bool { // Walk the module graph and keep sums needed by MVS. modkey := func(m module.Version) module.Version { return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} @@ -862,9 +938,6 @@ func keepSums() map[module.Version]bool { walk = func(m module.Version) { // If we build using a replacement module, keep the sum for the replacement, // since that's the code we'll actually use during a build. - // - // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the - // sums for both sets of transitive requirements. r := Replacement(m) if r.Path == "" { keep[modkey(m)] = true @@ -894,9 +967,27 @@ func keepSums() map[module.Version]bool { } } + // Add entries for modules directly required by go.mod. + if addDirect { + for m := range index.require { + var kept module.Version + if r := Replacement(m); r.Path != "" { + kept = r + } else { + kept = m + } + keep[kept] = true + keep[module.Version{Path: kept.Path, Version: kept.Version + "/go.mod"}] = true + } + } + return keep } func TrimGoSum() { - modfetch.TrimGoSum(keepSums()) + // Don't retain sums for direct requirements in go.mod. When TrimGoSum is + // called, go.mod has not been updated, and it may contain requirements on + // modules deleted from the build list. + addDirect := false + modfetch.TrimGoSum(keepSums(addDirect)) } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 7bf4e86c8d..3491f941cd 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -20,12 +20,12 @@ import ( "golang.org/x/mod/module" ) -func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(ctx, args, listVersions) +func ListModules(ctx context.Context, args []string, listU, listVersions, listRetracted bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions, listRetracted) type token struct{} sem := make(chan token, runtime.GOMAXPROCS(0)) - if listU || listVersions { + if listU || listVersions || listRetracted { for _, m := range mods { add := func(m *modinfo.ModulePublic) { sem <- token{} @@ -34,7 +34,10 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ addUpdate(ctx, m) } if listVersions { - addVersions(m) + addVersions(ctx, m, listRetracted) + } + if listRetracted || listU { + addRetraction(ctx, m) } <-sem }() @@ -54,10 +57,10 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ return mods } -func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList(ctx) +func listModules(ctx context.Context, args []string, listVersions, listRetracted bool) []*modinfo.ModulePublic { + LoadAllModules(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true, listRetracted)} } var mods []*modinfo.ModulePublic @@ -83,7 +86,13 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(ctx, path, vers, current, nil) + allowed := CheckAllowed + if IsRevisionQuery(vers) || listRetracted { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := Query(ctx, path, vers, current, allowed) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +101,8 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) continue } @@ -117,7 +127,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(ctx, m, true)) + mods = append(mods, moduleInfo(ctx, m, true, listRetracted)) } } } @@ -129,7 +139,8 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Instead, resolve the module, even if it isn't an existing dependency. info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 686d491219..1664d8c5be 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -4,6 +4,95 @@ package modload +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are several exported entry points into package loading (such as +// ImportPathsQuiet and LoadALL), but they are all implemented in terms of +// loadFromRoots, which itself manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern and related functions +// (LoadALL, LoadVendor), the final set of packages is computed from the package +// import graph, and therefore cannot be an initial input to loading that graph. +// Instead, the root packages for the "all" pattern are those contained in the +// main module, and allPatternIsRoot parameter to the loader instructs it to +// dynamically expand those roots to the full "all" pattern as loading +// progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When we implement lazy loading, we will record the modules providing packages +// in "all" even when we are only loading individual packages, so we set the +// pkgInAll flag regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The allClosesOverTests parameter to the loader indicates whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". That configuration does not arise in Go +// 1.11–1.15, but it will be possible with lazy loading in Go 1.16+. +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + import ( "bytes" "context" @@ -14,8 +103,12 @@ import ( "path" pathpkg "path" "path/filepath" + "reflect" + "runtime" "sort" "strings" + "sync" + "sync/atomic" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -25,28 +118,12 @@ import ( "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" - "cmd/go/internal/trace" "golang.org/x/mod/module" ) -// buildList is the list of modules to use for building packages. -// It is initialized by calling ImportPaths, ImportFromFiles, -// LoadALL, or LoadBuildList, each of which uses loaded.load. -// -// Ideally, exactly ONE of those functions would be called, -// and exactly once. Most of the time, that's true. -// During "go get" it may not be. TODO(rsc): Figure out if -// that restriction can be established, or else document why not. -// -var buildList []module.Version - // loaded is the most recently-used package loader. // It holds details about individual packages. -// -// Note that loaded.buildList is only valid during a load operation; -// afterward, it is copied back into the global buildList, -// which should be used instead. var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), @@ -63,7 +140,18 @@ func ImportPaths(ctx context.Context, patterns []string) []*search.Match { // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { - updateMatches := func(matches []*search.Match, iterating bool) { + InitMod(ctx) + + allPatternIsRoot := false + var matches []*search.Match + for _, pattern := range search.CleanPatterns(patterns) { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(ld *loader) { for _, m := range matches { switch { case m.IsLocal(): @@ -90,7 +178,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo // indicates that. ModRoot() - if !iterating { + if ld != nil { m.AddError(err) } continue @@ -103,19 +191,18 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, tags, includeStd, buildList) case m.Pattern() == "all": - loaded.testAll = true - if iterating { - // Enumerate the packages in the main module. - // We'll load the dependencies as we find them. + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". - m.Pkgs = loaded.computePatternAll(m.Pkgs) + m.Pkgs = ld.computePatternAll() } case m.Pattern() == "std" || m.Pattern() == "cmd": @@ -129,51 +216,28 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo } } - InitMod(ctx) - - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { - matches = append(matches, search.NewMatch(pattern)) - } + loaded = loadFromRoots(loaderParams{ + tags: tags, + allPatternIsRoot: allPatternIsRoot, + allClosesOverTests: index.allPatternClosesOverTests(), - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - updateMatches(matches, true) - for _, m := range matches { - roots = append(roots, m.Pkgs...) - } - return roots + listRoots: func() (roots []string) { + updateMatches(nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, }) // One last pass to finalize wildcards. - updateMatches(matches, false) + updateMatches(loaded) checkMultiplePaths() WriteGoMod() return matches } -// checkMultiplePaths verifies that a given module path is used as itself -// or as a replacement for another module, but not both at the same time. -// -// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) -func checkMultiplePaths() { - firstPath := make(map[module.Version]string, len(buildList)) - for _, mod := range buildList { - src := mod - if rep := Replacement(mod); rep.Path != "" { - src = rep - } - if prev, ok := firstPath[src]; !ok { - firstPath[src] = mod.Path - } else if prev != mod.Path { - base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) - } - } - base.ExitIfErrors() -} - // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories // outside of the standard library and active modules. func matchLocalDirs(m *search.Match) { @@ -310,7 +374,7 @@ var ( // pathInModuleCache returns the import path of the directory dir, // if dir is in the module cache copy of a module in our build list. func pathInModuleCache(dir string) string { - for _, m := range buildList[1:] { + tryMod := func(m module.Version) (string, bool) { var root string var err error if repl := Replacement(m); repl.Path != "" && repl.Version == "" { @@ -324,13 +388,26 @@ func pathInModuleCache(dir string) string { root, err = modfetch.DownloadDir(m) } if err != nil { - continue + return "", false } - if sub := search.InDir(dir, root); sub != "" { - sub = filepath.ToSlash(sub) - if !strings.Contains(sub, "/vendor/") && !strings.HasPrefix(sub, "vendor/") && !strings.Contains(sub, "@") { - return path.Join(m.Path, filepath.ToSlash(sub)) - } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + for _, m := range buildList[1:] { + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath } } return "" @@ -347,12 +424,14 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { base.Fatalf("go: %v", err) } - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - roots = append(roots, imports...) - roots = append(roots, testImports...) - return roots + loaded = loadFromRoots(loaderParams{ + tags: tags, + listRoots: func() (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, + allClosesOverTests: index.allPatternClosesOverTests(), }) WriteGoMod() } @@ -383,34 +462,19 @@ func DirImportPath(dir string) string { return "." } -// LoadBuildList loads and returns the build list from go.mod. -// The loading of the build list happens automatically in ImportPaths: -// LoadBuildList need only be called if ImportPaths is not -// (typically in commands that care about the module but -// no particular package). -func LoadBuildList(ctx context.Context) []module.Version { - ctx, span := trace.StartSpan(ctx, "LoadBuildList") - defer span.Done() - InitMod(ctx) - ReloadBuildList() - WriteGoMod() - return buildList -} - -func ReloadBuildList() []module.Version { - loaded = newLoader(imports.Tags()) - loaded.load(func() []string { return nil }) - return buildList -} - // LoadALL returns the set of all packages in the current module // and their dependencies in any other modules, without filtering // due to build tags, except "+build ignore". // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. +// +// In modules that specify "go 1.16" or higher, ALL follows only one layer of +// test dependencies. In "go 1.15" or lower, ALL follows the imports of tests of +// dependencies of tests. func LoadALL(ctx context.Context) []string { - return loadAll(ctx, true) + InitMod(ctx) + return loadAll(ctx, index.allPatternClosesOverTests()) } // LoadVendor is like LoadALL but only follows test dependencies @@ -418,20 +482,20 @@ func LoadALL(ctx context.Context) []string { // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. func LoadVendor(ctx context.Context) []string { - return loadAll(ctx, false) -} - -func loadAll(ctx context.Context, testAll bool) []string { InitMod(ctx) + // 'go mod vendor' has never followed test dependencies since Go 1.11. + const closeOverTests = false + return loadAll(ctx, closeOverTests) +} - loaded = newLoader(imports.AnyTags()) - loaded.isALL = true - loaded.testAll = testAll - if !testAll { - loaded.testRoots = true - } - all := TargetPackages(ctx, "...") - loaded.load(func() []string { return all.Pkgs }) +func loadAll(ctx context.Context, closeOverTests bool) []string { + inTarget := TargetPackages(ctx, "...") + loaded = loadFromRoots(loaderParams{ + tags: imports.AnyTags(), + listRoots: func() []string { return inTarget.Pkgs }, + allPatternIsRoot: true, + allClosesOverTests: closeOverTests, + }) checkMultiplePaths() WriteGoMod() @@ -443,7 +507,7 @@ func loadAll(ctx context.Context, testAll bool) []string { } paths = append(paths, pkg.path) } - for _, err := range all.Errs { + for _, err := range inTarget.Errs { base.Errorf("%v", err) } base.ExitIfErrors() @@ -463,52 +527,6 @@ func TargetPackages(ctx context.Context, pattern string) *search.Match { return m } -// BuildList returns the module build list, -// typically constructed by a previous call to -// LoadBuildList or ImportPaths. -// The caller must not modify the returned list. -func BuildList() []module.Version { - return buildList -} - -// SetBuildList sets the module build list. -// The caller is responsible for ensuring that the list is valid. -// SetBuildList does not retain a reference to the original list. -func SetBuildList(list []module.Version) { - buildList = append([]module.Version{}, list...) -} - -// TidyBuildList trims the build list to the minimal requirements needed to -// retain the same versions of all packages from the preceding Load* or -// ImportPaths* call. -func TidyBuildList() { - used := map[module.Version]bool{Target: true} - for _, pkg := range loaded.pkgs { - used[pkg.mod] = true - } - - keep := []module.Version{Target} - var direct []string - for _, m := range buildList[1:] { - if used[m] { - keep = append(keep, m) - if loaded.direct[m.Path] { - direct = append(direct, m.Path) - } - } else if cfg.BuildV { - if _, ok := index.require[m]; ok { - fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) - } - } - } - - min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) - if err != nil { - base.Fatalf("go: %v", err) - } - buildList = append([]module.Version{Target}, min...) -} - // ImportMap returns the actual package import path // for an import path found in source code. // If the given import path does not appear in the source code @@ -563,12 +581,6 @@ func PackageImports(path string) (imports, testImports []string) { return imports, testImports } -// ModuleUsedDirectly reports whether the main module directly imports -// some package in the module with the given path. -func ModuleUsedDirectly(path string) bool { - return loaded.direct[path] -} - // Lookup returns the source directory, import path, and any loading error for // the package at path as imported from the package in parentDir. // Lookup requires that one of the Load functions in this package has already @@ -604,76 +616,148 @@ func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath str // the required packages for a particular build, // checking that the packages are available in the module set, // and updating the module set if needed. -// Loading is an iterative process: try to load all the needed packages, -// but if imports are missing, try to resolve those imports, and repeat. -// -// Although most of the loading state is maintained in the loader struct, -// one key piece - the build list - is a global, so that it can be modified -// separate from the loading operation, such as during "go get" -// upgrades/downgrades or in "go mod" operations. -// TODO(rsc): It might be nice to make the loader take and return -// a buildList rather than hard-coding use of the global. type loader struct { - tags map[string]bool // tags for scanDir - testRoots bool // include tests for roots - isALL bool // created with LoadALL - testAll bool // include tests for all packages - forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree + loaderParams + + work *par.Queue // reset on each iteration roots []*loadPkg - pkgs []*loadPkg - work *par.Work // current work queue - pkgCache *par.Cache // map from string to *loadPkg + pkgCache *par.Cache // package path (string) → *loadPkg + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks // computed at end of iterations - direct map[string]bool // imported directly by main module - goVersion map[string]string // go version recorded in each module + direct map[string]bool // imported directly by main module +} + +type loaderParams struct { + tags map[string]bool // tags for scanDir + listRoots func() []string + allPatternIsRoot bool // Is the "all" pattern an additional root? + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? } // LoadTests controls whether the loaders load tests of the root packages. var LoadTests bool -func newLoader(tags map[string]bool) *loader { - ld := new(loader) - ld.tags = tags - ld.testRoots = LoadTests - - // Inside the "std" and "cmd" modules, we prefer to use the vendor directory - // unless the command explicitly changes the module graph. - if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) { - ld.forceStdVendor = true +func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") } - return ld -} - -func (ld *loader) reset() { ld.roots = nil - ld.pkgs = nil - ld.work = new(par.Work) ld.pkgCache = new(par.Cache) + ld.pkgs = nil } // A loadPkg records information about a single loaded package. type loadPkg struct { - path string // import path + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: mod module.Version // module providing package dir string // directory containing source code - imports []*loadPkg // packages imported by this one err error // error loading package - stack *loadPkg // package importing this one in minimal import stack for this pkg - test *loadPkg // package with test imports, if we need test - testOf *loadPkg - testImports []string // test-only imports, saved for use by pkg.test. + imports []*loadPkg // packages imported by this one + testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := atomic.LoadInt32(&af.bits) + new := old | int32(flags) + if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil } var errMissing = errors.New("cannot find package") -// load attempts to load the build graph needed to process a set of root packages. -// The set of root packages is defined by the addRoots function, -// which must call add(path) with the import path of each root package. -func (ld *loader) load(roots func() []string) { +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + var err error reqs := Reqs() buildList, err = mvs.BuildList(Target, reqs) @@ -681,47 +765,34 @@ func (ld *loader) load(roots func() []string) { base.Fatalf("go: %v", err) } - added := make(map[string]bool) + addedModuleFor := make(map[string]bool) for { ld.reset() - if roots != nil { - // Note: the returned roots can change on each iteration, - // since the expansion of package patterns depends on the - // build list we're using. - for _, path := range roots() { - ld.work.Add(ld.pkg(path, true)) + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + inRoots := map[*loadPkg]bool{} + for _, path := range ld.listRoots() { + root := ld.pkg(path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true } } - ld.work.Do(10, ld.doPkg) + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + ld.buildStacks() - numAdded := 0 - haveMod := make(map[module.Version]bool) - for _, m := range buildList { - haveMod[m] = true - } - modAddedBy := make(map[module.Version]*loadPkg) - for _, pkg := range ld.pkgs { - if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" { - if err.newMissingVersion != "" { - base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion) - } - fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, err.Module.Path, err.Module.Version) - if added[pkg.path] { - base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) - } - added[pkg.path] = true - numAdded++ - if !haveMod[err.Module] { - haveMod[err.Module] = true - modAddedBy[err.Module] = pkg - buildList = append(buildList, err.Module) - } - continue - } - // Leave other errors for Import or load.Packages to report. - } - base.ExitIfErrors() - if numAdded == 0 { + + modAddedBy := ld.resolveMissingImports(addedModuleFor) + if len(modAddedBy) == 0 { break } @@ -754,99 +825,264 @@ func (ld *loader) load(roots func() []string) { } } - // Add Go versions, computed during walk. - ld.goVersion = make(map[string]string) - for _, m := range buildList { - v, _ := reqs.(*mvsReqs).versions.Load(m) - ld.goVersion[m.Path], _ = v.(string) - } - - // Mix in direct markings (really, lack of indirect markings) - // from go.mod, unless we scanned the whole module - // and can therefore be sure we know better than go.mod. - if !ld.isALL && modFile != nil { + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing + // dependencies as indirect-only. + // Conservatively mark those dependencies as direct. + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true } } } + + return ld } -// pkg returns the *loadPkg for path, creating and queuing it if needed. -// If the package should be tested, its test is created but not queued -// (the test is queued after processing pkg). -// If isRoot is true, the pkg is being queued as one of the roots of the work graph. -func (ld *loader) pkg(path string, isRoot bool) *loadPkg { - return ld.pkgCache.Do(path, func() interface{} { - pkg := &loadPkg{ - path: path, +// resolveMissingImports adds module dependencies to the global build list +// in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each newly-added module version to +// the first package for which that module was added. +func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { + var needPkgs []*loadPkg + for _, pkg := range ld.pkgs { + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue } - if ld.testRoots && isRoot || ld.testAll { - test := &loadPkg{ - path: path, - testOf: pkg, - } - pkg.test = test + if pkg.err != errImportMissing { + // Leave other errors for Import or load.Packages to report. + continue + } + + needPkgs = append(needPkgs, pkg) + + pkg := pkg + ld.work.Add(func() { + pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) + }) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + for _, pkg := range needPkgs { + if pkg.err != nil { + continue } - if isRoot { - ld.roots = append(ld.roots, pkg) + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) + if addedModuleFor[pkg.path] { + // TODO(bcmills): This should only be an error if pkg.mod is the same + // version we already tried to add previously. + base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) + } + if modAddedBy[pkg.mod] == nil { + modAddedBy[pkg.mod] = pkg + buildList = append(buildList, pkg.mod) + } + } + + return modAddedBy +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := &loadPkg{ + path: path, } - ld.work.Add(pkg) + ld.applyPkgFlags(pkg, flags) + + ld.work.Add(func() { ld.load(pkg) }) return pkg }).(*loadPkg) + + ld.applyPkgFlags(pkg, flags) + return pkg } -// doPkg processes a package on the work queue. -func (ld *loader) doPkg(item interface{}) { - // TODO: what about replacements? - pkg := item.(*loadPkg) - var imports []string - if pkg.testOf != nil { - pkg.dir = pkg.testOf.dir - pkg.mod = pkg.testOf.mod - imports = pkg.testOf.testImports - } else { - if strings.Contains(pkg.path, "@") { - // Leave for error during load. - return - } - if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { - // Leave for error during load. - // (Module mode does not allow local imports.) - return - } +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } - // TODO(matloob): Handle TODO context. This needs to be threaded through Do. - pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) - if pkg.dir == "" { - return + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && pkg.mod == Target: + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true } - var testImports []string - var err error - imports, testImports, err = scanDir(pkg.dir, ld.tags) - if err != nil { - pkg.err = err - return + + if wantTest { + var testFlags loadPkgFlags + if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(pkg, testFlags) } - if pkg.test != nil { - pkg.testImports = testImports + } + + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(dep, pkgInAll) } } +} + +// load loads an individual package. +func (ld *loader) load(pkg *loadPkg) { + if strings.Contains(pkg.path, "@") { + // Leave for error during load. + return + } + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { + // Leave for error during load. + // (Module mode does not allow local imports.) + return + } - inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path) + if pkg.dir == "" { + return + } + if pkg.mod == Target { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(pkg, pkgInAll) + } + + imports, testImports, err := scanDir(pkg.dir, ld.tags) + if err != nil { + pkg.err = err + return + } + + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } for _, path := range imports { - if inStd { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. path = ld.stdVendor(pkg.path, path) } - pkg.imports = append(pkg.imports, ld.pkg(path, false)) + pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) } + pkg.testImports = testImports - // Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test. - // TODO: All that's left is creating new imports. Why not just do it now? - if pkg.test != nil { - ld.work.Add(pkg.test) + ld.applyPkgFlags(pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") + } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(test, testFlags) } + + return test } // stdVendor returns the canonical import path for the package with the given @@ -857,13 +1093,21 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if ld.forceStdVendor || Target.Path != "cmd" { + if Target.Path != "cmd" { vendorPath := pathpkg.Join("cmd", "vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if ld.forceStdVendor || Target.Path != "std" { + } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import internal + // packages from the real module. vendorPath := pathpkg.Join("vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath @@ -876,30 +1120,13 @@ func (ld *loader) stdVendor(parentPath, path string) string { // computePatternAll returns the list of packages matching pattern "all", // starting with a list of the import paths for the packages in the main module. -func (ld *loader) computePatternAll(paths []string) []string { - seen := make(map[*loadPkg]bool) - var all []string - var walk func(*loadPkg) - walk = func(pkg *loadPkg) { - if seen[pkg] { - return - } - seen[pkg] = true - if pkg.testOf == nil { +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { all = append(all, pkg.path) } - for _, p := range pkg.imports { - walk(p) - } - if p := pkg.test; p != nil { - walk(p) - } - } - for _, path := range paths { - walk(ld.pkg(path, false)) } sort.Strings(all) - return all } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 9f4ec5a49f..18dd293ac9 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -5,13 +5,31 @@ package modload import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "unicode" + "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/trace" "golang.org/x/mod/modfile" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) +// lazyLoadingVersion is the Go version (plus leading "v") at which lazy module +// loading takes effect. +const lazyLoadingVersionV = "v1.16" +const go116EnableLazyLoading = true + var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile @@ -20,7 +38,7 @@ type modFileIndex struct { data []byte dataNeedsFix bool // true if fixVersion applied a change while parsing data module module.Version - goVersion string + goVersionV string // GoVersion with "v" prefix require map[module.Version]requireMeta replace map[module.Version]module.Version exclude map[module.Version]bool @@ -33,9 +51,151 @@ type requireMeta struct { indirect bool } -// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. -func Allowed(m module.Version) bool { - return index == nil || !index.exclude[m] +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := checkRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + if index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// checkRetractions returns an error if module m has been retracted by +// its author. +func checkRetractions(ctx context.Context, m module.Version) error { + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + + // Look up retraction information from the latest available version of + // the module. Cache retraction information so we don't parse the go.mod + // file repeatedly. + type entry struct { + retract []retraction + err error + } + path := m.Path + e := retractCache.Do(path, func() (v interface{}) { + ctx, span := trace.StartSpan(ctx, "checkRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced with a local directory. + // Don't load retractions. + return &entry{nil, nil} + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + // We may need to account for the current version: for example, + // v2.0.0+incompatible is not "latest" if v1.0.0 is current. + rev, err := Query(ctx, path, "latest", findCurrentVersion(path), nil) + if err != nil { + return &entry{err: err} + } + + // Load go.mod for that version. + // If the version is replaced, we'll load retractions from the replacement. + // If there's an error loading the go.mod, we'll return it here. + // These errors should generally be ignored by callers of checkRetractions, + // since they happen frequently when we're offline. These errors are not + // equivalent to ErrDisallowed, so they may be distinguished from + // retraction errors. + summary, err := goModSummary(module.Version{Path: path, Version: rev.Version}) + if err != nil { + return &entry{err: err} + } + return &entry{retract: summary.retract} + }).(*entry) + + if e.err != nil { + return fmt.Errorf("loading module retractions: %v", e.err) + } + + var rationale []string + isRetracted := false + for _, r := range e.retract { + if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return &retractedError{rationale: rationale} + } + return nil +} + +var retractCache par.Cache + +type retractedError struct { + rationale []string +} + +func (e *retractedError) Error() string { + msg := "retracted by module author" + if len(e.rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortRetractionRationale(e.rationale[0]) + } + return msg +} + +func (e *retractedError) Is(err error) bool { + return err == ErrDisallowed +} + +// ShortRetractionRationale returns a retraction rationale string that is safe +// to print in a terminal. It returns hard-coded strings if the rationale +// is empty, too long, or contains non-printable characters. +func ShortRetractionRationale(rationale string) string { + const maxRationaleBytes = 500 + if i := strings.Index(rationale, "\n"); i >= 0 { + rationale = rationale[:i] + } + rationale = strings.TrimSpace(rationale) + if rationale == "" { + return "retracted by module author" + } + if len(rationale) > maxRationaleBytes { + return "(rationale omitted: too long)" + } + for _, r := range rationale { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(rationale omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return rationale } // Replacement returns the replacement for mod, if any, from go.mod. @@ -66,9 +226,11 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.module = modFile.Module.Mod } - i.goVersion = "" + i.goVersionV = "" if modFile.Go != nil { - i.goVersion = modFile.Go.Version + // We're going to use the semver package to compare Go versions, so go ahead + // and add the "v" prefix it expects once instead of every time. + i.goVersionV = "v" + modFile.Go.Version } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -92,6 +254,23 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd return i } +// allPatternClosesOverTests reports whether the "all" pattern includes +// dependencies of tests outside the main module (as in Go 1.11–1.15). +// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages +// transitively *imported by* the packages and tests in the main module.) +func (i *modFileIndex) allPatternClosesOverTests() bool { + if !go116EnableLazyLoading { + return true + } + if i != nil && semver.Compare(i.goVersionV, lazyLoadingVersionV) < 0 { + // The module explicitly predates the change in "all" for lazy loading, so + // continue to use the older interpretation. (If i == nil, we not in any + // module at all and should use the latest semantics.) + return true + } + return false +} + // modFileIsDirty reports whether the go.mod file differs meaningfully // from what was indexed. // If modFile has been changed (even cosmetically) since it was first read, @@ -114,11 +293,11 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { } if modFile.Go == nil { - if i.goVersion != "" { + if i.goVersionV != "" { return true } - } else if modFile.Go.Version != i.goVersion { - if i.goVersion == "" && cfg.BuildMod == "readonly" { + } else if "v"+modFile.Go.Version != i.goVersionV { + if i.goVersionV == "" && cfg.BuildMod == "readonly" { // go.mod files did not always require a 'go' version, so do not error out // if one is missing — we may be inside an older module in the module // cache, and should bias toward providing useful behavior. @@ -162,3 +341,190 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { return false } + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersionV string // GoVersion with "v" prefix + require []module.Version + retract []retraction +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// goModSummary cannot be used on the Target module, as its requirements +// may change. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: goModSummary called on the Target module") + } + + type cached struct { + summary *modFileSummary + err error + } + c := goModSummaryCache.Do(m, func() interface{} { + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return cached{summary, nil} + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. + + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return cached{summary, nil} + } + + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + summary, err := rawGoModSummary(actual) + if err != nil { + return cached{nil, err} + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return cached{nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line"))} + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return cached{nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, m.Path))} + } + } + + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + nonExcluded := summary.require[:0] + for _, r := range summary.require { + if !index.exclude[r] { + nonExcluded = append(nonExcluded, r) + } + } + summary.require = nonExcluded + } + return cached{summary, nil} + }).(cached) + + return c.summary, c.err +} + +var goModSummaryCache par.Cache // module.Version → goModSummary result + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the Target module. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: rawGoModSummary called on the Target module") + } + + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + + data, err := lockedfile.Read(gomod) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err)) + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err)) + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return nil, err + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err)) + } + } + + if f.Module != nil { + summary.module = f.Module.Mod + } + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return summary, nil +} diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 67eb2c2e19..24856260d4 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -11,16 +11,10 @@ import ( "os" "path/filepath" "sort" - "sync" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" "cmd/go/internal/mvs" - "cmd/go/internal/par" - "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -29,8 +23,6 @@ import ( // with any exclusions or replacements applied internally. type mvsReqs struct { buildList []module.Version - cache par.Cache - versions sync.Map } // Reqs returns the current module requirement graph. @@ -44,118 +36,21 @@ func Reqs() mvs.Reqs { } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - type cached struct { - list []module.Version - err error - } - - c := r.cache.Do(mod, func() interface{} { - list, err := r.required(mod) - if err != nil { - return cached{nil, err} - } - for i, mv := range list { - if index != nil { - for index.exclude[mv] { - mv1, err := r.next(mv) - if err != nil { - return cached{nil, err} - } - if mv1.Version == "none" { - return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} - } - mv = mv1 - } - } - list[i] = mv - } - - return cached{list, nil} - }).(cached) - - return c.list, c.err -} - -func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { - list := make([]module.Version, 0, len(f.Require)) - for _, r := range f.Require { - list = append(list, r.Mod) - } - return list -} - -// required returns a unique copy of the requirements of mod. -func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { if mod == Target { - if modFile != nil && modFile.Go != nil { - r.versions.LoadOrStore(mod, modFile.Go.Version) - } - return append([]module.Version(nil), r.buildList[1:]...), nil - } - - if cfg.BuildMod == "vendor" { - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() - return append([]module.Version(nil), vendorList...), nil - } - - origPath := mod.Path - if repl := Replacement(mod); repl.Path != "" { - if repl.Version == "" { - // TODO: need to slip the new version into the tags list etc. - dir := repl.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") - data, err := lockedfile.Read(gomod) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - f, err := modfile.ParseLax(gomod, data, nil) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - return r.modFileToList(f), nil - } - mod = repl + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.buildList[1:], nil } if mod.Version == "none" { return nil, nil } - if !semver.IsValid(mod.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) - } - - data, err := modfetch.GoMod(mod.Path, mod.Version) + summary, err := goModSummary(mod) if err != nil { return nil, err } - f, err := modfile.ParseLax("go.mod", data, nil) - if err != nil { - return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err)) - } - - if f.Module == nil { - return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line")) - } - if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { - return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod: - module declares its path as: %s - but was required as: %s`, mpath, origPath)) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - - return r.modFileToList(f), nil + return summary.require, nil } // Max returns the maximum of v1 and v2 according to semver.Compare. @@ -177,16 +72,29 @@ func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { return m, nil } -func versions(path string) ([]string, error) { +func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) { // Note: modfetch.Lookup and repo.Versions are cached, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { repo, err := modfetch.Lookup(proxy, path) - if err == nil { - versions, err = repo.Versions("") + if err != nil { + return err + } + allVersions, err := repo.Versions("") + if err != nil { + return err + } + allowedVersions := make([]string, 0, len(allVersions)) + for _, v := range allVersions { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } } - return err + versions = allowedVersions + return nil }) return versions, err } @@ -194,7 +102,8 @@ func versions(path string) ([]string, error) { // Previous returns the tagged version of m.Path immediately prior to // m.Version, or version "none" if no prior version is tagged. func (*mvsReqs) Previous(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { return module.Version{}, err } @@ -209,7 +118,8 @@ func (*mvsReqs) Previous(m module.Version) (module.Version, error) { // It is only used by the exclusion processing in the Required method, // not called directly by MVS. func (*mvsReqs) next(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { return module.Version{}, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e82eb1506f..f67a738677 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -52,12 +52,16 @@ import ( // version that would otherwise be chosen. This prevents accidental downgrades // from newer pre-release or development versions. // -// If the allowed function is non-nil, Query excludes any versions for which -// allowed returns false. +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { info, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -66,6 +70,17 @@ func Query(ctx context.Context, path, query, current string, allowed func(module return info, err } +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowd may be ignored. +type AllowedFunc func(context.Context, module.Version) error + var errQueryDisabled error = queryDisabledError{} type queryDisabledError struct{} @@ -77,7 +92,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) defer span.Done() @@ -88,7 +103,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, errQueryDisabled } if allowed == nil { - allowed = func(module.Version) bool { return true } + allowed = func(context.Context, module.Version) error { return nil } } // Parse query to detect parse errors (and possibly handle query) @@ -104,7 +119,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return module.CheckPathMajor(v, pathMajor) == nil } var ( - ok func(module.Version) bool + match = func(m module.Version) bool { return true } + prefix string preferOlder bool mayUseLatest bool @@ -112,21 +128,18 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed ) switch { case query == "latest": - ok = allowed mayUseLatest = true case query == "upgrade": - ok = allowed mayUseLatest = true case query == "patch": if current == "" { - ok = allowed mayUseLatest = true } else { prefix = semver.MajorMinor(current) - ok = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) && allowed(m) + match = func(m module.Version) bool { + return matchSemverPrefix(prefix, m.Version) } } @@ -139,8 +152,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) <= 0 } if !matchesMajor(v) { preferIncompatible = true @@ -151,8 +164,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if !semver.IsValid(v) { return badVersion(v) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) < 0 } if !matchesMajor(v) { preferIncompatible = true @@ -163,8 +176,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if !semver.IsValid(v) { return badVersion(v) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) >= 0 } preferOlder = true if !matchesMajor(v) { @@ -180,8 +193,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) > 0 } preferOlder = true if !matchesMajor(v) { @@ -189,8 +202,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } case semver.IsValid(query) && isSemverPrefix(query): - ok = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) && allowed(m) + match = func(m module.Version) bool { + return matchSemverPrefix(query, m.Version) } prefix = query + "." if !matchesMajor(query) { @@ -219,8 +232,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, queryErr } } - if !allowed(module.Version{Path: path, Version: info.Version}) { - return nil, fmt.Errorf("%s@%s excluded", path, info.Version) + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err } return info, nil } @@ -229,8 +242,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if query != "latest" { return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: main module version is not allowed") + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } return &modfetch.RevInfo{Version: Target.Version}, nil } @@ -248,7 +261,13 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if err != nil { return nil, err } - releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) + matchAndAllowed := func(ctx context.Context, m module.Version) error { + if !match(m) { + return ErrDisallowed + } + return allowed(ctx, m) + } + releases, prereleases, err := filterVersions(ctx, path, versions, matchAndAllowed, preferIncompatible) if err != nil { return nil, err } @@ -288,11 +307,12 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo, - // provided it is not excluded. + // Special case for "latest": if no tags match, use latest commit in repo + // if it is allowed. latest, err := repo.Latest() if err == nil { - if allowed(module.Version{Path: path, Version: latest.Version}) { + m := module.Version{Path: path, Version: latest.Version} + if err := allowed(ctx, m); !errors.Is(err, ErrDisallowed) { return lookup(latest.Version) } } else if !errors.Is(err, os.ErrNotExist) { @@ -303,6 +323,22 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, &NoMatchingVersionError{query: query, current: current} } +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (semver.IsValid(vers) && isSemverPrefix(vers)) { + return false + } + return true +} + // isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). // The caller is assumed to have checked that semver.IsValid(v) is true. func isSemverPrefix(v string) bool { @@ -329,13 +365,16 @@ func matchSemverPrefix(p, v string) bool { // filterVersions classifies versions into releases and pre-releases, filtering // out: -// 1. versions that do not satisfy the 'ok' predicate, and +// 1. versions that do not satisfy the 'allowed' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func filterVersions(ctx context.Context, path string, versions []string, allowed AllowedFunc, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { - if !ok(module.Version{Path: path, Version: v}) { + if err := allowed(ctx, module.Version{Path: path, Version: v}); errors.Is(err, ErrDisallowed) { continue } @@ -385,7 +424,7 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed AllowedFunc) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) @@ -406,7 +445,7 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFunc) ([]QueryResult, error) { ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) defer span.Done() @@ -450,8 +489,8 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul if query != "latest" { return nil, fmt.Errorf("can't query specific version for package %s in the main module (%s)", pattern, Target.Path) } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", pattern, Target.Path) + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) } return []QueryResult{{ Mod: Target, diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 77080e9b5b..351826f2ab 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -187,9 +187,11 @@ func TestQuery(t *testing.T) { if allow == "" { allow = "*" } - allowed := func(m module.Version) bool { - ok, _ := path.Match(allow, m.Version) - return ok + allowed := func(ctx context.Context, m module.Version) error { + if ok, _ := path.Match(allow, m.Version); !ok { + return ErrDisallowed + } + return nil } tt := tt t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 71f68efbcc..9f34b829fc 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -133,7 +133,7 @@ func checkVendorConsistency() { readVendorList() pre114 := false - if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 { + if semver.Compare(index.goVersionV, "v1.14") < 0 { // Go versions before 1.14 did not include enough information in // vendor/modules.txt to check for consistency. // If we know that we're on an earlier version, relax the consistency check. @@ -150,6 +150,8 @@ func checkVendorConsistency() { } } + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. for _, r := range modFile.Require { if !vendorMeta[r.Mod].Explicit { if pre114 { diff --git a/src/cmd/go/internal/mvs/errors.go b/src/cmd/go/internal/mvs/errors.go new file mode 100644 index 0000000000..5564965fb5 --- /dev/null +++ b/src/cmd/go/internal/mvs/errors.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isUpgrade function reports whether a path step is due to an upgrade. +// A nil isUpgrade function indicates that none of the path steps are due to upgrades. +func NewBuildListError(err error, path []module.Version, isUpgrade func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isUpgrade != nil && isUpgrade(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} diff --git a/src/cmd/go/internal/mvs/mvs.go b/src/cmd/go/internal/mvs/mvs.go index 1f8eaa1f60..ea23a9f45e 100644 --- a/src/cmd/go/internal/mvs/mvs.go +++ b/src/cmd/go/internal/mvs/mvs.go @@ -9,7 +9,6 @@ package mvs import ( "fmt" "sort" - "strings" "sync" "sync/atomic" @@ -61,59 +60,6 @@ type Reqs interface { Previous(m module.Version) (module.Version, error) } -// BuildListError decorates an error that occurred gathering requirements -// while constructing a build list. BuildListError prints the chain -// of requirements to the module where the error occurred. -type BuildListError struct { - Err error - stack []buildListErrorElem -} - -type buildListErrorElem struct { - m module.Version - - // nextReason is the reason this module depends on the next module in the - // stack. Typically either "requires", or "upgraded to". - nextReason string -} - -// Module returns the module where the error occurred. If the module stack -// is empty, this returns a zero value. -func (e *BuildListError) Module() module.Version { - if len(e.stack) == 0 { - return module.Version{} - } - return e.stack[0].m -} - -func (e *BuildListError) Error() string { - b := &strings.Builder{} - stack := e.stack - - // Don't print modules at the beginning of the chain without a - // version. These always seem to be the main module or a - // synthetic module ("target@"). - for len(stack) > 0 && stack[len(stack)-1].m.Version == "" { - stack = stack[:len(stack)-1] - } - - for i := len(stack) - 1; i >= 1; i-- { - fmt.Fprintf(b, "%s@%s %s\n\t", stack[i].m.Path, stack[i].m.Version, stack[i].nextReason) - } - if len(stack) == 0 { - b.WriteString(e.Err.Error()) - } else { - // Ensure that the final module path and version are included as part of the - // error message. - if _, ok := e.Err.(*module.ModuleError); ok { - fmt.Fprintf(b, "%v", e.Err) - } else { - fmt.Fprintf(b, "%v", module.VersionError(stack[0].m, e.Err)) - } - } - return b.String() -} - // BuildList returns the build list for the target module. // // target is the root vertex of a module requirement graph. For cmd/go, this is @@ -202,18 +148,30 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m q = q[1:] if node.err != nil { - err := &BuildListError{ - Err: node.err, - stack: []buildListErrorElem{{m: node.m}}, - } + pathUpgrade := map[module.Version]module.Version{} + + // Construct the error path reversed (from the error to the main module), + // then reverse it to obtain the usual order (from the main module to + // the error). + errPath := []module.Version{node.m} for n, prev := neededBy[node], node; n != nil; n, prev = neededBy[n], n { - reason := "requires" if n.upgrade == prev.m { - reason = "updating to" + pathUpgrade[n.m] = prev.m } - err.stack = append(err.stack, buildListErrorElem{m: n.m, nextReason: reason}) + errPath = append(errPath, n.m) } - return nil, err + i, j := 0, len(errPath)-1 + for i < j { + errPath[i], errPath[j] = errPath[j], errPath[i] + i++ + j-- + } + + isUpgrade := func(from, to module.Version) bool { + return pathUpgrade[from] == to + } + + return nil, NewBuildListError(node.err, errPath, isUpgrade) } neighbors := node.required diff --git a/src/cmd/go/internal/par/queue.go b/src/cmd/go/internal/par/queue.go new file mode 100644 index 0000000000..180bc75e34 --- /dev/null +++ b/src/cmd/go/internal/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/src/cmd/go/internal/par/queue_test.go b/src/cmd/go/internal/par/queue_test.go new file mode 100644 index 0000000000..1331e65f98 --- /dev/null +++ b/src/cmd/go/internal/par/queue_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import ( + "sync" + "testing" +) + +func TestQueueIdle(t *testing.T) { + q := NewQueue(1) + select { + case <-q.Idle(): + default: + t.Errorf("NewQueue(1) is not initially idle.") + } + + started := make(chan struct{}) + unblock := make(chan struct{}) + q.Add(func() { + close(started) + <-unblock + }) + + <-started + idle := q.Idle() + select { + case <-idle: + t.Errorf("NewQueue(1) is marked idle while processing work.") + default: + } + + close(unblock) + <-idle // Should be closed as soon as the Add callback returns. +} + +func TestQueueBacklog(t *testing.T) { + const ( + maxActive = 2 + totalWork = 3 * maxActive + ) + + q := NewQueue(maxActive) + t.Logf("q = NewQueue(%d)", maxActive) + + var wg sync.WaitGroup + wg.Add(totalWork) + started := make([]chan struct{}, totalWork) + unblock := make(chan struct{}) + for i := range started { + started[i] = make(chan struct{}) + i := i + q.Add(func() { + close(started[i]) + <-unblock + wg.Done() + }) + } + + for i, c := range started { + if i < maxActive { + <-c // Work item i should be started immediately. + } else { + select { + case <-c: + t.Errorf("Work item %d started before previous items finished.", i) + default: + } + } + } + + close(unblock) + for _, c := range started[maxActive:] { + <-c + } + wg.Wait() +} diff --git a/src/cmd/go/internal/str/str_test.go b/src/cmd/go/internal/str/str_test.go new file mode 100644 index 0000000000..147ce1a63e --- /dev/null +++ b/src/cmd/go/internal/str/str_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package str + +import "testing" + +var foldDupTests = []struct { + list []string + f1, f2 string +}{ + {StringList("math/rand", "math/big"), "", ""}, + {StringList("math", "strings"), "", ""}, + {StringList("strings"), "", ""}, + {StringList("strings", "strings"), "strings", "strings"}, + {StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, +} + +func TestFoldDup(t *testing.T) { + for _, tt := range foldDupTests { + f1, f2 := FoldDup(tt.list) + if f1 != tt.f1 || f2 != tt.f2 { + t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) + } + } +} diff --git a/src/cmd/go/internal/test/flagdefs_test.go b/src/cmd/go/internal/test/flagdefs_test.go index 7562415298..ab5440b380 100644 --- a/src/cmd/go/internal/test/flagdefs_test.go +++ b/src/cmd/go/internal/test/flagdefs_test.go @@ -16,9 +16,14 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { return } name := strings.TrimPrefix(f.Name, "test.") - if name != "testlogfile" && !passFlagToTest[name] { - t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) - t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + switch name { + case "testlogfile", "paniconexit0": + // These are internal flags. + default: + if !passFlagToTest[name] { + t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) + t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + } } }) diff --git a/src/cmd/go/internal/test/genflags.go b/src/cmd/go/internal/test/genflags.go index 512fa1671e..5e83d53980 100644 --- a/src/cmd/go/internal/test/genflags.go +++ b/src/cmd/go/internal/test/genflags.go @@ -62,9 +62,10 @@ func testFlags() []string { } name := strings.TrimPrefix(f.Name, "test.") - if name == "testlogfile" { - // test.testlogfile is “for use only by cmd/go” - } else { + switch name { + case "testlogfile", "paniconexit0": + // These flags are only for use by cmd/go. + default: names = append(names, name) } }) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 3aee6939d2..1ea6d2881e 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1164,7 +1164,8 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. if !c.disableCache && len(execCmd) == 0 { testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, testArgs) + panicArg := "-test.paniconexit0" + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. diff --git a/src/cmd/go/internal/get/discovery.go b/src/cmd/go/internal/vcs/discovery.go index afa6ef455f..327b44cb9a 100644 --- a/src/cmd/go/internal/get/discovery.go +++ b/src/cmd/go/internal/vcs/discovery.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/xml" diff --git a/src/cmd/go/internal/get/pkg_test.go b/src/cmd/go/internal/vcs/discovery_test.go index fc6a179c2e..eb99fdf64c 100644 --- a/src/cmd/go/internal/get/pkg_test.go +++ b/src/cmd/go/internal/vcs/discovery_test.go @@ -2,35 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( - "cmd/go/internal/str" "reflect" "strings" "testing" ) -var foldDupTests = []struct { - list []string - f1, f2 string -}{ - {str.StringList("math/rand", "math/big"), "", ""}, - {str.StringList("math", "strings"), "", ""}, - {str.StringList("strings"), "", ""}, - {str.StringList("strings", "strings"), "strings", "strings"}, - {str.StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, -} - -func TestFoldDup(t *testing.T) { - for _, tt := range foldDupTests { - f1, f2 := str.FoldDup(tt.list) - if f1 != tt.f1 || f2 != tt.f2 { - t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) - } - } -} - var parseMetaGoImportsTests = []struct { in string mod ModuleMode diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/vcs/vcs.go index fd37fcb76f..e535998d89 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/json" @@ -27,23 +27,23 @@ import ( // A vcsCmd describes how to use a version control system // like Mercurial, Git, or Subversion. -type vcsCmd struct { - name string - cmd string // name of binary to invoke command +type Cmd struct { + Name string + Cmd string // name of binary to invoke command - createCmd []string // commands to download a fresh copy of a repository - downloadCmd []string // commands to download updates into an existing repository + CreateCmd []string // commands to download a fresh copy of a repository + DownloadCmd []string // commands to download updates into an existing repository - tagCmd []tagCmd // commands to list tags - tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd []string // commands to sync to specific tag - tagSyncDefault []string // commands to sync to default tag + TagCmd []tagCmd // commands to list tags + TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + TagSyncCmd []string // commands to sync to specific tag + TagSyncDefault []string // commands to sync to default tag - scheme []string - pingCmd string + Scheme []string + PingCmd string - remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) - resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) + RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) + ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) } var defaultSecureScheme = map[string]bool{ @@ -54,7 +54,7 @@ var defaultSecureScheme = map[string]bool{ "ssh": true, } -func (v *vcsCmd) isSecure(repo string) bool { +func (v *Cmd) IsSecure(repo string) bool { u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. @@ -63,8 +63,8 @@ func (v *vcsCmd) isSecure(repo string) bool { return v.isSecureScheme(u.Scheme) } -func (v *vcsCmd) isSecureScheme(scheme string) bool { - switch v.cmd { +func (v *Cmd) isSecureScheme(scheme string) bool { + switch v.Cmd { case "git": // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a // colon-separated list of schemes that are allowed to be used with git @@ -89,7 +89,7 @@ type tagCmd struct { } // vcsList lists the known version control systems -var vcsList = []*vcsCmd{ +var vcsList = []*Cmd{ vcsHg, vcsGit, vcsSvn, @@ -97,11 +97,15 @@ var vcsList = []*vcsCmd{ vcsFossil, } +// vcsMod is a stub for the "mod" scheme. It's returned by +// repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. +var vcsMod = &Cmd{Name: "mod"} + // vcsByCmd returns the version control system for the given // command name (hg, git, svn, bzr). -func vcsByCmd(cmd string) *vcsCmd { +func vcsByCmd(cmd string) *Cmd { for _, vcs := range vcsList { - if vcs.cmd == cmd { + if vcs.Cmd == cmd { return vcs } } @@ -109,31 +113,31 @@ func vcsByCmd(cmd string) *vcsCmd { } // vcsHg describes how to use Mercurial. -var vcsHg = &vcsCmd{ - name: "Mercurial", - cmd: "hg", +var vcsHg = &Cmd{ + Name: "Mercurial", + Cmd: "hg", - createCmd: []string{"clone -U -- {repo} {dir}"}, - downloadCmd: []string{"pull"}, + CreateCmd: []string{"clone -U -- {repo} {dir}"}, + DownloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' // for selecting a version. This lets people have // a go.release.r60 branch and a go1 branch // and make changes in both, without constantly // editing .hgtags. - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ {"tags", `^(\S+)`}, {"branches", `^(\S+)`}, }, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update default"}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update default"}, - scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify -- {scheme}://{repo}", - remoteRepo: hgRemoteRepo, + Scheme: []string{"https", "http", "ssh"}, + PingCmd: "identify -- {scheme}://{repo}", + RemoteRepo: hgRemoteRepo, } -func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { +func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsHg.runOutput(rootDir, "paths default") if err != nil { return "", err @@ -142,45 +146,45 @@ func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) } // vcsGit describes how to use Git. -var vcsGit = &vcsCmd{ - name: "Git", - cmd: "git", +var vcsGit = &Cmd{ + Name: "Git", + Cmd: "git", - createCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, - downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ // tags/xxx matches a git tag named xxx // origin/xxx matches a git branch named xxx on the default remote repository {"show-ref", `(?:tags|origin)/(\S+)$`}, }, - tagLookupCmd: []tagCmd{ + TagLookupCmd: []tagCmd{ {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, }, - tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, // both createCmd and downloadCmd update the working dir. // No need to do more here. We used to 'checkout master' // but that doesn't work if the default branch is not named master. // DO NOT add 'checkout master' here. // See golang.org/issue/9032. - tagSyncDefault: []string{"submodule update --init --recursive"}, + TagSyncDefault: []string{"submodule update --init --recursive"}, - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not // support such a separator for that command, and this use should be safe // without it because the {scheme} value comes from the predefined list above. // See golang.org/issue/33836. - pingCmd: "ls-remote {scheme}://{repo}", + PingCmd: "ls-remote {scheme}://{repo}", - remoteRepo: gitRemoteRepo, + RemoteRepo: gitRemoteRepo, } // scpSyntaxRe matches the SCP-like addresses used by Git to access // repositories by SSH. var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) -func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { +func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { cmd := "config remote.origin.url" errParse := errors.New("unable to parse output of git " + cmd) errRemoteOriginNotFound := errors.New("remote origin not found") @@ -216,7 +220,7 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error // Iterate over insecure schemes too, because this function simply // reports the state of the repo. If we can't see insecure schemes then // we can't report the actual repo URL. - for _, s := range vcsGit.scheme { + for _, s := range vcsGit.Scheme { if repoURL.Scheme == s { return repoURL.String(), nil } @@ -225,27 +229,27 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } // vcsBzr describes how to use Bazaar. -var vcsBzr = &vcsCmd{ - name: "Bazaar", - cmd: "bzr", +var vcsBzr = &Cmd{ + Name: "Bazaar", + Cmd: "bzr", - createCmd: []string{"branch -- {repo} {dir}"}, + CreateCmd: []string{"branch -- {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: []string{"pull --overwrite"}, + DownloadCmd: []string{"pull --overwrite"}, - tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update -r revno:-1"}, + TagCmd: []tagCmd{{"tags", `^(\S+)`}}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update -r revno:-1"}, - scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: bzrRemoteRepo, - resolveRepo: bzrResolveRepo, + Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: bzrRemoteRepo, + ResolveRepo: bzrResolveRepo, } -func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { +func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "config parent_location") if err != nil { return "", err @@ -253,7 +257,7 @@ func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error return strings.TrimSpace(string(outb)), nil } -func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { +func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) if err != nil { return "", err @@ -287,22 +291,22 @@ func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string } // vcsSvn describes how to use Subversion. -var vcsSvn = &vcsCmd{ - name: "Subversion", - cmd: "svn", +var vcsSvn = &Cmd{ + Name: "Subversion", + Cmd: "svn", - createCmd: []string{"checkout -- {repo} {dir}"}, - downloadCmd: []string{"update"}, + CreateCmd: []string{"checkout -- {repo} {dir}"}, + DownloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. - scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: svnRemoteRepo, + Scheme: []string{"https", "http", "svn", "svn+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: svnRemoteRepo, } -func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { +func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsSvn.runOutput(rootDir, "info") if err != nil { return "", err @@ -337,22 +341,22 @@ func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error const fossilRepoName = ".fossil" // vcsFossil describes how to use Fossil (fossil-scm.org) -var vcsFossil = &vcsCmd{ - name: "Fossil", - cmd: "fossil", +var vcsFossil = &Cmd{ + Name: "Fossil", + Cmd: "fossil", - createCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, - downloadCmd: []string{"up"}, + CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + DownloadCmd: []string{"up"}, - tagCmd: []tagCmd{{"tag ls", `(.*)`}}, - tagSyncCmd: []string{"up tag:{tag}"}, - tagSyncDefault: []string{"up trunk"}, + TagCmd: []tagCmd{{"tag ls", `(.*)`}}, + TagSyncCmd: []string{"up tag:{tag}"}, + TagSyncDefault: []string{"up trunk"}, - scheme: []string{"https", "http"}, - remoteRepo: fossilRemoteRepo, + Scheme: []string{"https", "http"}, + RemoteRepo: fossilRemoteRepo, } -func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err error) { +func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsFossil.runOutput(rootDir, "remote-url") if err != nil { return "", err @@ -360,8 +364,8 @@ func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err return strings.TrimSpace(string(out)), nil } -func (v *vcsCmd) String() string { - return v.name +func (v *Cmd) String() string { + return v.Name } // run runs the command line cmd in the given directory. @@ -371,24 +375,24 @@ func (v *vcsCmd) String() string { // If an error occurs, run prints the command line and the // command's combined stdout+stderr to standard error. // Otherwise run discards the command's output. -func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { +func (v *Cmd) run(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, true) return err } // runVerboseOnly is like run but only generates error output to standard error in verbose mode. -func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { +func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, false) return err } // runOutput is like run but returns the output of the command. -func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { +func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { return v.run1(dir, cmd, keyval, true) } // run1 is the generalized implementation of run and runOutput. -func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { +func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { m := make(map[string]string) for i := 0; i < len(keyval); i += 2 { m[keyval[i]] = keyval[i+1] @@ -420,25 +424,25 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) args = args[2:] } - _, err := exec.LookPath(v.cmd) + _, err := exec.LookPath(v.Cmd) if err != nil { fmt.Fprintf(os.Stderr, "go: missing %s command. See https://golang.org/s/gogetcmd\n", - v.name) + v.Name) return nil, err } - cmd := exec.Command(v.cmd, args...) + cmd := exec.Command(v.Cmd, args...) cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) if cfg.BuildX { fmt.Fprintf(os.Stderr, "cd %s\n", dir) - fmt.Fprintf(os.Stderr, "%s %s\n", v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) } out, err := cmd.Output() if err != nil { if verbose || cfg.BuildV { - fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { os.Stderr.Write(ee.Stderr) } else { @@ -449,15 +453,15 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) return out, err } -// ping pings to determine scheme to use. -func (v *vcsCmd) ping(scheme, repo string) error { - return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) +// Ping pings to determine scheme to use. +func (v *Cmd) Ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo) } -// create creates a new copy of repo in dir. +// Create creates a new copy of repo in dir. // The parent of dir must exist; dir must not. -func (v *vcsCmd) create(dir, repo string) error { - for _, cmd := range v.createCmd { +func (v *Cmd) Create(dir, repo string) error { + for _, cmd := range v.CreateCmd { if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { return err } @@ -465,9 +469,9 @@ func (v *vcsCmd) create(dir, repo string) error { return nil } -// download downloads any new changes for the repo in dir. -func (v *vcsCmd) download(dir string) error { - for _, cmd := range v.downloadCmd { +// Download downloads any new changes for the repo in dir. +func (v *Cmd) Download(dir string) error { + for _, cmd := range v.DownloadCmd { if err := v.run(dir, cmd); err != nil { return err } @@ -475,10 +479,10 @@ func (v *vcsCmd) download(dir string) error { return nil } -// tags returns the list of available tags for the repo in dir. -func (v *vcsCmd) tags(dir string) ([]string, error) { +// Tags returns the list of available tags for the repo in dir. +func (v *Cmd) Tags(dir string) ([]string, error) { var tags []string - for _, tc := range v.tagCmd { + for _, tc := range v.TagCmd { out, err := v.runOutput(dir, tc.cmd) if err != nil { return nil, err @@ -493,12 +497,12 @@ func (v *vcsCmd) tags(dir string) ([]string, error) { // tagSync syncs the repo in dir to the named tag, // which either is a tag returned by tags or is v.tagDefault. -func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == nil { +func (v *Cmd) TagSync(dir, tag string) error { + if v.TagSyncCmd == nil { return nil } if tag != "" { - for _, tc := range v.tagLookupCmd { + for _, tc := range v.TagLookupCmd { out, err := v.runOutput(dir, tc.cmd, "tag", tag) if err != nil { return err @@ -512,8 +516,8 @@ func (v *vcsCmd) tagSync(dir, tag string) error { } } - if tag == "" && v.tagSyncDefault != nil { - for _, cmd := range v.tagSyncDefault { + if tag == "" && v.TagSyncDefault != nil { + for _, cmd := range v.TagSyncDefault { if err := v.run(dir, cmd); err != nil { return err } @@ -521,7 +525,7 @@ func (v *vcsCmd) tagSync(dir, tag string) error { return nil } - for _, cmd := range v.tagSyncCmd { + for _, cmd := range v.TagSyncCmd { if err := v.run(dir, cmd, "tag", tag); err != nil { return err } @@ -540,11 +544,11 @@ type vcsPath struct { schemelessRepo bool // if true, the repo pattern lacks a scheme } -// vcsFromDir inspects dir and its parents to determine the +// FromDir inspects dir and its parents to determine the // version control system and code repository to use. // On return, root is the import path // corresponding to the root of the repository. -func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { +func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { // Clean and double-check that dir is in (a subdirectory of) srcRoot. dir = filepath.Clean(dir) srcRoot = filepath.Clean(srcRoot) @@ -552,13 +556,13 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } - var vcsRet *vcsCmd + var vcsRet *Cmd var rootRet string origDir := dir for len(dir) > len(srcRoot) { for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { root := filepath.ToSlash(dir[len(srcRoot)+1:]) // Record first VCS we find, but keep looking, // to detect mistakes like one kind of VCS inside another. @@ -568,12 +572,12 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { continue } // Allow .git inside .git, which can arise due to submodules. - if vcsRet == vcs && vcs.cmd == "git" { + if vcsRet == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", - filepath.Join(srcRoot, rootRet), vcsRet.cmd, filepath.Join(srcRoot, root), vcs.cmd) + filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) } } @@ -593,9 +597,9 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) } -// checkNestedVCS checks for an incorrectly-nested VCS-inside-VCS +// CheckNested checks for an incorrectly-nested VCS-inside-VCS // situation for dir, checking parents up until srcRoot. -func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { +func CheckNested(vcs *Cmd, dir, srcRoot string) error { if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } @@ -603,17 +607,17 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { otherDir := dir for len(otherDir) > len(srcRoot) { for _, otherVCS := range vcsList { - if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { // Allow expected vcs in original dir. if otherDir == dir && otherVCS == vcs { continue } // Allow .git inside .git, which can arise due to submodules. - if otherVCS == vcs && vcs.cmd == "git" { + if otherVCS == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. - return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.cmd, otherDir, otherVCS.cmd) + return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) } } // Move to parent. @@ -633,9 +637,7 @@ type RepoRoot struct { Repo string // repository URL, including scheme Root string // import path corresponding to root of repo IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) - VCS string // vcs type ("mod", "git", ...) - - vcs *vcsCmd // internal: vcs command access + VCS *Cmd } func httpPrefix(s string) string { @@ -735,15 +737,15 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if !srv.schemelessRepo { repoURL = match["repo"] } else { - scheme := vcs.scheme[0] // default to first scheme + scheme := vcs.Scheme[0] // default to first scheme repo := match["repo"] - if vcs.pingCmd != "" { + if vcs.PingCmd != "" { // If we know how to test schemes, scan to find one. - for _, s := range vcs.scheme { + for _, s := range vcs.Scheme { if security == web.SecureOnly && !vcs.isSecureScheme(s) { continue } - if vcs.ping(s, repo) == nil { + if vcs.Ping(s, repo) == nil { scheme = s break } @@ -754,8 +756,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths rr := &RepoRoot{ Repo: repoURL, Root: match["root"], - VCS: vcs.cmd, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -846,17 +847,21 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if err := validateRepoRoot(mmi.RepoRoot); err != nil { return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) } - vcs := vcsByCmd(mmi.VCS) - if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + var vcs *Cmd + if mmi.VCS == "mod" { + vcs = vcsMod + } else { + vcs = vcsByCmd(mmi.VCS) + if vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + } } rr := &RepoRoot{ Repo: mmi.RepoRoot, Root: mmi.Prefix, IsCustom: true, - VCS: mmi.VCS, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -1027,7 +1032,7 @@ var vcsPaths = []*vcsPath{ // Github { prefix: "github.com/", - regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`), + regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", check: noVCSSuffix, @@ -1103,7 +1108,7 @@ var vcsPathsAfterDynamic = []*vcsPath{ func noVCSSuffix(match map[string]string) error { repo := match["repo"] for _, vcs := range vcsList { - if strings.HasSuffix(repo, "."+vcs.cmd) { + if strings.HasSuffix(repo, "."+vcs.Cmd) { return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) } } @@ -1133,7 +1138,7 @@ func bitbucketVCS(match map[string]string) error { // VCS it uses. See issue 5375. root := match["root"] for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).ping("https", root) == nil { + if vcsByCmd(vcs).Ping("https", root) == nil { resp.SCM = vcs break } diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/vcs/vcs_test.go index 91800baa83..5b874204f1 100644 --- a/src/cmd/go/internal/get/vcs_test.go +++ b/src/cmd/go/internal/vcs/vcs_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "errors" @@ -28,30 +28,27 @@ func TestRepoRootForImportPath(t *testing.T) { { "github.com/golang/groupcache", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://github.com/golang/groupcache", }, }, - // Unicode letters in directories (issue 18660). + // Unicode letters in directories are not valid. { "github.com/user/unicode/испытание", - &RepoRoot{ - vcs: vcsGit, - Repo: "https://github.com/user/unicode", - }, + nil, }, // IBM DevOps Services tests { "hub.jazz.net/git/user1/pkgname", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, { "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, @@ -92,7 +89,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "hub.jazz.net/git/user/pkg.name", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user/pkg.name", }, }, @@ -105,7 +102,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -115,14 +112,14 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift.git", }, }, { "git.openstack.org/openstack/swift/go/hummingbird", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -151,21 +148,21 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.apache.org/package-name.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name.git", }, }, { "git.apache.org/package-name_2.x.git/path/to/lib", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name_2.x.git", }, }, { "chiselapp.com/user/kyle/repository/fossilgg", &RepoRoot{ - vcs: vcsFossil, + VCS: vcsFossil, Repo: "https://chiselapp.com/user/kyle/repository/fossilgg", }, }, @@ -194,8 +191,8 @@ func TestRepoRootForImportPath(t *testing.T) { t.Errorf("RepoRootForImportPath(%q): %v", test.path, err) continue } - if got.vcs.name != want.vcs.name || got.Repo != want.Repo { - t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo) + if got.VCS.Name != want.VCS.Name || got.Repo != want.Repo { + t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.VCS, got.Repo, want.VCS, want.Repo) } } } @@ -209,7 +206,7 @@ func TestFromDir(t *testing.T) { defer os.RemoveAll(tempDir) for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) + dir := filepath.Join(tempDir, "example.com", vcs.Name, "."+vcs.Cmd) if j&1 == 0 { err := os.MkdirAll(dir, 0755) if err != nil { @@ -228,24 +225,24 @@ func TestFromDir(t *testing.T) { } want := RepoRoot{ - vcs: vcs, - Root: path.Join("example.com", vcs.name), + VCS: vcs, + Root: path.Join("example.com", vcs.Name), } var got RepoRoot - got.vcs, got.Root, err = vcsFromDir(dir, tempDir) + got.VCS, got.Root, err = FromDir(dir, tempDir) if err != nil { t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) continue } - if got.vcs.name != want.vcs.name || got.Root != want.Root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root) + if got.VCS.Name != want.VCS.Name || got.Root != want.Root { + t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.VCS, got.Root, want.VCS, want.Root) } } } func TestIsSecure(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -270,7 +267,7 @@ func TestIsSecure(t *testing.T) { } for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } @@ -279,7 +276,7 @@ func TestIsSecure(t *testing.T) { func TestIsSecureGitAllowProtocol(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -310,7 +307,7 @@ func TestIsSecureGitAllowProtocol(t *testing.T) { defer os.Unsetenv("GIT_ALLOW_PROTOCOL") os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index d020aa6e9f..e99982ed36 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -240,13 +240,12 @@ const ( // AddBuildFlags adds the flags common to the build, clean, get, // install, list, run, and test commands. func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { + base.AddBuildFlagsNX(&cmd.Flag) cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") - cmd.Flag.BoolVar(&cfg.BuildN, "n", false, "") cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") if mask&OmitVFlag == 0 { cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") } - cmd.Flag.BoolVar(&cfg.BuildX, "x", false, "") cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") cmd.Flag.Var(buildCompiler{}, "compiler", "") @@ -254,10 +253,10 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") if mask&OmitModFlag == 0 { - cmd.Flag.StringVar(&cfg.BuildMod, "mod", "", "") + base.AddModFlag(&cmd.Flag) } if mask&OmitModCommonFlags == 0 { - AddModCommonFlags(cmd) + base.AddModCommonFlags(&cmd.Flag) } cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") @@ -275,13 +274,6 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") } -// AddModCommonFlags adds the module-related flags common to build commands -// and 'go mod' subcommands. -func AddModCommonFlags(cmd *base.Command) { - cmd.Flag.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") - cmd.Flag.StringVar(&cfg.ModFile, "modfile", "", "") -} - // tagsFlag is the implementation of the -tags flag. type tagsFlag []string diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index f1d08e0268..6031897f88 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -259,6 +259,15 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } + if p.ImportPath == "runtime" && objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { // Define GOMIPS_value from cfg.GOMIPS. diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index dad3b10111..f78020032c 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -252,7 +252,7 @@ func buildModeInit() { switch cfg.BuildMod { case "": - // ok + // Behavior will be determined automatically, as if no flag were passed. case "readonly", "vendor", "mod": if !cfg.ModulesEnabled && !inGOFLAGS("-mod") { base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index 3ee68ac1b4..d2a2697f0f 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -177,6 +177,7 @@ var validLinkerFlags = []*lazyregexp.Regexp{ re(`-Wl,-Bdynamic`), re(`-Wl,-berok`), re(`-Wl,-Bstatic`), + re(`-Wl,-Bsymbolic-functions`), re(`-WL,-O([^@,\-][^,]*)?`), re(`-Wl,-d[ny]`), re(`-Wl,--disable-new-dtags`), |