diff options
Diffstat (limited to 'src/cmd/go')
164 files changed, 5212 insertions, 1599 deletions
diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 68bad3cff1..8ad4f66d09 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -916,6 +916,7 @@ // Dir string // directory holding files for this module, if any // GoMod string // path to go.mod file used when loading this module, if any // GoVersion string // go version used in module +// Retracted string // retraction information, if any (with -retracted or -u) // Error *ModuleError // error loading module // } // @@ -947,14 +948,16 @@ // The -u flag adds information about available upgrades. // When the latest version of a given module is newer than // the current one, list -u sets the Module's Update field -// to information about the newer module. +// to information about the newer module. list -u will also set +// the module's Retracted field if the current version is retracted. // The Module's String method indicates an available upgrade by // formatting the newer version in brackets after the current version. +// If a version is retracted, the string "(retracted)" will follow it. // For example, 'go list -m -u all' might print: // // my/main/module // golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text -// rsc.io/pdf v0.1.1 [v0.1.2] +// rsc.io/pdf v0.1.1 (retracted) [v0.1.2] // // (For tools, 'go list -m -u -json all' may be more convenient to parse.) // @@ -964,6 +967,14 @@ // the default output format to display the module path followed by the // space-separated version list. // +// The -retracted flag causes list to report information about retracted +// module versions. When -retracted is used with -f or -json, the Retracted +// field will be set to a string explaining why the version was retracted. +// The string is taken from comments on the retract directive in the +// module's go.mod file. When -retracted is used with -versions, retracted +// versions are listed together with unretracted versions. The -retracted +// flag may be used with or without -m. +// // The arguments to list -m are interpreted as a list of modules, not packages. // The main module is the module containing the current directory. // The active modules are the main module and its dependencies. @@ -1100,9 +1111,14 @@ // module path and version pair. If the @v is omitted, a replacement without // a version on the left side is dropped. // +// The -retract=version and -dropretract=version flags add and drop a +// retraction on the given version. The version may be a single version +// like "v1.2.3" or a closed interval like "[v1.1.0-v1.1.9]". Note that +// -retract=version is a no-op if that retraction already exists. +// // The -require, -droprequire, -exclude, -dropexclude, -replace, -// and -dropreplace editing flags may be repeated, and the changes -// are applied in the order given. +// -dropreplace, -retract, and -dropretract editing flags may be repeated, +// and the changes are applied in the order given. // // The -go=version flag sets the expected Go language version. // @@ -1136,6 +1152,15 @@ // New Module // } // +// type Retract struct { +// Low string +// High string +// Rationale string +// } +// +// Retract entries representing a single version (not an interval) will have +// the "Low" and "High" fields set to the same value. +// // Note that this only describes the go.mod file itself, not other modules // referred to indirectly. For the full set of modules available to a build, // use 'go list -m -json all'. @@ -1894,15 +1919,17 @@ // require new/thing/v2 v2.3.4 // exclude old/thing v1.2.3 // replace bad/thing v1.4.5 => good/thing v1.4.5 +// retract v1.5.6 // // The verbs are // module, to define the module path; // go, to set the expected language version; // require, to require a particular module at a given version or later; -// exclude, to exclude a particular module version from use; and -// replace, to replace a module version with a different module version. +// exclude, to exclude a particular module version from use; +// replace, to replace a module version with a different module version; and +// retract, to indicate a previously released version should not be used. // Exclude and replace apply only in the main module's go.mod and are ignored -// in dependencies. See https://research.swtch.com/vgo-mvs for details. +// in dependencies. See https://golang.org/ref/mod for details. // // The leading verb can be factored out of adjacent lines to create a block, // like in Go imports: @@ -2145,7 +2172,10 @@ // before resolving dependencies or building the code. // // The -insecure flag permits fetching from repositories and resolving -// custom domains using insecure schemes such as HTTP. Use with caution. +// custom domains using insecure schemes such as HTTP. Use with caution. The +// GOINSECURE environment variable is usually a better alternative, since it +// provides control over which modules may be retrieved using an insecure scheme. +// See 'go help environment' for details. // // The -t flag instructs get to also download the packages required to build // the tests for the specified packages. diff --git a/src/cmd/go/internal/base/flag.go b/src/cmd/go/internal/base/flag.go index 6727196816..c97c744520 100644 --- a/src/cmd/go/internal/base/flag.go +++ b/src/cmd/go/internal/base/flag.go @@ -28,13 +28,42 @@ func (v *StringsFlag) String() string { return "<StringsFlag>" } +// explicitStringFlag is like a regular string flag, but it also tracks whether +// the string was set explicitly to a non-empty value. +type explicitStringFlag struct { + value *string + explicit *bool +} + +func (f explicitStringFlag) String() string { + if f.value == nil { + return "" + } + return *f.value +} + +func (f explicitStringFlag) Set(v string) error { + *f.value = v + if v != "" { + *f.explicit = true + } + return nil +} + // AddBuildFlagsNX adds the -n and -x build flags to the flag set. func AddBuildFlagsNX(flags *flag.FlagSet) { flags.BoolVar(&cfg.BuildN, "n", false, "") flags.BoolVar(&cfg.BuildX, "x", false, "") } -// AddLoadFlags adds the -mod build flag to the flag set. -func AddLoadFlags(flags *flag.FlagSet) { - flags.StringVar(&cfg.BuildMod, "mod", "", "") +// AddModFlag adds the -mod build flag to the flag set. +func AddModFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") +} + +// AddModCommonFlags adds the module-related flags common to build commands +// and 'go mod' subcommands. +func AddModCommonFlags(flags *flag.FlagSet) { + flags.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") + flags.StringVar(&cfg.ModFile, "modfile", "", "") } diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index f9bbcd9180..9bf1db73ef 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -27,7 +27,8 @@ var ( BuildBuildmode string // -buildmode flag BuildContext = defaultContext() BuildMod string // -mod flag - BuildModReason string // reason -mod flag is set, if set by default + BuildModExplicit bool // whether -mod was set explicitly + BuildModReason string // reason -mod was set, if set by default BuildI bool // -i flag BuildLinkshared bool // -linkshared flag BuildMSan bool // -msan flag @@ -48,6 +49,8 @@ var ( ModCacheRW bool // -modcacherw flag ModFile string // -modfile flag + Insecure bool // -insecure flag + CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index f96cff429c..b0c1c59b40 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -23,7 +23,8 @@ import ( func init() { base.AddBuildFlagsNX(&CmdFmt.Flag) - base.AddLoadFlags(&CmdFmt.Flag) + base.AddModFlag(&CmdFmt.Flag) + base.AddModCommonFlags(&CmdFmt.Flag) } var CmdFmt = &base.Command{ diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index e5bacadaa3..3f7a66384a 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -18,8 +18,11 @@ import ( "cmd/go/internal/load" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/vcs" "cmd/go/internal/web" "cmd/go/internal/work" + + "golang.org/x/mod/module" ) var CmdGet = &base.Command{ @@ -41,7 +44,10 @@ The -fix flag instructs get to run the fix tool on the downloaded packages before resolving dependencies or building the code. The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. +custom domains using insecure schemes such as HTTP. Use with caution. The +GOINSECURE environment variable is usually a better alternative, since it +provides control over which modules may be retrieved using an insecure scheme. +See 'go help environment' for details. The -t flag instructs get to also download the packages required to build the tests for the specified packages. @@ -103,14 +109,12 @@ var ( getT = CmdGet.Flag.Bool("t", false, "") getU = CmdGet.Flag.Bool("u", false, "") getFix = CmdGet.Flag.Bool("fix", false, "") - - Insecure bool ) func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag|work.OmitModCommonFlags) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") } func runGet(ctx context.Context, cmd *base.Command, args []string) { @@ -403,17 +407,12 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) // to make the first copy of or update a copy of the given package. func downloadPackage(p *load.Package) error { var ( - vcs *vcsCmd + vcsCmd *vcs.Cmd repo, rootPath string err error blindRepo bool // set if the repo has unusual configuration ) - security := web.SecureOnly - if Insecure { - security = web.Insecure - } - // p can be either a real package, or a pseudo-package whose “import path” is // actually a wildcard pattern. // Trim the path at the element containing the first wildcard, @@ -427,22 +426,26 @@ func downloadPackage(p *load.Package) error { } importPrefix = importPrefix[:slash] } - if err := CheckImportPath(importPrefix); err != nil { + if err := module.CheckImportPath(importPrefix); err != nil { return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err) } + security := web.SecureOnly + if cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, importPrefix) { + security = web.Insecure + } if p.Internal.Build.SrcRoot != "" { // Directory exists. Look for checkout along path to src. - vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot) + vcsCmd, rootPath, err = vcs.FromDir(p.Dir, p.Internal.Build.SrcRoot) if err != nil { return err } repo = "<local>" // should be unused; make distinctive // Double-check where it came from. - if *getU && vcs.remoteRepo != nil { + if *getU && vcsCmd.RemoteRepo != nil { dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - remote, err := vcs.remoteRepo(vcs, dir) + remote, err := vcsCmd.RemoteRepo(vcsCmd, dir) if err != nil { // Proceed anyway. The package is present; we likely just don't understand // the repo configuration (e.g. unusual remote protocol). @@ -450,10 +453,10 @@ func downloadPackage(p *load.Package) error { } repo = remote if !*getF && err == nil { - if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil { + if rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security); err == nil { repo := rr.Repo - if rr.vcs.resolveRepo != nil { - resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) + if rr.VCS.ResolveRepo != nil { + resolved, err := rr.VCS.ResolveRepo(rr.VCS, dir, repo) if err == nil { repo = resolved } @@ -467,13 +470,13 @@ func downloadPackage(p *load.Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(importPrefix, vcs.IgnoreMod, security) if err != nil { return err } - vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root + vcsCmd, repo, rootPath = rr.VCS, rr.Repo, rr.Root } - if !blindRepo && !vcs.isSecure(repo) && !Insecure { + if !blindRepo && !vcsCmd.IsSecure(repo) && security != web.Insecure { return fmt.Errorf("cannot download, %v uses insecure protocol", repo) } @@ -496,7 +499,7 @@ func downloadPackage(p *load.Package) error { } root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) - if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil { + if err := vcs.CheckNested(vcsCmd, root, p.Internal.Build.SrcRoot); err != nil { return err } @@ -512,7 +515,7 @@ func downloadPackage(p *load.Package) error { // Check that this is an appropriate place for the repo to be checked out. // The target directory must either not exist or have a repo checked out already. - meta := filepath.Join(root, "."+vcs.cmd) + meta := filepath.Join(root, "."+vcsCmd.Cmd) if _, err := os.Stat(meta); err != nil { // Metadata file or directory does not exist. Prepare to checkout new copy. // Some version control tools require the target directory not to exist. @@ -533,12 +536,12 @@ func downloadPackage(p *load.Package) error { fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root) } - if err = vcs.create(root, repo); err != nil { + if err = vcsCmd.Create(root, repo); err != nil { return err } } else { // Metadata directory does exist; download incremental updates. - if err = vcs.download(root); err != nil { + if err = vcsCmd.Download(root); err != nil { return err } } @@ -547,12 +550,12 @@ func downloadPackage(p *load.Package) error { // Do not show tag sync in -n; it's noise more than anything, // and since we're not running commands, no tag will be found. // But avoid printing nothing. - fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) + fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcsCmd.Cmd) return nil } // Select and sync to appropriate version of the repository. - tags, err := vcs.tags(root) + tags, err := vcsCmd.Tags(root) if err != nil { return err } @@ -560,7 +563,7 @@ func downloadPackage(p *load.Package) error { if i := strings.Index(vers, " "); i >= 0 { vers = vers[:i] } - if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { + if err := vcsCmd.TagSync(root, selectTag(vers, tags)); err != nil { return err } diff --git a/src/cmd/go/internal/get/path.go b/src/cmd/go/internal/get/path.go deleted file mode 100644 index ce2e0cdd70..0000000000 --- a/src/cmd/go/internal/get/path.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package get - -import ( - "fmt" - "strings" - "unicode" - "unicode/utf8" -) - -// The following functions are copied verbatim from golang.org/x/mod/module/module.go, -// with a change to additionally reject Windows short-names, -// and one to accept arbitrary letters (golang.org/issue/29101). -// -// TODO(bcmills): After the call site for this function is backported, -// consolidate this back down to a single copy. -// -// NOTE: DO NOT MERGE THESE UNTIL WE DECIDE ABOUT ARBITRARY LETTERS IN MODULE MODE. - -// CheckImportPath checks that an import path is valid. -func CheckImportPath(path string) error { - if err := checkPath(path, false); err != nil { - return fmt.Errorf("malformed import path %q: %v", path, err) - } - return nil -} - -// checkPath checks that a general path is valid. -// It returns an error describing why but not mentioning path. -// Because these checks apply to both module paths and import paths, -// the caller is expected to add the "malformed ___ path %q: " prefix. -// fileName indicates whether the final element of the path is a file name -// (as opposed to a directory name). -func checkPath(path string, fileName bool) error { - if !utf8.ValidString(path) { - return fmt.Errorf("invalid UTF-8") - } - if path == "" { - return fmt.Errorf("empty string") - } - if path[0] == '-' { - return fmt.Errorf("leading dash") - } - if strings.Contains(path, "//") { - return fmt.Errorf("double slash") - } - if path[len(path)-1] == '/' { - return fmt.Errorf("trailing slash") - } - elemStart := 0 - for i, r := range path { - if r == '/' { - if err := checkElem(path[elemStart:i], fileName); err != nil { - return err - } - elemStart = i + 1 - } - } - if err := checkElem(path[elemStart:], fileName); err != nil { - return err - } - return nil -} - -// checkElem checks whether an individual path element is valid. -// fileName indicates whether the element is a file name (not a directory name). -func checkElem(elem string, fileName bool) error { - if elem == "" { - return fmt.Errorf("empty path element") - } - if strings.Count(elem, ".") == len(elem) { - return fmt.Errorf("invalid path element %q", elem) - } - if elem[0] == '.' && !fileName { - return fmt.Errorf("leading dot in path element") - } - if elem[len(elem)-1] == '.' { - return fmt.Errorf("trailing dot in path element") - } - - charOK := pathOK - if fileName { - charOK = fileNameOK - } - for _, r := range elem { - if !charOK(r) { - return fmt.Errorf("invalid char %q", r) - } - } - - // Windows disallows a bunch of path elements, sadly. - // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file - short := elem - if i := strings.Index(short, "."); i >= 0 { - short = short[:i] - } - for _, bad := range badWindowsNames { - if strings.EqualFold(bad, short) { - return fmt.Errorf("disallowed path element %q", elem) - } - } - - // Reject path components that look like Windows short-names. - // Those usually end in a tilde followed by one or more ASCII digits. - if tilde := strings.LastIndexByte(short, '~'); tilde >= 0 && tilde < len(short)-1 { - suffix := short[tilde+1:] - suffixIsDigits := true - for _, r := range suffix { - if r < '0' || r > '9' { - suffixIsDigits = false - break - } - } - if suffixIsDigits { - return fmt.Errorf("trailing tilde and digits in path element") - } - } - - return nil -} - -// pathOK reports whether r can appear in an import path element. -// -// NOTE: This function DIVERGES from module mode pathOK by accepting Unicode letters. -func pathOK(r rune) bool { - if r < utf8.RuneSelf { - return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' || - '0' <= r && r <= '9' || - 'A' <= r && r <= 'Z' || - 'a' <= r && r <= 'z' - } - return unicode.IsLetter(r) -} - -// fileNameOK reports whether r can appear in a file name. -// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters. -// If we expand the set of allowed characters here, we have to -// work harder at detecting potential case-folding and normalization collisions. -// See note about "safe encoding" below. -func fileNameOK(r rune) bool { - if r < utf8.RuneSelf { - // Entire set of ASCII punctuation, from which we remove characters: - // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~ - // We disallow some shell special characters: " ' * < > ? ` | - // (Note that some of those are disallowed by the Windows file system as well.) - // We also disallow path separators / : and \ (fileNameOK is only called on path element characters). - // We allow spaces (U+0020) in file names. - const allowed = "!#$%&()+,-.=@[]^_{}~ " - if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' { - return true - } - for i := 0; i < len(allowed); i++ { - if rune(allowed[i]) == r { - return true - } - } - return false - } - // It may be OK to add more ASCII punctuation here, but only carefully. - // For example Windows disallows < > \, and macOS disallows :, so we must not allow those. - return unicode.IsLetter(r) -} - -// badWindowsNames are the reserved file path elements on Windows. -// See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file -var badWindowsNames = []string{ - "CON", - "PRN", - "AUX", - "NUL", - "COM1", - "COM2", - "COM3", - "COM4", - "COM5", - "COM6", - "COM7", - "COM8", - "COM9", - "LPT1", - "LPT2", - "LPT3", - "LPT4", - "LPT5", - "LPT6", - "LPT7", - "LPT8", - "LPT9", -} diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index e68c39f392..23500dd9d8 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "os" "sort" @@ -215,6 +216,7 @@ applied to a Go struct, but now a Module struct: Dir string // directory holding files for this module, if any GoMod string // path to go.mod file used when loading this module, if any GoVersion string // go version used in module + Retracted string // retraction information, if any (with -retracted or -u) Error *ModuleError // error loading module } @@ -246,14 +248,16 @@ the replaced source code.) The -u flag adds information about available upgrades. When the latest version of a given module is newer than the current one, list -u sets the Module's Update field -to information about the newer module. +to information about the newer module. list -u will also set +the module's Retracted field if the current version is retracted. The Module's String method indicates an available upgrade by formatting the newer version in brackets after the current version. +If a version is retracted, the string "(retracted)" will follow it. For example, 'go list -m -u all' might print: my/main/module golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text - rsc.io/pdf v0.1.1 [v0.1.2] + rsc.io/pdf v0.1.1 (retracted) [v0.1.2] (For tools, 'go list -m -u -json all' may be more convenient to parse.) @@ -263,6 +267,14 @@ to semantic versioning, earliest to latest. The flag also changes the default output format to display the module path followed by the space-separated version list. +The -retracted flag causes list to report information about retracted +module versions. When -retracted is used with -f or -json, the Retracted +field will be set to a string explaining why the version was retracted. +The string is taken from comments on the retract directive in the +module's go.mod file. When -retracted is used with -versions, retracted +versions are listed together with unretracted versions. The -retracted +flag may be used with or without -m. + The arguments to list -m are interpreted as a list of modules, not packages. The main module is the module containing the current directory. The active modules are the main module and its dependencies. @@ -296,17 +308,18 @@ func init() { } var ( - listCompiled = CmdList.Flag.Bool("compiled", false, "") - listDeps = CmdList.Flag.Bool("deps", false, "") - listE = CmdList.Flag.Bool("e", false, "") - listExport = CmdList.Flag.Bool("export", false, "") - listFmt = CmdList.Flag.String("f", "", "") - listFind = CmdList.Flag.Bool("find", false, "") - listJson = CmdList.Flag.Bool("json", false, "") - listM = CmdList.Flag.Bool("m", false, "") - listU = CmdList.Flag.Bool("u", false, "") - listTest = CmdList.Flag.Bool("test", false, "") - listVersions = CmdList.Flag.Bool("versions", false, "") + listCompiled = CmdList.Flag.Bool("compiled", false, "") + listDeps = CmdList.Flag.Bool("deps", false, "") + listE = CmdList.Flag.Bool("e", false, "") + listExport = CmdList.Flag.Bool("export", false, "") + listFmt = CmdList.Flag.String("f", "", "") + listFind = CmdList.Flag.Bool("find", false, "") + listJson = CmdList.Flag.Bool("json", false, "") + listM = CmdList.Flag.Bool("m", false, "") + listRetracted = CmdList.Flag.Bool("retracted", false, "") + listTest = CmdList.Flag.Bool("test", false, "") + listU = CmdList.Flag.Bool("u", false, "") + listVersions = CmdList.Flag.Bool("versions", false, "") ) var nl = []byte{'\n'} @@ -367,6 +380,16 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } + modload.Init() + if *listRetracted { + if cfg.BuildMod == "vendor" { + base.Fatalf("go list -retracted cannot be used when vendoring is enabled") + } + if !modload.Enabled() { + base.Fatalf("go list -retracted can only be used in module-aware mode") + } + } + if *listM { // Module mode. if *listCompiled { @@ -414,9 +437,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } - modload.LoadBuildList(ctx) - - mods := modload.ListModules(ctx, args, *listU, *listVersions) + mods := modload.ListModules(ctx, args, *listU, *listVersions, *listRetracted) if !*listE { for _, m := range mods { if m.Error != nil { @@ -522,7 +543,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { // Note that -deps is applied after -test, // so that you only get descriptions of tests for the things named // explicitly on the command line, not for all dependencies. - pkgs = load.PackageList(pkgs) + pkgs = loadPackageList(pkgs) } // Do we need to run a build to gather information? @@ -557,7 +578,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { if *listTest { all := pkgs if !*listDeps { - all = load.PackageList(pkgs) + all = loadPackageList(pkgs) } // Update import paths to distinguish the real package p // from p recompiled for q.test. @@ -607,6 +628,55 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } + // TODO(golang.org/issue/40676): This mechanism could be extended to support + // -u without -m. + if *listRetracted { + // Load retractions for modules that provide packages that will be printed. + // TODO(golang.org/issue/40775): Packages from the same module refer to + // distinct ModulePublic instance. It would be nice if they could all point + // to the same instance. This would require additional global state in + // modload.loaded, so that should be refactored first. For now, we update + // all instances. + modToArg := make(map[*modinfo.ModulePublic]string) + argToMods := make(map[string][]*modinfo.ModulePublic) + var args []string + addModule := func(mod *modinfo.ModulePublic) { + if mod.Version == "" { + return + } + arg := fmt.Sprintf("%s@%s", mod.Path, mod.Version) + if argToMods[arg] == nil { + args = append(args, arg) + } + argToMods[arg] = append(argToMods[arg], mod) + modToArg[mod] = arg + } + for _, p := range pkgs { + if p.Module == nil { + continue + } + addModule(p.Module) + if p.Module.Replace != nil { + addModule(p.Module.Replace) + } + } + + if len(args) > 0 { + listU := false + listVersions := false + rmods := modload.ListModules(ctx, args, listU, listVersions, *listRetracted) + for i, arg := range args { + rmod := rmods[i] + for _, mod := range argToMods[arg] { + mod.Retracted = rmod.Retracted + if rmod.Error != nil && mod.Error == nil { + mod.Error = rmod.Error + } + } + } + } + } + // Record non-identity import mappings in p.ImportMap. for _, p := range pkgs { for i, srcPath := range p.Internal.RawImports { @@ -625,6 +695,23 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } +// loadPackageList is like load.PackageList, but prints error messages and exits +// with nonzero status if listE is not set and any package in the expanded list +// has errors. +func loadPackageList(roots []*load.Package) []*load.Package { + pkgs := load.PackageList(roots) + + if !*listE { + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("%v", pkg.Error) + } + } + } + + return pkgs +} + // TrackingWriter tracks the last byte written on every write so // we can avoid printing a newline if one was already written or // if there is no output at all. diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index a0e275095b..e0f13323df 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -191,6 +191,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p GoFiles: p.XTestGoFiles, Imports: p.XTestImports, ForTest: p.ImportPath, + Module: p.Module, Error: pxtestErr, }, Internal: PackageInternal{ @@ -222,6 +223,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p ImportPath: p.ImportPath + ".test", Root: p.Root, Imports: str.StringList(TestMainDeps), + Module: p.Module, }, Internal: PackageInternal{ Build: &build.Package{Name: "main"}, diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index d4c161fca1..6227fd9f33 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -12,9 +12,8 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modload" "cmd/go/internal/modfetch" - "cmd/go/internal/work" + "cmd/go/internal/modload" "golang.org/x/mod/module" ) @@ -64,7 +63,7 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") - work.AddModCommonFlags(cmdDownload) + base.AddModCommonFlags(&cmdDownload.Flag) } type moduleJSON struct { @@ -136,9 +135,10 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { var mods []*moduleJSON listU := false listVersions := false + listRetractions := false type token struct{} sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + for _, info := range modload.ListModules(ctx, args, listU, listVersions, listRetractions) { if info.Replace != nil { info = info.Replace } @@ -187,4 +187,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } base.ExitIfErrors() } + + // Update go.mod and especially go.sum if needed. + modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go index a81c25270f..03a774b824 100644 --- a/src/cmd/go/internal/modcmd/edit.go +++ b/src/cmd/go/internal/modcmd/edit.go @@ -19,7 +19,6 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -68,9 +67,14 @@ The -dropreplace=old[@v] flag drops a replacement of the given module path and version pair. If the @v is omitted, a replacement without a version on the left side is dropped. +The -retract=version and -dropretract=version flags add and drop a +retraction on the given version. The version may be a single version +like "v1.2.3" or a closed interval like "[v1.1.0-v1.1.9]". Note that +-retract=version is a no-op if that retraction already exists. + The -require, -droprequire, -exclude, -dropexclude, -replace, -and -dropreplace editing flags may be repeated, and the changes -are applied in the order given. +-dropreplace, -retract, and -dropretract editing flags may be repeated, +and the changes are applied in the order given. The -go=version flag sets the expected Go language version. @@ -104,6 +108,15 @@ writing it back to go.mod. The JSON output corresponds to these Go types: New Module } + type Retract struct { + Low string + High string + Rationale string + } + +Retract entries representing a single version (not an interval) will have +the "Low" and "High" fields set to the same value. + Note that this only describes the go.mod file itself, not other modules referred to indirectly. For the full set of modules available to a build, use 'go list -m -json all'. @@ -137,8 +150,10 @@ func init() { cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "") + cmdEdit.Flag.Var(flagFunc(flagRetract), "retract", "") + cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "") - work.AddModCommonFlags(cmdEdit) + base.AddModCommonFlags(&cmdEdit.Flag) base.AddBuildFlagsNX(&cmdEdit.Flag) } @@ -252,12 +267,7 @@ func parsePathVersion(flag, arg string) (path, version string) { base.Fatalf("go mod: -%s=%s: invalid path: %v", flag, arg, err) } - // We don't call modfile.CheckPathVersion, because that insists - // on versions being in semver form, but here we want to allow - // versions like "master" or "1234abcdef", which the go command will resolve - // the next time it runs (or during -fix). - // Even so, we need to make sure the version is a valid token. - if modfile.MustQuote(version) { + if !allowedVersionArg(version) { base.Fatalf("go mod: -%s=%s: invalid version %q", flag, arg, version) } @@ -289,12 +299,48 @@ func parsePathVersionOptional(adj, arg string, allowDirPath bool) (path, version return path, version, fmt.Errorf("invalid %s path: %v", adj, err) } } - if path != arg && modfile.MustQuote(version) { + if path != arg && !allowedVersionArg(version) { return path, version, fmt.Errorf("invalid %s version: %q", adj, version) } return path, version, nil } +// parseVersionInterval parses a single version like "v1.2.3" or a closed +// interval like "[v1.2.3,v1.4.5]". Note that a single version has the same +// representation as an interval with equal upper and lower bounds: both +// Low and High are set. +func parseVersionInterval(arg string) (modfile.VersionInterval, error) { + if !strings.HasPrefix(arg, "[") { + if !allowedVersionArg(arg) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version: %q", arg) + } + return modfile.VersionInterval{Low: arg, High: arg}, nil + } + if !strings.HasSuffix(arg, "]") { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + s := arg[1 : len(arg)-1] + i := strings.Index(s, ",") + if i < 0 { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + low := strings.TrimSpace(s[:i]) + high := strings.TrimSpace(s[i+1:]) + if !allowedVersionArg(low) || !allowedVersionArg(high) { + return modfile.VersionInterval{}, fmt.Errorf("invalid version interval: %q", arg) + } + return modfile.VersionInterval{Low: low, High: high}, nil +} + +// allowedVersionArg returns whether a token may be used as a version in go.mod. +// We don't call modfile.CheckPathVersion, because that insists on versions +// being in semver form, but here we want to allow versions like "master" or +// "1234abcdef", which the go command will resolve the next time it runs (or +// during -fix). Even so, we need to make sure the version is a valid token. +func allowedVersionArg(arg string) bool { + return !modfile.MustQuote(arg) +} + // flagRequire implements the -require flag. func flagRequire(arg string) { path, version := parsePathVersion("require", arg) @@ -377,6 +423,32 @@ func flagDropReplace(arg string) { }) } +// flagRetract implements the -retract flag. +func flagRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.AddRetract(vi, ""); err != nil { + base.Fatalf("go mod: -retract=%s: %v", arg, err) + } + }) +} + +// flagDropRetract implements the -dropretract flag. +func flagDropRetract(arg string) { + vi, err := parseVersionInterval(arg) + if err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + edits = append(edits, func(f *modfile.File) { + if err := f.DropRetract(vi); err != nil { + base.Fatalf("go mod: -dropretract=%s: %v", arg, err) + } + }) +} + // fileJSON is the -json output data structure. type fileJSON struct { Module module.Version @@ -384,6 +456,7 @@ type fileJSON struct { Require []requireJSON Exclude []module.Version Replace []replaceJSON + Retract []retractJSON } type requireJSON struct { @@ -397,6 +470,12 @@ type replaceJSON struct { New module.Version } +type retractJSON struct { + Low string `json:",omitempty"` + High string `json:",omitempty"` + Rationale string `json:",omitempty"` +} + // editPrintJSON prints the -json output. func editPrintJSON(modFile *modfile.File) { var f fileJSON @@ -415,6 +494,9 @@ func editPrintJSON(modFile *modfile.File) { for _, r := range modFile.Replace { f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) } + for _, r := range modFile.Retract { + f.Retract = append(f.Retract, retractJSON{r.Low, r.High, r.Rationale}) + } data, err := json.MarshalIndent(&f, "", "\t") if err != nil { base.Fatalf("go: internal error: %v", err) diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 6da12b9cab..a149b65605 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -33,7 +32,7 @@ path@version, except for the main module, which has no @version suffix. } func init() { - work.AddModCommonFlags(cmdGraph) + base.AddModCommonFlags(&cmdGraph.Flag) } func runGraph(ctx context.Context, cmd *base.Command, args []string) { @@ -48,7 +47,7 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go: cannot find main module; see 'go help modules'") } } - modload.LoadBuildList(ctx) + modload.LoadAllModules(ctx) reqs := modload.MinReqs() format := func(m module.Version) string { diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index b6cffd332d..21b235653e 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -9,7 +9,6 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/modload" - "cmd/go/internal/work" "context" "os" "strings" @@ -30,7 +29,7 @@ To override this guess, supply the module path as an argument. } func init() { - work.AddModCommonFlags(cmdInit) + base.AddModCommonFlags(&cmdInit.Flag) } func runInit(ctx context.Context, cmd *base.Command, args []string) { diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index c7c53d7c0c..30df674ef6 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -10,7 +10,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/work" "context" ) @@ -32,7 +31,7 @@ to standard error. func init() { cmdTidy.Run = runTidy // break init cycle cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdTidy) + base.AddModCommonFlags(&cmdTidy.Flag) } func runTidy(ctx context.Context, cmd *base.Command, args []string) { @@ -40,6 +39,18 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } + // Tidy aims to make 'go test' reproducible for any package in 'all', so we + // need to include test dependencies. For modules that specify go 1.15 or + // earlier this is a no-op (because 'all' saturates transitive test + // dependencies). + // + // However, with lazy loading (go 1.16+) 'all' includes only the packages that + // are transitively imported by the main module, not the test dependencies of + // those packages. In order to make 'go test' reproducible for the packages + // that are in 'all' but outside of the main module, we must explicitly + // request that their test dependencies be included. + modload.LoadTests = true + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index e5353b5c7f..91d2509452 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -19,7 +19,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -41,7 +40,7 @@ modules and packages to standard error. func init() { cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "") - work.AddModCommonFlags(cmdVendor) + base.AddModCommonFlags(&cmdVendor.Flag) } func runVendor(ctx context.Context, cmd *base.Command, args []string) { @@ -77,7 +76,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } var buf bytes.Buffer - for _, m := range modload.BuildList()[1:] { + for _, m := range modload.LoadedModules()[1:] { if pkgs := modpkgs[m]; len(pkgs) > 0 || isExplicit[m] { line := moduleLine(m, modload.Replacement(m)) buf.WriteString(line) diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 73ab714d10..7700588bde 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -17,7 +17,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -38,7 +37,7 @@ non-zero status. } func init() { - work.AddModCommonFlags(cmdVerify) + base.AddModCommonFlags(&cmdVerify.Flag) } func runVerify(ctx context.Context, cmd *base.Command, args []string) { @@ -60,7 +59,7 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) { sem := make(chan token, runtime.GOMAXPROCS(0)) // Use a slice of result channels, so that the output is deterministic. - mods := modload.LoadBuildList(ctx)[1:] + mods := modload.LoadAllModules(ctx)[1:] errsChans := make([]<-chan []error, len(mods)) for i, mod := range mods { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index da33fff89e..8454fdfec6 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -11,7 +11,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" - "cmd/go/internal/work" "golang.org/x/mod/module" ) @@ -58,23 +57,26 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle - work.AddModCommonFlags(cmdWhy) + base.AddModCommonFlags(&cmdWhy.Flag) } func runWhy(ctx context.Context, cmd *base.Command, args []string) { loadALL := modload.LoadALL if *whyVendor { loadALL = modload.LoadVendor + } else { + modload.LoadTests = true } if *whyM { listU := false listVersions := false + listRetractions := false for _, arg := range args { if strings.Contains(arg, "@") { base.Fatalf("go mod why: module query not allowed") } } - mods := modload.ListModules(ctx, args, listU, listVersions) + mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) byModule := make(map[module.Version][]string) for _, path := range loadALL(ctx) { m := modload.PackageModule(path) diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e29eb0a942..01d8f007ac 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -503,6 +503,9 @@ func checkGoMod(path, version string, data []byte) error { } // checkModSum checks that the recorded checksum for mod is h. +// +// mod.Version may have the additional suffix "/go.mod" to request the checksum +// for the module's go.mod file only. func checkModSum(mod module.Version, h string) error { // We lock goSum when manipulating it, // but we arrange to release the lock when calling checkSumDB, @@ -579,9 +582,16 @@ func addModSumLocked(mod module.Version, h string) { // checkSumDB checks the mod, h pair against the Go checksum database. // It calls base.Fatalf if the hash is to be rejected. func checkSumDB(mod module.Version, h string) error { + modWithoutSuffix := mod + noun := "module" + if strings.HasSuffix(mod.Version, "/go.mod") { + noun = "go.mod" + modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod") + } + db, lines, err := lookupSumDB(mod) if err != nil { - return module.VersionError(mod, fmt.Errorf("verifying module: %v", err)) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err)) } have := mod.Path + " " + mod.Version + " " + h @@ -591,7 +601,7 @@ func checkSumDB(mod module.Version, h string) error { return nil } if strings.HasPrefix(line, prefix) { - return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):])) + return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):])) } } return nil diff --git a/src/cmd/go/internal/modfetch/insecure.go b/src/cmd/go/internal/modfetch/insecure.go index b692669cba..012d05f29d 100644 --- a/src/cmd/go/internal/modfetch/insecure.go +++ b/src/cmd/go/internal/modfetch/insecure.go @@ -6,12 +6,11 @@ package modfetch import ( "cmd/go/internal/cfg" - "cmd/go/internal/get" "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) + return cfg.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 34f805d58a..eed4dd4258 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -13,9 +13,9 @@ import ( "time" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" + "cmd/go/internal/vcs" web "cmd/go/internal/web" "golang.org/x/mod/module" @@ -261,13 +261,13 @@ func lookupDirect(path string) (Repo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.PreferMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security) if err != nil { // We don't know where to find code for a module with this path. return nil, notExistError{err: err} } - if rr.VCS == "mod" { + if rr.VCS.Name == "mod" { // Fetch module from proxy with base URL rr.Repo. return newProxyRepo(rr.Repo, path) } @@ -279,8 +279,8 @@ func lookupDirect(path string) (Repo, error) { return newCodeRepo(code, rr.Root, path) } -func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { - code, err := codehost.NewRepo(rr.VCS, rr.Repo) +func lookupCodeRepo(rr *vcs.RepoRoot) (codehost.Repo, error) { + code, err := codehost.NewRepo(rr.VCS.Cmd, rr.Repo) if err != nil { if _, ok := err.(*codehost.VCSError); ok { return nil, err @@ -306,7 +306,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { if allowInsecure(path) { security = web.Insecure } - rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, security) + rr, err := vcs.RepoRootForImportPath(path, vcs.IgnoreMod, security) if err != nil { return nil, nil, err } diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 783c4a433b..47a2571531 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -22,7 +22,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/get" "cmd/go/internal/lockedfile" "cmd/go/internal/web" @@ -33,7 +32,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !cfg.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index ee9757912b..829cfe055a 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -17,7 +17,7 @@ import ( "sync" "cmd/go/internal/base" - "cmd/go/internal/get" + "cmd/go/internal/cfg" "cmd/go/internal/imports" "cmd/go/internal/load" "cmd/go/internal/modload" @@ -181,7 +181,7 @@ var ( getM = CmdGet.Flag.Bool("m", false, "") getT = CmdGet.Flag.Bool("t", false, "") getU upgradeFlag - // -insecure is get.Insecure + // -insecure is cfg.Insecure // -v is cfg.BuildV ) @@ -206,7 +206,7 @@ func (v *upgradeFlag) String() string { return "" } func init() { work.AddBuildFlags(CmdGet, work.OmitModFlag) CmdGet.Run = runGet // break init loop - CmdGet.Flag.BoolVar(&get.Insecure, "insecure", get.Insecure, "") + CmdGet.Flag.BoolVar(&cfg.Insecure, "insecure", cfg.Insecure, "") CmdGet.Flag.Var(&getU, "u", "") } @@ -278,7 +278,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } modload.LoadTests = *getT - buildList := modload.LoadBuildList(ctx) + buildList := modload.LoadAllModules(ctx) buildList = buildList[:len(buildList):len(buildList)] // copy on append versionByPath := make(map[string]string) for _, m := range buildList { @@ -290,7 +290,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // what was requested. modload.DisallowWriteGoMod() - // Allow looking up modules for import paths outside of a module. + // Allow looking up modules for import paths when outside of a module. // 'go get' is expected to do this, unlike other commands. modload.AllowMissingModuleImports() @@ -599,7 +599,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { base.ExitIfErrors() // Stop if no changes have been made to the build list. - buildList = modload.BuildList() + buildList = modload.LoadedModules() eq := len(buildList) == len(prevBuildList) for i := 0; eq && i < len(buildList); i++ { eq = buildList[i] == prevBuildList[i] @@ -617,7 +617,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Handle downgrades. var down []module.Version - for _, m := range modload.BuildList() { + for _, m := range modload.LoadedModules() { q := byPath[m.Path] if q != nil && semver.Compare(m.Version, q.m.Version) > 0 { down = append(down, module.Version{Path: m.Path, Version: q.m.Version}) @@ -628,6 +628,10 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if err != nil { base.Fatalf("go: %v", err) } + + // TODO(bcmills) What should happen here under lazy loading? + // Downgrading may intentionally violate the lazy-loading invariants. + modload.SetBuildList(buildList) modload.ReloadBuildList() // note: does not update go.mod base.ExitIfErrors() @@ -637,7 +641,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { var lostUpgrades []*query if len(down) > 0 { versionByPath = make(map[string]string) - for _, m := range modload.BuildList() { + for _, m := range modload.LoadedModules() { versionByPath[m.Path] = m.Version } for _, q := range byPath { @@ -702,6 +706,15 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Everything succeeded. Update go.mod. modload.AllowWriteGoMod() modload.WriteGoMod() + modload.DisallowWriteGoMod() + + // Report warnings if any retracted versions are in the build list. + // This must be done after writing go.mod to avoid spurious '// indirect' + // comments. These functions read and write global state. + // TODO(golang.org/issue/40775): ListModules resets modload.loader, which + // contains information about direct dependencies that WriteGoMod uses. + // Refactor to avoid these kinds of global side effects. + reportRetractions(ctx) // If -d was specified, we're done after the module work. // We've already downloaded modules by loading packages above. @@ -804,6 +817,14 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc base.Fatalf("go get: internal error: prevM may be set if and only if forceModulePath is set") } + // If vers is a query like "latest", we should ignore retracted and excluded + // versions. If vers refers to a specific version or commit like "v1.0.0" + // or "master", we should only ignore excluded versions. + allowed := modload.CheckAllowed + if modload.IsRevisionQuery(vers) { + allowed = modload.CheckExclusions + } + // If the query must be a module path, try only that module path. if forceModulePath { if path == modload.Target.Path { @@ -812,7 +833,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc } } - info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +859,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -864,190 +885,41 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc return m, nil } -// An upgrader adapts an underlying mvs.Reqs to apply an -// upgrade policy to a list of targets and their dependencies. -type upgrader struct { - mvs.Reqs - - // cmdline maps a module path to a query made for that module at a - // specific target version. Each query corresponds to a module - // matched by a command line argument. - cmdline map[string]*query - - // upgrade is a set of modules providing dependencies of packages - // matched by command line arguments. If -u or -u=patch is set, - // these modules are upgraded accordingly. - upgrade map[string]bool -} - -// newUpgrader creates an upgrader. cmdline contains queries made at -// specific versions for modules matched by command line arguments. pkgs -// is the set of packages matched by command line arguments. If -u or -u=patch -// is set, modules providing dependencies of pkgs are upgraded accordingly. -func newUpgrader(cmdline map[string]*query, pkgs map[string]bool) *upgrader { - u := &upgrader{ - Reqs: modload.Reqs(), - cmdline: cmdline, - } - if getU != "" { - u.upgrade = make(map[string]bool) - - // Traverse package import graph. - // Initialize work queue with root packages. - seen := make(map[string]bool) - var work []string - add := func(path string) { - if !seen[path] { - seen[path] = true - work = append(work, path) - } - } - for pkg := range pkgs { - add(pkg) +// reportRetractions prints warnings if any modules in the build list are +// retracted. +func reportRetractions(ctx context.Context) { + // Query for retractions of modules in the build list. + // Use modload.ListModules, since that provides information in the same format + // as 'go list -m'. Don't query for "all", since that's not allowed outside a + // module. + buildList := modload.LoadedModules() + args := make([]string, 0, len(buildList)) + for _, m := range buildList { + if m.Version == "" { + // main module or dummy target module + continue } - for len(work) > 0 { - pkg := work[0] - work = work[1:] - m := modload.PackageModule(pkg) - u.upgrade[m.Path] = true - - // testImports is empty unless test imports were actually loaded, - // i.e., -t was set or "all" was one of the arguments. - imports, testImports := modload.PackageImports(pkg) - for _, imp := range imports { - add(imp) - } - for _, imp := range testImports { - add(imp) + args = append(args, m.Path+"@"+m.Version) + } + listU := false + listVersions := false + listRetractions := true + mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions) + retractPath := "" + for _, mod := range mods { + if len(mod.Retracted) > 0 { + if retractPath == "" { + retractPath = mod.Path + } else { + retractPath = "<module>" } + rationale := modload.ShortRetractionRationale(mod.Retracted[0]) + logOncef("go: warning: %s@%s is retracted: %s", mod.Path, mod.Version, rationale) } } - return u -} - -// Required returns the requirement list for m. -// For the main module, we override requirements with the modules named -// one the command line, and we include new requirements. Otherwise, -// we defer to u.Reqs. -func (u *upgrader) Required(m module.Version) ([]module.Version, error) { - rs, err := u.Reqs.Required(m) - if err != nil { - return nil, err - } - if m != modload.Target { - return rs, nil - } - - overridden := make(map[string]bool) - for i, m := range rs { - if q := u.cmdline[m.Path]; q != nil && q.m.Version != "none" { - rs[i] = q.m - overridden[q.m.Path] = true - } - } - for _, q := range u.cmdline { - if !overridden[q.m.Path] && q.m.Path != modload.Target.Path && q.m.Version != "none" { - rs = append(rs, q.m) - } - } - return rs, nil -} - -// Upgrade returns the desired upgrade for m. -// -// If m was requested at a specific version on the command line, then -// Upgrade returns that version. -// -// If -u is set and m provides a dependency of a package matched by -// command line arguments, then Upgrade may provider a newer tagged version. -// If m is a tagged version, then Upgrade will return the latest tagged -// version (with the same minor version number if -u=patch). -// If m is a pseudo-version, then Upgrade returns the latest tagged version -// only if that version has a time-stamp newer than m. This special case -// prevents accidental downgrades when already using a pseudo-version -// newer than the latest tagged version. -// -// If none of the above cases apply, then Upgrade returns m. -func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { - // Allow pkg@vers on the command line to override the upgrade choice v. - // If q's version is < m.Version, then we're going to downgrade anyway, - // and it's cleaner to avoid moving back and forth and picking up - // extraneous other newer dependencies. - // If q's version is > m.Version, then we're going to upgrade past - // m.Version anyway, and again it's cleaner to avoid moving back and forth - // picking up extraneous other newer dependencies. - if q := u.cmdline[m.Path]; q != nil { - return q.m, nil - } - - if !u.upgrade[m.Path] { - // Not involved in upgrade. Leave alone. - return m, nil - } - - // Run query required by upgrade semantics. - // Note that Query "latest" is not the same as using repo.Latest, - // which may return a pseudoversion for the latest commit. - // Query "latest" returns the newest tagged version or the newest - // prerelease version if there are no non-prereleases, or repo.Latest - // if there aren't any tagged versions. - // If we're querying "upgrade" or "patch", Query will compare the current - // version against the chosen version and will return the current version - // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) - if err != nil { - // Report error but return m, to let version selection continue. - // (Reporting the error will fail the command at the next base.ExitIfErrors.) - - // Special case: if the error is for m.Version itself and m.Version has a - // replacement, then keep it and don't report the error: the fact that the - // version is invalid is likely the reason it was replaced to begin with. - var vErr *module.InvalidVersionError - if errors.As(err, &vErr) && vErr.Version == m.Version && modload.Replacement(m).Path != "" { - return m, nil - } - - // Special case: if the error is "no matching versions" then don't - // even report the error. Because Query does not consider pseudo-versions, - // it may happen that we have a pseudo-version but during -u=patch - // the query v0.0 matches no versions (not even the one we're using). - var noMatch *modload.NoMatchingVersionError - if !errors.As(err, &noMatch) { - base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) - } - return m, nil - } - - if info.Version != m.Version { - logOncef("go: %s %s => %s", m.Path, getU, info.Version) - } - return module.Version{Path: m.Path, Version: info.Version}, nil -} - -// buildListForLostUpgrade returns the build list for the module graph -// rooted at lost. Unlike mvs.BuildList, the target module (lost) is not -// treated specially. The returned build list may contain a newer version -// of lost. -// -// buildListForLostUpgrade is used after a downgrade has removed a module -// requested at a specific version. This helps us understand the requirements -// implied by each downgrade. -func buildListForLostUpgrade(lost module.Version, reqs mvs.Reqs) ([]module.Version, error) { - return mvs.BuildList(lostUpgradeRoot, &lostUpgradeReqs{Reqs: reqs, lost: lost}) -} - -var lostUpgradeRoot = module.Version{Path: "lost-upgrade-root", Version: ""} - -type lostUpgradeReqs struct { - mvs.Reqs - lost module.Version -} - -func (r *lostUpgradeReqs) Required(mod module.Version) ([]module.Version, error) { - if mod == lostUpgradeRoot { - return []module.Version{r.lost}, nil + if modload.HasModRoot() && retractPath != "" { + logOncef("go: run 'go get %s@latest' to switch to the latest unretracted version", retractPath) } - return r.Reqs.Required(mod) } var loggedLines sync.Map diff --git a/src/cmd/go/internal/modget/mvs.go b/src/cmd/go/internal/modget/mvs.go new file mode 100644 index 0000000000..19fffd2947 --- /dev/null +++ b/src/cmd/go/internal/modget/mvs.go @@ -0,0 +1,202 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modget + +import ( + "context" + "errors" + + "cmd/go/internal/base" + "cmd/go/internal/modload" + "cmd/go/internal/mvs" + + "golang.org/x/mod/module" +) + +// An upgrader adapts an underlying mvs.Reqs to apply an +// upgrade policy to a list of targets and their dependencies. +type upgrader struct { + mvs.Reqs + + // cmdline maps a module path to a query made for that module at a + // specific target version. Each query corresponds to a module + // matched by a command line argument. + cmdline map[string]*query + + // upgrade is a set of modules providing dependencies of packages + // matched by command line arguments. If -u or -u=patch is set, + // these modules are upgraded accordingly. + upgrade map[string]bool +} + +// newUpgrader creates an upgrader. cmdline contains queries made at +// specific versions for modules matched by command line arguments. pkgs +// is the set of packages matched by command line arguments. If -u or -u=patch +// is set, modules providing dependencies of pkgs are upgraded accordingly. +func newUpgrader(cmdline map[string]*query, pkgs map[string]bool) *upgrader { + u := &upgrader{ + Reqs: modload.Reqs(), + cmdline: cmdline, + } + if getU != "" { + u.upgrade = make(map[string]bool) + + // Traverse package import graph. + // Initialize work queue with root packages. + seen := make(map[string]bool) + var work []string + add := func(path string) { + if !seen[path] { + seen[path] = true + work = append(work, path) + } + } + for pkg := range pkgs { + add(pkg) + } + for len(work) > 0 { + pkg := work[0] + work = work[1:] + m := modload.PackageModule(pkg) + u.upgrade[m.Path] = true + + // testImports is empty unless test imports were actually loaded, + // i.e., -t was set or "all" was one of the arguments. + imports, testImports := modload.PackageImports(pkg) + for _, imp := range imports { + add(imp) + } + for _, imp := range testImports { + add(imp) + } + } + } + return u +} + +// Required returns the requirement list for m. +// For the main module, we override requirements with the modules named +// one the command line, and we include new requirements. Otherwise, +// we defer to u.Reqs. +func (u *upgrader) Required(m module.Version) ([]module.Version, error) { + rs, err := u.Reqs.Required(m) + if err != nil { + return nil, err + } + if m != modload.Target { + return rs, nil + } + + overridden := make(map[string]bool) + for i, m := range rs { + if q := u.cmdline[m.Path]; q != nil && q.m.Version != "none" { + rs[i] = q.m + overridden[q.m.Path] = true + } + } + for _, q := range u.cmdline { + if !overridden[q.m.Path] && q.m.Path != modload.Target.Path && q.m.Version != "none" { + rs = append(rs, q.m) + } + } + return rs, nil +} + +// Upgrade returns the desired upgrade for m. +// +// If m was requested at a specific version on the command line, then +// Upgrade returns that version. +// +// If -u is set and m provides a dependency of a package matched by +// command line arguments, then Upgrade may provider a newer tagged version. +// If m is a tagged version, then Upgrade will return the latest tagged +// version (with the same minor version number if -u=patch). +// If m is a pseudo-version, then Upgrade returns the latest tagged version +// only if that version has a time-stamp newer than m. This special case +// prevents accidental downgrades when already using a pseudo-version +// newer than the latest tagged version. +// +// If none of the above cases apply, then Upgrade returns m. +func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { + // Allow pkg@vers on the command line to override the upgrade choice v. + // If q's version is < m.Version, then we're going to downgrade anyway, + // and it's cleaner to avoid moving back and forth and picking up + // extraneous other newer dependencies. + // If q's version is > m.Version, then we're going to upgrade past + // m.Version anyway, and again it's cleaner to avoid moving back and forth + // picking up extraneous other newer dependencies. + if q := u.cmdline[m.Path]; q != nil { + return q.m, nil + } + + if !u.upgrade[m.Path] { + // Not involved in upgrade. Leave alone. + return m, nil + } + + // Run query required by upgrade semantics. + // Note that Query "latest" is not the same as using repo.Latest, + // which may return a pseudoversion for the latest commit. + // Query "latest" returns the newest tagged version or the newest + // prerelease version if there are no non-prereleases, or repo.Latest + // if there aren't any tagged versions. + // If we're querying "upgrade" or "patch", Query will compare the current + // version against the chosen version and will return the current version + // if it is newer. + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.CheckAllowed) + if err != nil { + // Report error but return m, to let version selection continue. + // (Reporting the error will fail the command at the next base.ExitIfErrors.) + + // Special case: if the error is for m.Version itself and m.Version has a + // replacement, then keep it and don't report the error: the fact that the + // version is invalid is likely the reason it was replaced to begin with. + var vErr *module.InvalidVersionError + if errors.As(err, &vErr) && vErr.Version == m.Version && modload.Replacement(m).Path != "" { + return m, nil + } + + // Special case: if the error is "no matching versions" then don't + // even report the error. Because Query does not consider pseudo-versions, + // it may happen that we have a pseudo-version but during -u=patch + // the query v0.0 matches no versions (not even the one we're using). + var noMatch *modload.NoMatchingVersionError + if !errors.As(err, &noMatch) { + base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) + } + return m, nil + } + + if info.Version != m.Version { + logOncef("go: %s %s => %s", m.Path, getU, info.Version) + } + return module.Version{Path: m.Path, Version: info.Version}, nil +} + +// buildListForLostUpgrade returns the build list for the module graph +// rooted at lost. Unlike mvs.BuildList, the target module (lost) is not +// treated specially. The returned build list may contain a newer version +// of lost. +// +// buildListForLostUpgrade is used after a downgrade has removed a module +// requested at a specific version. This helps us understand the requirements +// implied by each downgrade. +func buildListForLostUpgrade(lost module.Version, reqs mvs.Reqs) ([]module.Version, error) { + return mvs.BuildList(lostUpgradeRoot, &lostUpgradeReqs{Reqs: reqs, lost: lost}) +} + +var lostUpgradeRoot = module.Version{Path: "lost-upgrade-root", Version: ""} + +type lostUpgradeReqs struct { + mvs.Reqs + lost module.Version +} + +func (r *lostUpgradeReqs) Required(mod module.Version) ([]module.Version, error) { + if mod == lostUpgradeRoot { + return []module.Version{r.lost}, nil + } + return r.Reqs.Required(mod) +} diff --git a/src/cmd/go/internal/modinfo/info.go b/src/cmd/go/internal/modinfo/info.go index 07248d1a61..897be56397 100644 --- a/src/cmd/go/internal/modinfo/info.go +++ b/src/cmd/go/internal/modinfo/info.go @@ -21,6 +21,7 @@ type ModulePublic struct { Dir string `json:",omitempty"` // directory holding local copy of files, if any GoMod string `json:",omitempty"` // path to go.mod file describing module, if any GoVersion string `json:",omitempty"` // go version used in module + Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u) Error *ModuleError `json:",omitempty"` // error loading module } @@ -30,18 +31,26 @@ type ModuleError struct { func (m *ModulePublic) String() string { s := m.Path + versionString := func(mm *ModulePublic) string { + v := mm.Version + if len(mm.Retracted) == 0 { + return v + } + return v + " (retracted)" + } + if m.Version != "" { - s += " " + m.Version + s += " " + versionString(m) if m.Update != nil { - s += " [" + m.Update.Version + "]" + s += " [" + versionString(m.Update) + "]" } } if m.Replace != nil { s += " => " + m.Replace.Path if m.Replace.Version != "" { - s += " " + m.Replace.Version + s += " " + versionString(m.Replace) if m.Replace.Update != nil { - s += " [" + m.Replace.Update.Version + "]" + s += " [" + versionString(m.Replace.Update) + "]" } } } diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index a101681a1f..9ca6230500 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/hex" + "errors" "fmt" "internal/goroot" "os" @@ -58,7 +59,9 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(context.TODO(), m, true) + fromBuildList := true + listRetracted := false + return moduleInfo(context.TODO(), m, fromBuildList, listRetracted) } func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { @@ -66,13 +69,17 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { return nil } + listRetracted := false if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) + m := module.Version{Path: path[:i], Version: path[i+1:]} + fromBuildList := false + return moduleInfo(ctx, m, fromBuildList, listRetracted) } - for _, m := range BuildList() { + for _, m := range LoadedModules() { if m.Path == path { - return moduleInfo(ctx, m, true) + fromBuildList := true + return moduleInfo(ctx, m, fromBuildList, listRetracted) } } @@ -90,7 +97,7 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { return } - if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -100,11 +107,37 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { } // addVersions fills in m.Versions with the list of known versions. -func addVersions(m *modinfo.ModulePublic) { - m.Versions, _ = versions(m.Path) +// Excluded versions will be omitted. If listRetracted is false, retracted +// versions will also be omitted. +func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted bool) { + allowed := CheckAllowed + if listRetracted { + allowed = CheckExclusions + } + m.Versions, _ = versions(ctx, m.Path, allowed) } -func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { +// addRetraction fills in m.Retracted if the module was retracted by its author. +// m.Error is set if there's an error loading retraction information. +func addRetraction(ctx context.Context, m *modinfo.ModulePublic) { + if m.Version == "" { + return + } + + err := checkRetractions(ctx, module.Version{Path: m.Path, Version: m.Version}) + var rerr *retractedError + if errors.As(err, &rerr) { + if len(rerr.rationale) == 0 { + m.Retracted = []string{"retracted by module author"} + } else { + m.Retracted = rerr.rationale + } + } else if err != nil && m.Error == nil { + m.Error = &modinfo.ModuleError{Err: err.Error()} + } +} + +func moduleInfo(ctx context.Context, m module.Version, fromBuildList, listRetracted bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -126,12 +159,14 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi Version: m.Version, Indirect: fromBuildList && loaded != nil && !loaded.direct[m.Path], } - if loaded != nil { - info.GoVersion = loaded.goVersion[m.Path] + if v, ok := rawGoVersion.Load(m); ok { + info.GoVersion = v.(string) } // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { + mod := module.Version{Path: m.Path, Version: m.Version} + if m.Version != "" { if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} @@ -140,7 +175,6 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi m.Time = &q.Time } - mod := module.Version{Path: m.Path, Version: m.Version} gomod, err := modfetch.CachePath(mod, "mod") if err == nil { if info, err := os.Stat(gomod); err == nil && info.Mode().IsRegular() { @@ -151,10 +185,22 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi if err == nil { m.Dir = dir } + + if listRetracted { + addRetraction(ctx, m) + } + } + + if m.GoVersion == "" { + if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" { + m.GoVersion = summary.goVersionV[1:] + } } } if !fromBuildList { + // If this was an explicitly-versioned argument to 'go mod download' or + // 'go list -m', report the actual requested version, not its replacement. completeFromModCache(info) // Will set m.Error in vendor mode. return info } @@ -178,9 +224,11 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi // worth the cost, and we're going to overwrite the GoMod and Dir from the // replacement anyway. See https://golang.org/issue/27859. info.Replace = &modinfo.ModulePublic{ - Path: r.Path, - Version: r.Version, - GoVersion: info.GoVersion, + Path: r.Path, + Version: r.Version, + } + if v, ok := rawGoVersion.Load(m); ok { + info.Replace.GoVersion = v.(string) } if r.Version == "" { if filepath.IsAbs(r.Path) { @@ -194,7 +242,9 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi completeFromModCache(info.Replace) info.Dir = info.Replace.Dir info.GoMod = info.Replace.GoMod + info.Retracted = info.Replace.Retracted } + info.GoVersion = info.Replace.GoVersion return info } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go new file mode 100644 index 0000000000..581a1b944a --- /dev/null +++ b/src/cmd/go/internal/modload/buildlist.go @@ -0,0 +1,122 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modload + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/mvs" + "context" + "fmt" + "os" + + "golang.org/x/mod/module" +) + +// buildList is the list of modules to use for building packages. +// It is initialized by calling ImportPaths, ImportFromFiles, +// LoadALL, or LoadBuildList, each of which uses loaded.load. +// +// Ideally, exactly ONE of those functions would be called, +// and exactly once. Most of the time, that's true. +// During "go get" it may not be. TODO(rsc): Figure out if +// that restriction can be established, or else document why not. +// +var buildList []module.Version + +// LoadAllModules loads and returns the list of modules matching the "all" +// module pattern, starting with the Target module and in a deterministic +// (stable) order, without loading any packages. +// +// Modules are loaded automatically (and lazily) in ImportPaths: +// LoadAllModules need only be called if ImportPaths is not, +// typically in commands that care about modules but no particular package. +// +// The caller must not modify the returned list. +func LoadAllModules(ctx context.Context) []module.Version { + InitMod(ctx) + ReloadBuildList() + WriteGoMod() + return buildList +} + +// LoadedModules returns the list of module requirements loaded or set by a +// previous call (typically LoadAllModules or ImportPaths), starting with the +// Target module and in a deterministic (stable) order. +// +// The caller must not modify the returned list. +func LoadedModules() []module.Version { + return buildList +} + +// SetBuildList sets the module build list. +// The caller is responsible for ensuring that the list is valid. +// SetBuildList does not retain a reference to the original list. +func SetBuildList(list []module.Version) { + buildList = append([]module.Version{}, list...) +} + +// ReloadBuildList resets the state of loaded packages, then loads and returns +// the build list set in SetBuildList. +func ReloadBuildList() []module.Version { + loaded = loadFromRoots(loaderParams{ + tags: imports.Tags(), + listRoots: func() []string { return nil }, + allClosesOverTests: index.allPatternClosesOverTests(), // but doesn't matter because the root list is empty. + }) + return buildList +} + +// TidyBuildList trims the build list to the minimal requirements needed to +// retain the same versions of all packages from the preceding Load* or +// ImportPaths* call. +func TidyBuildList() { + used := map[module.Version]bool{Target: true} + for _, pkg := range loaded.pkgs { + used[pkg.mod] = true + } + + keep := []module.Version{Target} + var direct []string + for _, m := range buildList[1:] { + if used[m] { + keep = append(keep, m) + if loaded.direct[m.Path] { + direct = append(direct, m.Path) + } + } else if cfg.BuildV { + if _, ok := index.require[m]; ok { + fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) + } + } + } + + min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) + if err != nil { + base.Fatalf("go: %v", err) + } + buildList = append([]module.Version{Target}, min...) +} + +// checkMultiplePaths verifies that a given module path is used as itself +// or as a replacement for another module, but not both at the same time. +// +// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) +func checkMultiplePaths() { + firstPath := make(map[module.Version]string, len(buildList)) + for _, mod := range buildList { + src := mod + if rep := Replacement(mod); rep.Path != "" { + src = rep + } + if prev, ok := firstPath[src]; !ok { + firstPath[src] = mod.Path + } else if prev != mod.Path { + base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) + } + } + base.ExitIfErrors() +} diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index d80206b194..37f23d967f 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -432,15 +432,17 @@ verb followed by arguments. For example: require new/thing/v2 v2.3.4 exclude old/thing v1.2.3 replace bad/thing v1.4.5 => good/thing v1.4.5 + retract v1.5.6 The verbs are module, to define the module path; go, to set the expected language version; require, to require a particular module at a given version or later; - exclude, to exclude a particular module version from use; and - replace, to replace a module version with a different module version. + exclude, to exclude a particular module version from use; + replace, to replace a module version with a different module version; and + retract, to indicate a previously released version should not be used. Exclude and replace apply only in the main module's go.mod and are ignored -in dependencies. See https://research.swtch.com/vgo-mvs for details. +in dependencies. See https://golang.org/ref/mod for details. The leading verb can be factored out of adjacent lines to create a block, like in Go imports: diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 5c51a79124..10b1e7f4b8 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -26,6 +26,8 @@ import ( "golang.org/x/mod/semver" ) +var errImportMissing = errors.New("import missing") + type ImportMissingError struct { Path string Module module.Version @@ -48,6 +50,11 @@ func (e *ImportMissingError) Error() string { } return "cannot find module providing package " + e.Path } + + if e.newMissingVersion != "" { + return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion) + } + return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path) } @@ -100,18 +107,39 @@ func (e *AmbiguousImportError) Error() string { var _ load.ImportPathError = &AmbiguousImportError{} -// Import finds the module and directory in the build list -// containing the package with the given import path. -// The answer must be unique: Import returns an error -// if multiple modules attempt to provide the same package. -// Import can return a module with an empty m.Path, for packages in the standard library. -// Import can return an empty directory string, for fake packages like "C" and "unsafe". +type invalidImportError struct { + importPath string + err error +} + +func (e *invalidImportError) ImportPath() string { + return e.importPath +} + +func (e *invalidImportError) Error() string { + return e.err.Error() +} + +func (e *invalidImportError) Unwrap() error { + return e.err +} + +var _ load.ImportPathError = &invalidImportError{} + +// importFromBuildList finds the module and directory in the build list +// containing the package with the given import path. The answer must be unique: +// importFromBuildList returns an error if multiple modules attempt to provide +// the same package. +// +// importFromBuildList can return a module with an empty m.Path, for packages in +// the standard library. +// +// importFromBuildList can return an empty directory string, for fake packages +// like "C" and "unsafe". // // If the package cannot be found in the current build list, -// Import returns an ImportMissingError as the error. -// If Import can identify a module that could be added to supply the package, -// the ImportMissingError records that module. -func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { +// importFromBuildList returns errImportMissing as the error. +func importFromBuildList(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -190,29 +218,25 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods} } - // Look up module containing the package, for addition to the build list. - // Goal is to determine the module, download it to dir, and return m, dir, ErrMissing. - if cfg.BuildMod == "readonly" { - var queryErr error - if !pathIsStd { - if cfg.BuildModReason == "" { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) - } else { - queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) - } - } - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr} - } + return module.Version{}, "", errImportMissing +} + +// queryImport attempts to locate a module that can be added to the current +// build list to provide the package with the given import path. +func queryImport(ctx context.Context, path string) (module.Version, error) { + pathIsStd := search.IsStandardImportPath(path) + if modRoot == "" && !allowMissingModuleImports { - return module.Version{}, "", &ImportMissingError{ + return module.Version{}, &ImportMissingError{ Path: path, QueryErr: errors.New("working directory is not part of a module"), } } // Not on build list. - // To avoid spurious remote fetches, next try the latest replacement for each module. - // (golang.org/issue/26241) + // To avoid spurious remote fetches, next try the latest replacement for each + // module (golang.org/issue/26241). This should give a useful message + // in -mod=readonly, and it will allow us to add a requirement with -mod=mod. if modFile != nil { latest := map[string]string{} // path -> version for _, r := range modFile.Replace { @@ -226,7 +250,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err } } - mods = make([]module.Version, 0, len(latest)) + mods := make([]module.Version, 0, len(latest)) for p, v := range latest { // If the replacement didn't specify a version, synthesize a // pseudo-version with an appropriate major version and a timestamp below @@ -252,19 +276,19 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. - return module.Version{}, "", err + return module.Version{}, err } if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, "", err + return m, err } else if ok { - return m, "", &ImportMissingError{Path: path, Module: m} + return m, nil } } if len(mods) > 0 && module.CheckPath(path) != nil { // The package path is not valid to fetch remotely, // so it can only exist if in a replaced module, // and we know from the above loop that it is not. - return module.Version{}, "", &PackageNotInModuleError{ + return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], Query: "latest", Pattern: path, @@ -273,6 +297,11 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err } } + // Before any further lookup, check that the path is valid. + if err := module.CheckImportPath(path); err != nil { + return module.Version{}, &invalidImportError{importPath: path, err: err} + } + if pathIsStd { // This package isn't in the standard library, isn't in any module already // in the build list, and isn't in any other module that the user has @@ -281,25 +310,39 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // QueryPackage cannot possibly find a module containing this package. // // Instead of trying QueryPackage, report an ImportMissingError immediately. - return module.Version{}, "", &ImportMissingError{Path: path} + return module.Version{}, &ImportMissingError{Path: path} + } + + if cfg.BuildMod == "readonly" { + var queryErr error + if cfg.BuildModExplicit { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod) + } else if cfg.BuildModReason != "" { + queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) + } + return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr} } + // Look up module containing the package, for addition to the build list. + // Goal is to determine the module, download it to dir, + // and return m, dir, ImpportMissingError. fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(ctx, path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", CheckAllowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever // low-level error QueryPackage produced. - return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: err} + return module.Version{}, &ImportMissingError{Path: path, QueryErr: err} } else { - return module.Version{}, "", err + return module.Version{}, err } } - m = candidates[0].Mod - newMissingVersion := "" - for _, c := range candidates { + + candidate0MissingVersion := "" + for i, c := range candidates { cm := c.Mod + canAdd := true for _, bm := range buildList { if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 { // QueryPackage proposed that we add module cm to provide the package, @@ -310,13 +353,22 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // version (e.g., v1.0.0) of a module, but we have a newer version // of the same module in the build list (e.g., v1.0.1-beta), and // the package is not present there. - m = cm - newMissingVersion = bm.Version + canAdd = false + if i == 0 { + candidate0MissingVersion = bm.Version + } break } } + if canAdd { + return cm, nil + } + } + return module.Version{}, &ImportMissingError{ + Path: path, + Module: candidates[0].Mod, + newMissingVersion: candidate0MissingVersion, } - return m, "", &ImportMissingError{Path: path, Module: m, newMissingVersion: newMissingVersion} } // maybeInModule reports whether, syntactically, diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index 47ce89a084..22d5b82e21 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -10,15 +10,20 @@ import ( "regexp" "strings" "testing" + + "golang.org/x/mod/module" ) var importTests = []struct { path string + m module.Version err string }{ { path: "golang.org/x/net/context", - err: "missing module for import: golang.org/x/net@.* provides golang.org/x/net/context", + m: module.Version{ + Path: "golang.org/x/net", + }, }, { path: "golang.org/x/net", @@ -26,15 +31,23 @@ var importTests = []struct { }, { path: "golang.org/x/text", - err: "missing module for import: golang.org/x/text@.* provides golang.org/x/text", + m: module.Version{ + Path: "golang.org/x/text", + }, }, { path: "github.com/rsc/quote/buggy", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote/buggy", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "github.com/rsc/quote", - err: "missing module for import: github.com/rsc/quote@v1.5.2 provides github.com/rsc/quote", + m: module.Version{ + Path: "github.com/rsc/quote", + Version: "v1.5.2", + }, }, { path: "golang.org/x/foo/bar", @@ -42,7 +55,7 @@ var importTests = []struct { }, } -func TestImport(t *testing.T) { +func TestQueryImport(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") defer func(old bool) { @@ -55,12 +68,23 @@ func TestImport(t *testing.T) { for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(ctx, tt.path) - if err == nil { - t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) + m, err := queryImport(ctx, tt.path) + + if tt.err == "" { + if err != nil { + t.Fatalf("queryImport(_, %q): %v", tt.path, err) + } + } else { + if err == nil { + t.Fatalf("queryImport(_, %q) = %v, nil; expected error", tt.path, m) + } + if !regexp.MustCompile(tt.err).MatchString(err.Error()) { + t.Fatalf("queryImport(_, %q): error %q, want error matching %#q", tt.path, err, tt.err) + } } - if !regexp.MustCompile(tt.err).MatchString(err.Error()) { - t.Fatalf("Import(%q): error %q, want error matching %#q", tt.path, err, tt.err) + + if m.Path != tt.m.Path || (tt.m.Version != "" && m.Version != tt.m.Version) { + t.Errorf("queryImport(_, %q) = %v, _; want %v", tt.path, m, tt.m) } }) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 93027c44c4..1f50dcb11c 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -368,13 +368,9 @@ func InitMod(ctx context.Context) { modFile = f index = indexModFile(data, f, fixed) - if len(f.Syntax.Stmt) == 0 || f.Module == nil { - // Empty mod file. Must add module path. - path, err := findModulePath(modRoot) - if err != nil { - base.Fatalf("go: %v", err) - } - f.AddModuleStmt(path) + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod.\n\tRun 'go mod edit -module=example.com/mod' to specify the module path.") } if len(f.Syntax.Stmt) == 1 && f.Module != nil { @@ -383,14 +379,61 @@ func InitMod(ctx context.Context) { legacyModInit() } - modFileToBuildList() + if err := checkModulePathLax(f.Module.Mod.Path); err != nil { + base.Fatalf("go: %v", err) + } + setDefaultBuildMod() + modFileToBuildList() if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() } } +// checkModulePathLax checks that the path meets some minimum requirements +// to avoid confusing users or the module cache. The requirements are weaker +// than those of module.CheckPath to allow room for weakening module path +// requirements in the future, but strong enough to help users avoid significant +// problems. +func checkModulePathLax(p string) error { + // TODO(matloob): Replace calls of this function in this CL with calls + // to module.CheckImportPath once it's been laxened, if it becomes laxened. + // See golang.org/issue/29101 for a discussion about whether to make CheckImportPath + // more lax or more strict. + + errorf := func(format string, args ...interface{}) error { + return fmt.Errorf("invalid module path %q: %s", p, fmt.Sprintf(format, args...)) + } + + // Disallow shell characters " ' * < > ? ` | to avoid triggering bugs + // with file systems and subcommands. Disallow file path separators : and \ + // because path separators other than / will confuse the module cache. + // See fileNameOK in golang.org/x/mod/module/module.go. + shellChars := "`" + `\"'*<>?|` + fsChars := `\:` + if i := strings.IndexAny(p, shellChars); i >= 0 { + return errorf("contains disallowed shell character %q", p[i]) + } + if i := strings.IndexAny(p, fsChars); i >= 0 { + return errorf("contains disallowed path separator character %q", p[i]) + } + + // Ensure path.IsAbs and build.IsLocalImport are false, and that the path is + // invariant under path.Clean, also to avoid confusing the module cache. + if path.IsAbs(p) { + return errorf("is an absolute path") + } + if build.IsLocalImport(p) { + return errorf("is a local import path") + } + if path.Clean(p) != p { + return errorf("is not clean") + } + + return nil +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, @@ -459,7 +502,15 @@ func modFileToBuildList() { list := []module.Version{Target} for _, r := range modFile.Require { - list = append(list, r.Mod) + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + } else { + list = append(list, r.Mod) + } } buildList = list } @@ -467,46 +518,49 @@ func modFileToBuildList() { // setDefaultBuildMod sets a default value for cfg.BuildMod // if it is currently empty. func setDefaultBuildMod() { - if cfg.BuildMod != "" { + if cfg.BuildModExplicit { // Don't override an explicit '-mod=' argument. return } - cfg.BuildMod = "mod" + if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // Don't set -mod implicitly for commands whose purpose is to - // manipulate the build list. + // 'get' and 'go mod' commands may update go.mod automatically. + // TODO(jayconrod): should this narrower? Should 'go mod download' or + // 'go mod graph' update go.mod by default? + cfg.BuildMod = "mod" return } if modRoot == "" { + cfg.BuildMod = "mod" return } if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { modGo := "unspecified" - if index.goVersion != "" { - if semver.Compare("v"+index.goVersion, "v1.14") >= 0 { + if index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { // The Go version is at least 1.14, and a vendor directory exists. // Set -mod=vendor by default. cfg.BuildMod = "vendor" cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." return } else { - modGo = index.goVersion + modGo = index.goVersionV[1:] } } - // Since a vendor directory exists, we have a non-trivial reason for - // choosing -mod=mod, although it probably won't be used for anything. - // Record the reason anyway for consistency. - // It may be overridden if we switch to mod=readonly below. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) } p := ModFilePath() if fi, err := os.Stat(p); err == nil && !hasWritePerm(p, fi) { cfg.BuildMod = "readonly" cfg.BuildModReason = "go.mod file is read-only." + return } + cfg.BuildMod = "mod" } func legacyModInit() { @@ -674,16 +728,35 @@ func findModulePath(dir string) (string, error) { } // Look for path in GOPATH. + var badPathErr error for _, gpdir := range filepath.SplitList(cfg.BuildContext.GOPATH) { if gpdir == "" { continue } if rel := search.InDir(dir, filepath.Join(gpdir, "src")); rel != "" && rel != "." { - return filepath.ToSlash(rel), nil + path := filepath.ToSlash(rel) + // TODO(matloob): replace this with module.CheckImportPath + // once it's been laxened. + // Only checkModulePathLax here. There are some unpublishable + // module names that are compatible with checkModulePathLax + // but they already work in GOPATH so don't break users + // trying to do a build with modules. gorelease will alert users + // publishing their modules to fix their paths. + if err := checkModulePathLax(path); err != nil { + badPathErr = err + break + } + return path, nil } } - msg := `cannot determine module path for source directory %s (outside GOPATH, module path must be specified) + reason := "outside GOPATH, module path must be specified" + if badPathErr != nil { + // return a different error message if the module was in GOPATH, but + // the module path determined above would be an invalid path. + reason = fmt.Sprintf("bad module path inferred from directory in GOPATH: %v", badPathErr) + } + msg := `cannot determine module path for source directory %s (%s) Example usage: 'go mod init example.com/m' to initialize a v0 or v1 module @@ -691,7 +764,7 @@ Example usage: Run 'go help mod init' for more information. ` - return "", fmt.Errorf(msg, dir) + return "", fmt.Errorf(msg, dir, reason) } var ( @@ -787,19 +860,16 @@ func WriteGoMod() { // prefer to report a dirty go.mod over a dirty go.sum if cfg.BuildModReason != "" { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly\n\t(%s)", cfg.BuildModReason) - } else { + } else if cfg.BuildModExplicit { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") } } - // Always update go.sum, even if we didn't change go.mod: we may have - // downloaded modules that we didn't have before. - modfetch.WriteGoSum(keepSums()) - if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before // (but not necessarily the same exact bytes). - // Ignore any intervening edits. + // Don't write go.mod, but write go.sum in case we added or trimmed sums. + modfetch.WriteGoSum(keepSums(true)) return } @@ -810,6 +880,9 @@ func WriteGoMod() { defer func() { // At this point we have determined to make the go.mod file on disk equal to new. index = indexModFile(new, modFile, false) + + // Update go.sum after releasing the side lock and refreshing the index. + modfetch.WriteGoSum(keepSums(true)) }() // Make a best-effort attempt to acquire the side lock, only to exclude @@ -850,7 +923,10 @@ func WriteGoMod() { // the last load function like ImportPaths, LoadALL, etc.). It also contains // entries for go.mod files needed for MVS (the version of these entries // ends with "/go.mod"). -func keepSums() map[module.Version]bool { +// +// If addDirect is true, the set also includes sums for modules directly +// required by go.mod, as represented by the index, with replacements applied. +func keepSums(addDirect bool) map[module.Version]bool { // Walk the module graph and keep sums needed by MVS. modkey := func(m module.Version) module.Version { return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} @@ -862,9 +938,6 @@ func keepSums() map[module.Version]bool { walk = func(m module.Version) { // If we build using a replacement module, keep the sum for the replacement, // since that's the code we'll actually use during a build. - // - // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the - // sums for both sets of transitive requirements. r := Replacement(m) if r.Path == "" { keep[modkey(m)] = true @@ -894,9 +967,27 @@ func keepSums() map[module.Version]bool { } } + // Add entries for modules directly required by go.mod. + if addDirect { + for m := range index.require { + var kept module.Version + if r := Replacement(m); r.Path != "" { + kept = r + } else { + kept = m + } + keep[kept] = true + keep[module.Version{Path: kept.Path, Version: kept.Version + "/go.mod"}] = true + } + } + return keep } func TrimGoSum() { - modfetch.TrimGoSum(keepSums()) + // Don't retain sums for direct requirements in go.mod. When TrimGoSum is + // called, go.mod has not been updated, and it may contain requirements on + // modules deleted from the build list. + addDirect := false + modfetch.TrimGoSum(keepSums(addDirect)) } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 7bf4e86c8d..3491f941cd 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -20,12 +20,12 @@ import ( "golang.org/x/mod/module" ) -func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(ctx, args, listVersions) +func ListModules(ctx context.Context, args []string, listU, listVersions, listRetracted bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions, listRetracted) type token struct{} sem := make(chan token, runtime.GOMAXPROCS(0)) - if listU || listVersions { + if listU || listVersions || listRetracted { for _, m := range mods { add := func(m *modinfo.ModulePublic) { sem <- token{} @@ -34,7 +34,10 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ addUpdate(ctx, m) } if listVersions { - addVersions(m) + addVersions(ctx, m, listRetracted) + } + if listRetracted || listU { + addRetraction(ctx, m) } <-sem }() @@ -54,10 +57,10 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ return mods } -func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList(ctx) +func listModules(ctx context.Context, args []string, listVersions, listRetracted bool) []*modinfo.ModulePublic { + LoadAllModules(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true, listRetracted)} } var mods []*modinfo.ModulePublic @@ -83,7 +86,13 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(ctx, path, vers, current, nil) + allowed := CheckAllowed + if IsRevisionQuery(vers) || listRetracted { + // Allow excluded and retracted versions if the user asked for a + // specific revision or used 'go list -retracted'. + allowed = nil + } + info, err := Query(ctx, path, vers, current, allowed) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +101,8 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) continue } @@ -117,7 +127,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(ctx, m, true)) + mods = append(mods, moduleInfo(ctx, m, true, listRetracted)) } } } @@ -129,7 +139,8 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Instead, resolve the module, even if it isn't an existing dependency. info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) + mod := moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false, listRetracted) + mods = append(mods, mod) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 686d491219..1664d8c5be 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -4,6 +4,95 @@ package modload +// This file contains the module-mode package loader, as well as some accessory +// functions pertaining to the package import graph. +// +// There are several exported entry points into package loading (such as +// ImportPathsQuiet and LoadALL), but they are all implemented in terms of +// loadFromRoots, which itself manipulates an instance of the loader struct. +// +// Although most of the loading state is maintained in the loader struct, +// one key piece - the build list - is a global, so that it can be modified +// separate from the loading operation, such as during "go get" +// upgrades/downgrades or in "go mod" operations. +// TODO(#40775): It might be nice to make the loader take and return +// a buildList rather than hard-coding use of the global. +// +// Loading is an iterative process. On each iteration, we try to load the +// requested packages and their transitive imports, then try to resolve modules +// for any imported packages that are still missing. +// +// The first step of each iteration identifies a set of “root” packages. +// Normally the root packages are exactly those matching the named pattern +// arguments. However, for the "all" meta-pattern and related functions +// (LoadALL, LoadVendor), the final set of packages is computed from the package +// import graph, and therefore cannot be an initial input to loading that graph. +// Instead, the root packages for the "all" pattern are those contained in the +// main module, and allPatternIsRoot parameter to the loader instructs it to +// dynamically expand those roots to the full "all" pattern as loading +// progresses. +// +// The pkgInAll flag on each loadPkg instance tracks whether that +// package is known to match the "all" meta-pattern. +// A package matches the "all" pattern if: +// - it is in the main module, or +// - it is imported by any test in the main module, or +// - it is imported by another package in "all", or +// - the main module specifies a go version ≤ 1.15, and the package is imported +// by a *test of* another package in "all". +// +// When we implement lazy loading, we will record the modules providing packages +// in "all" even when we are only loading individual packages, so we set the +// pkgInAll flag regardless of the whether the "all" pattern is a root. +// (This is necessary to maintain the “import invariant” described in +// https://golang.org/design/36460-lazy-module-loading.) +// +// Because "go mod vendor" prunes out the tests of vendored packages, the +// behavior of the "all" pattern with -mod=vendor in Go 1.11–1.15 is the same +// as the "all" pattern (regardless of the -mod flag) in 1.16+. +// The allClosesOverTests parameter to the loader indicates whether the "all" +// pattern should close over tests (as in Go 1.11–1.15) or stop at only those +// packages transitively imported by the packages and tests in the main module +// ("all" in Go 1.16+ and "go mod vendor" in Go 1.11+). +// +// Note that it is possible for a loaded package NOT to be in "all" even when we +// are loading the "all" pattern. For example, packages that are transitive +// dependencies of other roots named on the command line must be loaded, but are +// not in "all". (The mod_notall test illustrates this behavior.) +// Similarly, if the LoadTests flag is set but the "all" pattern does not close +// over test dependencies, then when we load the test of a package that is in +// "all" but outside the main module, the dependencies of that test will not +// necessarily themselves be in "all". That configuration does not arise in Go +// 1.11–1.15, but it will be possible with lazy loading in Go 1.16+. +// +// Loading proceeds from the roots, using a parallel work-queue with a limit on +// the amount of active work (to avoid saturating disks, CPU cores, and/or +// network connections). Each package is added to the queue the first time it is +// imported by another package. When we have finished identifying the imports of +// a package, we add the test for that package if it is needed. A test may be +// needed if: +// - the package matches a root pattern and tests of the roots were requested, or +// - the package is in the main module and the "all" pattern is requested +// (because the "all" pattern includes the dependencies of tests in the main +// module), or +// - the package is in "all" and the definition of "all" we are using includes +// dependencies of tests (as is the case in Go ≤1.15). +// +// After all available packages have been loaded, we examine the results to +// identify any requested or imported packages that are still missing, and if +// so, which modules we could add to the module graph in order to make the +// missing packages available. We add those to the module graph and iterate, +// until either all packages resolve successfully or we cannot identify any +// module that would resolve any remaining missing package. +// +// If the main module is “tidy” (that is, if "go mod tidy" is a no-op for it) +// and all requested packages are in "all", then loading completes in a single +// iteration. +// TODO(bcmills): We should also be able to load in a single iteration if the +// requested packages all come from modules that are themselves tidy, regardless +// of whether those packages are in "all". Today, that requires two iterations +// if those packages are not found in existing dependencies of the main module. + import ( "bytes" "context" @@ -14,8 +103,12 @@ import ( "path" pathpkg "path" "path/filepath" + "reflect" + "runtime" "sort" "strings" + "sync" + "sync/atomic" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -25,28 +118,12 @@ import ( "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" - "cmd/go/internal/trace" "golang.org/x/mod/module" ) -// buildList is the list of modules to use for building packages. -// It is initialized by calling ImportPaths, ImportFromFiles, -// LoadALL, or LoadBuildList, each of which uses loaded.load. -// -// Ideally, exactly ONE of those functions would be called, -// and exactly once. Most of the time, that's true. -// During "go get" it may not be. TODO(rsc): Figure out if -// that restriction can be established, or else document why not. -// -var buildList []module.Version - // loaded is the most recently-used package loader. // It holds details about individual packages. -// -// Note that loaded.buildList is only valid during a load operation; -// afterward, it is copied back into the global buildList, -// which should be used instead. var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), @@ -63,7 +140,18 @@ func ImportPaths(ctx context.Context, patterns []string) []*search.Match { // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { - updateMatches := func(matches []*search.Match, iterating bool) { + InitMod(ctx) + + allPatternIsRoot := false + var matches []*search.Match + for _, pattern := range search.CleanPatterns(patterns) { + matches = append(matches, search.NewMatch(pattern)) + if pattern == "all" { + allPatternIsRoot = true + } + } + + updateMatches := func(ld *loader) { for _, m := range matches { switch { case m.IsLocal(): @@ -90,7 +178,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo // indicates that. ModRoot() - if !iterating { + if ld != nil { m.AddError(err) } continue @@ -103,19 +191,18 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, tags, includeStd, buildList) case m.Pattern() == "all": - loaded.testAll = true - if iterating { - // Enumerate the packages in the main module. - // We'll load the dependencies as we find them. + if ld == nil { + // The initial roots are the packages in the main module. + // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". - m.Pkgs = loaded.computePatternAll(m.Pkgs) + m.Pkgs = ld.computePatternAll() } case m.Pattern() == "std" || m.Pattern() == "cmd": @@ -129,51 +216,28 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo } } - InitMod(ctx) - - var matches []*search.Match - for _, pattern := range search.CleanPatterns(patterns) { - matches = append(matches, search.NewMatch(pattern)) - } + loaded = loadFromRoots(loaderParams{ + tags: tags, + allPatternIsRoot: allPatternIsRoot, + allClosesOverTests: index.allPatternClosesOverTests(), - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - updateMatches(matches, true) - for _, m := range matches { - roots = append(roots, m.Pkgs...) - } - return roots + listRoots: func() (roots []string) { + updateMatches(nil) + for _, m := range matches { + roots = append(roots, m.Pkgs...) + } + return roots + }, }) // One last pass to finalize wildcards. - updateMatches(matches, false) + updateMatches(loaded) checkMultiplePaths() WriteGoMod() return matches } -// checkMultiplePaths verifies that a given module path is used as itself -// or as a replacement for another module, but not both at the same time. -// -// (See https://golang.org/issue/26607 and https://golang.org/issue/34650.) -func checkMultiplePaths() { - firstPath := make(map[module.Version]string, len(buildList)) - for _, mod := range buildList { - src := mod - if rep := Replacement(mod); rep.Path != "" { - src = rep - } - if prev, ok := firstPath[src]; !ok { - firstPath[src] = mod.Path - } else if prev != mod.Path { - base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path) - } - } - base.ExitIfErrors() -} - // matchLocalDirs is like m.MatchDirs, but tries to avoid scanning directories // outside of the standard library and active modules. func matchLocalDirs(m *search.Match) { @@ -310,7 +374,7 @@ var ( // pathInModuleCache returns the import path of the directory dir, // if dir is in the module cache copy of a module in our build list. func pathInModuleCache(dir string) string { - for _, m := range buildList[1:] { + tryMod := func(m module.Version) (string, bool) { var root string var err error if repl := Replacement(m); repl.Path != "" && repl.Version == "" { @@ -324,13 +388,26 @@ func pathInModuleCache(dir string) string { root, err = modfetch.DownloadDir(m) } if err != nil { - continue + return "", false } - if sub := search.InDir(dir, root); sub != "" { - sub = filepath.ToSlash(sub) - if !strings.Contains(sub, "/vendor/") && !strings.HasPrefix(sub, "vendor/") && !strings.Contains(sub, "@") { - return path.Join(m.Path, filepath.ToSlash(sub)) - } + + sub := search.InDir(dir, root) + if sub == "" { + return "", false + } + sub = filepath.ToSlash(sub) + if strings.Contains(sub, "/vendor/") || strings.HasPrefix(sub, "vendor/") || strings.Contains(sub, "@") { + return "", false + } + + return path.Join(m.Path, filepath.ToSlash(sub)), true + } + + for _, m := range buildList[1:] { + if importPath, ok := tryMod(m); ok { + // checkMultiplePaths ensures that a module can be used for at most one + // requirement, so this must be it. + return importPath } } return "" @@ -347,12 +424,14 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { base.Fatalf("go: %v", err) } - loaded = newLoader(tags) - loaded.load(func() []string { - var roots []string - roots = append(roots, imports...) - roots = append(roots, testImports...) - return roots + loaded = loadFromRoots(loaderParams{ + tags: tags, + listRoots: func() (roots []string) { + roots = append(roots, imports...) + roots = append(roots, testImports...) + return roots + }, + allClosesOverTests: index.allPatternClosesOverTests(), }) WriteGoMod() } @@ -383,34 +462,19 @@ func DirImportPath(dir string) string { return "." } -// LoadBuildList loads and returns the build list from go.mod. -// The loading of the build list happens automatically in ImportPaths: -// LoadBuildList need only be called if ImportPaths is not -// (typically in commands that care about the module but -// no particular package). -func LoadBuildList(ctx context.Context) []module.Version { - ctx, span := trace.StartSpan(ctx, "LoadBuildList") - defer span.Done() - InitMod(ctx) - ReloadBuildList() - WriteGoMod() - return buildList -} - -func ReloadBuildList() []module.Version { - loaded = newLoader(imports.Tags()) - loaded.load(func() []string { return nil }) - return buildList -} - // LoadALL returns the set of all packages in the current module // and their dependencies in any other modules, without filtering // due to build tags, except "+build ignore". // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. +// +// In modules that specify "go 1.16" or higher, ALL follows only one layer of +// test dependencies. In "go 1.15" or lower, ALL follows the imports of tests of +// dependencies of tests. func LoadALL(ctx context.Context) []string { - return loadAll(ctx, true) + InitMod(ctx) + return loadAll(ctx, index.allPatternClosesOverTests()) } // LoadVendor is like LoadALL but only follows test dependencies @@ -418,20 +482,20 @@ func LoadALL(ctx context.Context) []string { // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. func LoadVendor(ctx context.Context) []string { - return loadAll(ctx, false) -} - -func loadAll(ctx context.Context, testAll bool) []string { InitMod(ctx) + // 'go mod vendor' has never followed test dependencies since Go 1.11. + const closeOverTests = false + return loadAll(ctx, closeOverTests) +} - loaded = newLoader(imports.AnyTags()) - loaded.isALL = true - loaded.testAll = testAll - if !testAll { - loaded.testRoots = true - } - all := TargetPackages(ctx, "...") - loaded.load(func() []string { return all.Pkgs }) +func loadAll(ctx context.Context, closeOverTests bool) []string { + inTarget := TargetPackages(ctx, "...") + loaded = loadFromRoots(loaderParams{ + tags: imports.AnyTags(), + listRoots: func() []string { return inTarget.Pkgs }, + allPatternIsRoot: true, + allClosesOverTests: closeOverTests, + }) checkMultiplePaths() WriteGoMod() @@ -443,7 +507,7 @@ func loadAll(ctx context.Context, testAll bool) []string { } paths = append(paths, pkg.path) } - for _, err := range all.Errs { + for _, err := range inTarget.Errs { base.Errorf("%v", err) } base.ExitIfErrors() @@ -463,52 +527,6 @@ func TargetPackages(ctx context.Context, pattern string) *search.Match { return m } -// BuildList returns the module build list, -// typically constructed by a previous call to -// LoadBuildList or ImportPaths. -// The caller must not modify the returned list. -func BuildList() []module.Version { - return buildList -} - -// SetBuildList sets the module build list. -// The caller is responsible for ensuring that the list is valid. -// SetBuildList does not retain a reference to the original list. -func SetBuildList(list []module.Version) { - buildList = append([]module.Version{}, list...) -} - -// TidyBuildList trims the build list to the minimal requirements needed to -// retain the same versions of all packages from the preceding Load* or -// ImportPaths* call. -func TidyBuildList() { - used := map[module.Version]bool{Target: true} - for _, pkg := range loaded.pkgs { - used[pkg.mod] = true - } - - keep := []module.Version{Target} - var direct []string - for _, m := range buildList[1:] { - if used[m] { - keep = append(keep, m) - if loaded.direct[m.Path] { - direct = append(direct, m.Path) - } - } else if cfg.BuildV { - if _, ok := index.require[m]; ok { - fmt.Fprintf(os.Stderr, "unused %s\n", m.Path) - } - } - } - - min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep}) - if err != nil { - base.Fatalf("go: %v", err) - } - buildList = append([]module.Version{Target}, min...) -} - // ImportMap returns the actual package import path // for an import path found in source code. // If the given import path does not appear in the source code @@ -563,12 +581,6 @@ func PackageImports(path string) (imports, testImports []string) { return imports, testImports } -// ModuleUsedDirectly reports whether the main module directly imports -// some package in the module with the given path. -func ModuleUsedDirectly(path string) bool { - return loaded.direct[path] -} - // Lookup returns the source directory, import path, and any loading error for // the package at path as imported from the package in parentDir. // Lookup requires that one of the Load functions in this package has already @@ -604,76 +616,148 @@ func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath str // the required packages for a particular build, // checking that the packages are available in the module set, // and updating the module set if needed. -// Loading is an iterative process: try to load all the needed packages, -// but if imports are missing, try to resolve those imports, and repeat. -// -// Although most of the loading state is maintained in the loader struct, -// one key piece - the build list - is a global, so that it can be modified -// separate from the loading operation, such as during "go get" -// upgrades/downgrades or in "go mod" operations. -// TODO(rsc): It might be nice to make the loader take and return -// a buildList rather than hard-coding use of the global. type loader struct { - tags map[string]bool // tags for scanDir - testRoots bool // include tests for roots - isALL bool // created with LoadALL - testAll bool // include tests for all packages - forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree + loaderParams + + work *par.Queue // reset on each iteration roots []*loadPkg - pkgs []*loadPkg - work *par.Work // current work queue - pkgCache *par.Cache // map from string to *loadPkg + pkgCache *par.Cache // package path (string) → *loadPkg + pkgs []*loadPkg // transitive closure of loaded packages and tests; populated in buildStacks // computed at end of iterations - direct map[string]bool // imported directly by main module - goVersion map[string]string // go version recorded in each module + direct map[string]bool // imported directly by main module +} + +type loaderParams struct { + tags map[string]bool // tags for scanDir + listRoots func() []string + allPatternIsRoot bool // Is the "all" pattern an additional root? + allClosesOverTests bool // Does the "all" pattern include the transitive closure of tests of packages in "all"? } // LoadTests controls whether the loaders load tests of the root packages. var LoadTests bool -func newLoader(tags map[string]bool) *loader { - ld := new(loader) - ld.tags = tags - ld.testRoots = LoadTests - - // Inside the "std" and "cmd" modules, we prefer to use the vendor directory - // unless the command explicitly changes the module graph. - if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) { - ld.forceStdVendor = true +func (ld *loader) reset() { + select { + case <-ld.work.Idle(): + default: + panic("loader.reset when not idle") } - return ld -} - -func (ld *loader) reset() { ld.roots = nil - ld.pkgs = nil - ld.work = new(par.Work) ld.pkgCache = new(par.Cache) + ld.pkgs = nil } // A loadPkg records information about a single loaded package. type loadPkg struct { - path string // import path + // Populated at construction time: + path string // import path + testOf *loadPkg + + // Populated at construction time and updated by (*loader).applyPkgFlags: + flags atomicLoadPkgFlags + + // Populated by (*loader).load: mod module.Version // module providing package dir string // directory containing source code - imports []*loadPkg // packages imported by this one err error // error loading package - stack *loadPkg // package importing this one in minimal import stack for this pkg - test *loadPkg // package with test imports, if we need test - testOf *loadPkg - testImports []string // test-only imports, saved for use by pkg.test. + imports []*loadPkg // packages imported by this one + testImports []string // test-only imports, saved for use by pkg.test. + inStd bool + + // Populated by (*loader).pkgTest: + testOnce sync.Once + test *loadPkg + + // Populated by postprocessing in (*loader).buildStacks: + stack *loadPkg // package importing this one in minimal import stack for this pkg +} + +// loadPkgFlags is a set of flags tracking metadata about a package. +type loadPkgFlags int8 + +const ( + // pkgInAll indicates that the package is in the "all" package pattern, + // regardless of whether we are loading the "all" package pattern. + // + // When the pkgInAll flag and pkgImportsLoaded flags are both set, the caller + // who set the last of those flags must propagate the pkgInAll marking to all + // of the imports of the marked package. + // + // A test is marked with pkgInAll if that test would promote the packages it + // imports to be in "all" (such as when the test is itself within the main + // module, or when ld.allClosesOverTests is true). + pkgInAll loadPkgFlags = 1 << iota + + // pkgIsRoot indicates that the package matches one of the root package + // patterns requested by the caller. + // + // If LoadTests is set, then when pkgIsRoot and pkgImportsLoaded are both set, + // the caller who set the last of those flags must populate a test for the + // package (in the pkg.test field). + // + // If the "all" pattern is included as a root, then non-test packages in "all" + // are also roots (and must be marked pkgIsRoot). + pkgIsRoot + + // pkgImportsLoaded indicates that the imports and testImports fields of a + // loadPkg have been populated. + pkgImportsLoaded +) + +// has reports whether all of the flags in cond are set in f. +func (f loadPkgFlags) has(cond loadPkgFlags) bool { + return f&cond == cond +} + +// An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be +// added atomically. +type atomicLoadPkgFlags struct { + bits int32 +} + +// update sets the given flags in af (in addition to any flags already set). +// +// update returns the previous flag state so that the caller may determine which +// flags were newly-set. +func (af *atomicLoadPkgFlags) update(flags loadPkgFlags) (old loadPkgFlags) { + for { + old := atomic.LoadInt32(&af.bits) + new := old | int32(flags) + if new == old || atomic.CompareAndSwapInt32(&af.bits, old, new) { + return loadPkgFlags(old) + } + } +} + +// has reports whether all of the flags in cond are set in af. +func (af *atomicLoadPkgFlags) has(cond loadPkgFlags) bool { + return loadPkgFlags(atomic.LoadInt32(&af.bits))&cond == cond +} + +// isTest reports whether pkg is a test of another package. +func (pkg *loadPkg) isTest() bool { + return pkg.testOf != nil } var errMissing = errors.New("cannot find package") -// load attempts to load the build graph needed to process a set of root packages. -// The set of root packages is defined by the addRoots function, -// which must call add(path) with the import path of each root package. -func (ld *loader) load(roots func() []string) { +// loadFromRoots attempts to load the build graph needed to process a set of +// root packages and their dependencies. +// +// The set of root packages is returned by the params.listRoots function, and +// expanded to the full set of packages by tracing imports (and possibly tests) +// as needed. +func loadFromRoots(params loaderParams) *loader { + ld := &loader{ + loaderParams: params, + work: par.NewQueue(runtime.GOMAXPROCS(0)), + } + var err error reqs := Reqs() buildList, err = mvs.BuildList(Target, reqs) @@ -681,47 +765,34 @@ func (ld *loader) load(roots func() []string) { base.Fatalf("go: %v", err) } - added := make(map[string]bool) + addedModuleFor := make(map[string]bool) for { ld.reset() - if roots != nil { - // Note: the returned roots can change on each iteration, - // since the expansion of package patterns depends on the - // build list we're using. - for _, path := range roots() { - ld.work.Add(ld.pkg(path, true)) + + // Load the root packages and their imports. + // Note: the returned roots can change on each iteration, + // since the expansion of package patterns depends on the + // build list we're using. + inRoots := map[*loadPkg]bool{} + for _, path := range ld.listRoots() { + root := ld.pkg(path, pkgIsRoot) + if !inRoots[root] { + ld.roots = append(ld.roots, root) + inRoots[root] = true } } - ld.work.Do(10, ld.doPkg) + + // ld.pkg adds imported packages to the work queue and calls applyPkgFlags, + // which adds tests (and test dependencies) as needed. + // + // When all of the work in the queue has completed, we'll know that the + // transitive closure of dependencies has been loaded. + <-ld.work.Idle() + ld.buildStacks() - numAdded := 0 - haveMod := make(map[module.Version]bool) - for _, m := range buildList { - haveMod[m] = true - } - modAddedBy := make(map[module.Version]*loadPkg) - for _, pkg := range ld.pkgs { - if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" { - if err.newMissingVersion != "" { - base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion) - } - fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, err.Module.Path, err.Module.Version) - if added[pkg.path] { - base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) - } - added[pkg.path] = true - numAdded++ - if !haveMod[err.Module] { - haveMod[err.Module] = true - modAddedBy[err.Module] = pkg - buildList = append(buildList, err.Module) - } - continue - } - // Leave other errors for Import or load.Packages to report. - } - base.ExitIfErrors() - if numAdded == 0 { + + modAddedBy := ld.resolveMissingImports(addedModuleFor) + if len(modAddedBy) == 0 { break } @@ -754,99 +825,264 @@ func (ld *loader) load(roots func() []string) { } } - // Add Go versions, computed during walk. - ld.goVersion = make(map[string]string) - for _, m := range buildList { - v, _ := reqs.(*mvsReqs).versions.Load(m) - ld.goVersion[m.Path], _ = v.(string) - } - - // Mix in direct markings (really, lack of indirect markings) - // from go.mod, unless we scanned the whole module - // and can therefore be sure we know better than go.mod. - if !ld.isALL && modFile != nil { + // If we didn't scan all of the imports from the main module, or didn't use + // imports.AnyTags, then we didn't necessarily load every package that + // contributes “direct” imports — so we can't safely mark existing + // dependencies as indirect-only. + // Conservatively mark those dependencies as direct. + if modFile != nil && (!ld.allPatternIsRoot || !reflect.DeepEqual(ld.tags, imports.AnyTags())) { for _, r := range modFile.Require { if !r.Indirect { ld.direct[r.Mod.Path] = true } } } + + return ld } -// pkg returns the *loadPkg for path, creating and queuing it if needed. -// If the package should be tested, its test is created but not queued -// (the test is queued after processing pkg). -// If isRoot is true, the pkg is being queued as one of the roots of the work graph. -func (ld *loader) pkg(path string, isRoot bool) *loadPkg { - return ld.pkgCache.Do(path, func() interface{} { - pkg := &loadPkg{ - path: path, +// resolveMissingImports adds module dependencies to the global build list +// in order to resolve missing packages from pkgs. +// +// The newly-resolved packages are added to the addedModuleFor map, and +// resolveMissingImports returns a map from each newly-added module version to +// the first package for which that module was added. +func (ld *loader) resolveMissingImports(addedModuleFor map[string]bool) (modAddedBy map[module.Version]*loadPkg) { + var needPkgs []*loadPkg + for _, pkg := range ld.pkgs { + if pkg.isTest() { + // If we are missing a test, we are also missing its non-test version, and + // we should only add the missing import once. + continue } - if ld.testRoots && isRoot || ld.testAll { - test := &loadPkg{ - path: path, - testOf: pkg, - } - pkg.test = test + if pkg.err != errImportMissing { + // Leave other errors for Import or load.Packages to report. + continue + } + + needPkgs = append(needPkgs, pkg) + + pkg := pkg + ld.work.Add(func() { + pkg.mod, pkg.err = queryImport(context.TODO(), pkg.path) + }) + } + <-ld.work.Idle() + + modAddedBy = map[module.Version]*loadPkg{} + for _, pkg := range needPkgs { + if pkg.err != nil { + continue } - if isRoot { - ld.roots = append(ld.roots, pkg) + + fmt.Fprintf(os.Stderr, "go: found %s in %s %s\n", pkg.path, pkg.mod.Path, pkg.mod.Version) + if addedModuleFor[pkg.path] { + // TODO(bcmills): This should only be an error if pkg.mod is the same + // version we already tried to add previously. + base.Fatalf("go: %s: looping trying to add package", pkg.stackText()) + } + if modAddedBy[pkg.mod] == nil { + modAddedBy[pkg.mod] = pkg + buildList = append(buildList, pkg.mod) + } + } + + return modAddedBy +} + +// pkg locates the *loadPkg for path, creating and queuing it for loading if +// needed, and updates its state to reflect the given flags. +// +// The imports of the returned *loadPkg will be loaded asynchronously in the +// ld.work queue, and its test (if requested) will also be populated once +// imports have been resolved. When ld.work goes idle, all transitive imports of +// the requested package (and its test, if requested) will have been loaded. +func (ld *loader) pkg(path string, flags loadPkgFlags) *loadPkg { + if flags.has(pkgImportsLoaded) { + panic("internal error: (*loader).pkg called with pkgImportsLoaded flag set") + } + + pkg := ld.pkgCache.Do(path, func() interface{} { + pkg := &loadPkg{ + path: path, } - ld.work.Add(pkg) + ld.applyPkgFlags(pkg, flags) + + ld.work.Add(func() { ld.load(pkg) }) return pkg }).(*loadPkg) + + ld.applyPkgFlags(pkg, flags) + return pkg } -// doPkg processes a package on the work queue. -func (ld *loader) doPkg(item interface{}) { - // TODO: what about replacements? - pkg := item.(*loadPkg) - var imports []string - if pkg.testOf != nil { - pkg.dir = pkg.testOf.dir - pkg.mod = pkg.testOf.mod - imports = pkg.testOf.testImports - } else { - if strings.Contains(pkg.path, "@") { - // Leave for error during load. - return - } - if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { - // Leave for error during load. - // (Module mode does not allow local imports.) - return - } +// applyPkgFlags updates pkg.flags to set the given flags and propagate the +// (transitive) effects of those flags, possibly loading or enqueueing further +// packages as a result. +func (ld *loader) applyPkgFlags(pkg *loadPkg, flags loadPkgFlags) { + if flags == 0 { + return + } - // TODO(matloob): Handle TODO context. This needs to be threaded through Do. - pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) - if pkg.dir == "" { - return + if flags.has(pkgInAll) && ld.allPatternIsRoot && !pkg.isTest() { + // This package matches a root pattern by virtue of being in "all". + flags |= pkgIsRoot + } + + old := pkg.flags.update(flags) + new := old | flags + if new == old || !new.has(pkgImportsLoaded) { + // We either didn't change the state of pkg, or we don't know anything about + // its dependencies yet. Either way, we can't usefully load its test or + // update its dependencies. + return + } + + if !pkg.isTest() { + // Check whether we should add (or update the flags for) a test for pkg. + // ld.pkgTest is idempotent and extra invocations are inexpensive, + // so it's ok if we call it more than is strictly necessary. + wantTest := false + switch { + case ld.allPatternIsRoot && pkg.mod == Target: + // We are loading the "all" pattern, which includes packages imported by + // tests in the main module. This package is in the main module, so we + // need to identify the imports of its test even if LoadTests is not set. + // + // (We will filter out the extra tests explicitly in computePatternAll.) + wantTest = true + + case ld.allPatternIsRoot && ld.allClosesOverTests && new.has(pkgInAll): + // This variant of the "all" pattern includes imports of tests of every + // package that is itself in "all", and pkg is in "all", so its test is + // also in "all" (as above). + wantTest = true + + case LoadTests && new.has(pkgIsRoot): + // LoadTest explicitly requests tests of “the root packages”. + wantTest = true } - var testImports []string - var err error - imports, testImports, err = scanDir(pkg.dir, ld.tags) - if err != nil { - pkg.err = err - return + + if wantTest { + var testFlags loadPkgFlags + if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + // Tests of packages in the main module are in "all", in the sense that + // they cause the packages they import to also be in "all". So are tests + // of packages in "all" if "all" closes over test dependencies. + testFlags |= pkgInAll + } + ld.pkgTest(pkg, testFlags) } - if pkg.test != nil { - pkg.testImports = testImports + } + + if new.has(pkgInAll) && !old.has(pkgInAll|pkgImportsLoaded) { + // We have just marked pkg with pkgInAll, or we have just loaded its + // imports, or both. Now is the time to propagate pkgInAll to the imports. + for _, dep := range pkg.imports { + ld.applyPkgFlags(dep, pkgInAll) } } +} + +// load loads an individual package. +func (ld *loader) load(pkg *loadPkg) { + if strings.Contains(pkg.path, "@") { + // Leave for error during load. + return + } + if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { + // Leave for error during load. + // (Module mode does not allow local imports.) + return + } - inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + pkg.mod, pkg.dir, pkg.err = importFromBuildList(context.TODO(), pkg.path) + if pkg.dir == "" { + return + } + if pkg.mod == Target { + // Go ahead and mark pkg as in "all". This provides the invariant that a + // package that is *only* imported by other packages in "all" is always + // marked as such before loading its imports. + // + // We don't actually rely on that invariant at the moment, but it may + // improve efficiency somewhat and makes the behavior a bit easier to reason + // about (by reducing churn on the flag bits of dependencies), and costs + // essentially nothing (these atomic flag ops are essentially free compared + // to scanning source code for imports). + ld.applyPkgFlags(pkg, pkgInAll) + } + + imports, testImports, err := scanDir(pkg.dir, ld.tags) + if err != nil { + pkg.err = err + return + } + + pkg.inStd = (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "") + + pkg.imports = make([]*loadPkg, 0, len(imports)) + var importFlags loadPkgFlags + if pkg.flags.has(pkgInAll) { + importFlags = pkgInAll + } for _, path := range imports { - if inStd { + if pkg.inStd { + // Imports from packages in "std" and "cmd" should resolve using + // GOROOT/src/vendor even when "std" is not the main module. path = ld.stdVendor(pkg.path, path) } - pkg.imports = append(pkg.imports, ld.pkg(path, false)) + pkg.imports = append(pkg.imports, ld.pkg(path, importFlags)) } + pkg.testImports = testImports - // Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test. - // TODO: All that's left is creating new imports. Why not just do it now? - if pkg.test != nil { - ld.work.Add(pkg.test) + ld.applyPkgFlags(pkg, pkgImportsLoaded) +} + +// pkgTest locates the test of pkg, creating it if needed, and updates its state +// to reflect the given flags. +// +// pkgTest requires that the imports of pkg have already been loaded (flagged +// with pkgImportsLoaded). +func (ld *loader) pkgTest(pkg *loadPkg, testFlags loadPkgFlags) *loadPkg { + if pkg.isTest() { + panic("pkgTest called on a test package") + } + + createdTest := false + pkg.testOnce.Do(func() { + pkg.test = &loadPkg{ + path: pkg.path, + testOf: pkg, + mod: pkg.mod, + dir: pkg.dir, + err: pkg.err, + inStd: pkg.inStd, + } + ld.applyPkgFlags(pkg.test, testFlags) + createdTest = true + }) + + test := pkg.test + if createdTest { + test.imports = make([]*loadPkg, 0, len(pkg.testImports)) + var importFlags loadPkgFlags + if test.flags.has(pkgInAll) { + importFlags = pkgInAll + } + for _, path := range pkg.testImports { + if pkg.inStd { + path = ld.stdVendor(test.path, path) + } + test.imports = append(test.imports, ld.pkg(path, importFlags)) + } + pkg.testImports = nil + ld.applyPkgFlags(test, pkgImportsLoaded) + } else { + ld.applyPkgFlags(test, testFlags) } + + return test } // stdVendor returns the canonical import path for the package with the given @@ -857,13 +1093,21 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if ld.forceStdVendor || Target.Path != "cmd" { + if Target.Path != "cmd" { vendorPath := pathpkg.Join("cmd", "vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if ld.forceStdVendor || Target.Path != "std" { + } else if Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + // If we are outside of the 'std' module, resolve imports from within 'std' + // to the vendor directory. + // + // Do the same for importers beginning with the prefix 'vendor/' even if we + // are *inside* of the 'std' module: the 'vendor/' packages that resolve + // globally from GOROOT/src/vendor (and are listed as part of 'go list std') + // are distinct from the real module dependencies, and cannot import internal + // packages from the real module. vendorPath := pathpkg.Join("vendor", path) if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath @@ -876,30 +1120,13 @@ func (ld *loader) stdVendor(parentPath, path string) string { // computePatternAll returns the list of packages matching pattern "all", // starting with a list of the import paths for the packages in the main module. -func (ld *loader) computePatternAll(paths []string) []string { - seen := make(map[*loadPkg]bool) - var all []string - var walk func(*loadPkg) - walk = func(pkg *loadPkg) { - if seen[pkg] { - return - } - seen[pkg] = true - if pkg.testOf == nil { +func (ld *loader) computePatternAll() (all []string) { + for _, pkg := range ld.pkgs { + if pkg.flags.has(pkgInAll) && !pkg.isTest() { all = append(all, pkg.path) } - for _, p := range pkg.imports { - walk(p) - } - if p := pkg.test; p != nil { - walk(p) - } - } - for _, path := range paths { - walk(ld.pkg(path, false)) } sort.Strings(all) - return all } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 9f4ec5a49f..18dd293ac9 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -5,13 +5,31 @@ package modload import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "unicode" + "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modfetch" + "cmd/go/internal/par" + "cmd/go/internal/trace" "golang.org/x/mod/modfile" "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) +// lazyLoadingVersion is the Go version (plus leading "v") at which lazy module +// loading takes effect. +const lazyLoadingVersionV = "v1.16" +const go116EnableLazyLoading = true + var modFile *modfile.File // A modFileIndex is an index of data corresponding to a modFile @@ -20,7 +38,7 @@ type modFileIndex struct { data []byte dataNeedsFix bool // true if fixVersion applied a change while parsing data module module.Version - goVersion string + goVersionV string // GoVersion with "v" prefix require map[module.Version]requireMeta replace map[module.Version]module.Version exclude map[module.Version]bool @@ -33,9 +51,151 @@ type requireMeta struct { indirect bool } -// Allowed reports whether module m is allowed (not excluded) by the main module's go.mod. -func Allowed(m module.Version) bool { - return index == nil || !index.exclude[m] +// CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by +// the main module's go.mod or retracted by its author. Most version queries use +// this to filter out versions that should not be used. +func CheckAllowed(ctx context.Context, m module.Version) error { + if err := CheckExclusions(ctx, m); err != nil { + return err + } + if err := checkRetractions(ctx, m); err != nil { + return err + } + return nil +} + +// ErrDisallowed is returned by version predicates passed to Query and similar +// functions to indicate that a version should not be considered. +var ErrDisallowed = errors.New("disallowed module version") + +// CheckExclusions returns an error equivalent to ErrDisallowed if module m is +// excluded by the main module's go.mod file. +func CheckExclusions(ctx context.Context, m module.Version) error { + if index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } + return nil +} + +var errExcluded = &excludedError{} + +type excludedError struct{} + +func (e *excludedError) Error() string { return "excluded by go.mod" } +func (e *excludedError) Is(err error) bool { return err == ErrDisallowed } + +// checkRetractions returns an error if module m has been retracted by +// its author. +func checkRetractions(ctx context.Context, m module.Version) error { + if m.Version == "" { + // Main module, standard library, or file replacement module. + // Cannot be retracted. + return nil + } + + // Look up retraction information from the latest available version of + // the module. Cache retraction information so we don't parse the go.mod + // file repeatedly. + type entry struct { + retract []retraction + err error + } + path := m.Path + e := retractCache.Do(path, func() (v interface{}) { + ctx, span := trace.StartSpan(ctx, "checkRetractions "+path) + defer span.Done() + + if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + // All versions of the module were replaced with a local directory. + // Don't load retractions. + return &entry{nil, nil} + } + + // Find the latest version of the module. + // Ignore exclusions from the main module's go.mod. + // We may need to account for the current version: for example, + // v2.0.0+incompatible is not "latest" if v1.0.0 is current. + rev, err := Query(ctx, path, "latest", findCurrentVersion(path), nil) + if err != nil { + return &entry{err: err} + } + + // Load go.mod for that version. + // If the version is replaced, we'll load retractions from the replacement. + // If there's an error loading the go.mod, we'll return it here. + // These errors should generally be ignored by callers of checkRetractions, + // since they happen frequently when we're offline. These errors are not + // equivalent to ErrDisallowed, so they may be distinguished from + // retraction errors. + summary, err := goModSummary(module.Version{Path: path, Version: rev.Version}) + if err != nil { + return &entry{err: err} + } + return &entry{retract: summary.retract} + }).(*entry) + + if e.err != nil { + return fmt.Errorf("loading module retractions: %v", e.err) + } + + var rationale []string + isRetracted := false + for _, r := range e.retract { + if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 { + isRetracted = true + if r.Rationale != "" { + rationale = append(rationale, r.Rationale) + } + } + } + if isRetracted { + return &retractedError{rationale: rationale} + } + return nil +} + +var retractCache par.Cache + +type retractedError struct { + rationale []string +} + +func (e *retractedError) Error() string { + msg := "retracted by module author" + if len(e.rationale) > 0 { + // This is meant to be a short error printed on a terminal, so just + // print the first rationale. + msg += ": " + ShortRetractionRationale(e.rationale[0]) + } + return msg +} + +func (e *retractedError) Is(err error) bool { + return err == ErrDisallowed +} + +// ShortRetractionRationale returns a retraction rationale string that is safe +// to print in a terminal. It returns hard-coded strings if the rationale +// is empty, too long, or contains non-printable characters. +func ShortRetractionRationale(rationale string) string { + const maxRationaleBytes = 500 + if i := strings.Index(rationale, "\n"); i >= 0 { + rationale = rationale[:i] + } + rationale = strings.TrimSpace(rationale) + if rationale == "" { + return "retracted by module author" + } + if len(rationale) > maxRationaleBytes { + return "(rationale omitted: too long)" + } + for _, r := range rationale { + if !unicode.IsGraphic(r) && !unicode.IsSpace(r) { + return "(rationale omitted: contains non-printable characters)" + } + } + // NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here. + return rationale } // Replacement returns the replacement for mod, if any, from go.mod. @@ -66,9 +226,11 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.module = modFile.Module.Mod } - i.goVersion = "" + i.goVersionV = "" if modFile.Go != nil { - i.goVersion = modFile.Go.Version + // We're going to use the semver package to compare Go versions, so go ahead + // and add the "v" prefix it expects once instead of every time. + i.goVersionV = "v" + modFile.Go.Version } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -92,6 +254,23 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd return i } +// allPatternClosesOverTests reports whether the "all" pattern includes +// dependencies of tests outside the main module (as in Go 1.11–1.15). +// (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages +// transitively *imported by* the packages and tests in the main module.) +func (i *modFileIndex) allPatternClosesOverTests() bool { + if !go116EnableLazyLoading { + return true + } + if i != nil && semver.Compare(i.goVersionV, lazyLoadingVersionV) < 0 { + // The module explicitly predates the change in "all" for lazy loading, so + // continue to use the older interpretation. (If i == nil, we not in any + // module at all and should use the latest semantics.) + return true + } + return false +} + // modFileIsDirty reports whether the go.mod file differs meaningfully // from what was indexed. // If modFile has been changed (even cosmetically) since it was first read, @@ -114,11 +293,11 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { } if modFile.Go == nil { - if i.goVersion != "" { + if i.goVersionV != "" { return true } - } else if modFile.Go.Version != i.goVersion { - if i.goVersion == "" && cfg.BuildMod == "readonly" { + } else if "v"+modFile.Go.Version != i.goVersionV { + if i.goVersionV == "" && cfg.BuildMod == "readonly" { // go.mod files did not always require a 'go' version, so do not error out // if one is missing — we may be inside an older module in the module // cache, and should bias toward providing useful behavior. @@ -162,3 +341,190 @@ func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool { return false } + +// rawGoVersion records the Go version parsed from each module's go.mod file. +// +// If a module is replaced, the version of the replacement is keyed by the +// replacement module.Version, not the version being replaced. +var rawGoVersion sync.Map // map[module.Version]string + +// A modFileSummary is a summary of a go.mod file for which we do not need to +// retain complete information — for example, the go.mod file of a dependency +// module. +type modFileSummary struct { + module module.Version + goVersionV string // GoVersion with "v" prefix + require []module.Version + retract []retraction +} + +// A retraction consists of a retracted version interval and rationale. +// retraction is like modfile.Retract, but it doesn't point to the syntax tree. +type retraction struct { + modfile.VersionInterval + Rationale string +} + +// goModSummary returns a summary of the go.mod file for module m, +// taking into account any replacements for m, exclusions of its dependencies, +// and/or vendoring. +// +// goModSummary cannot be used on the Target module, as its requirements +// may change. +// +// The caller must not modify the returned summary. +func goModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: goModSummary called on the Target module") + } + + type cached struct { + summary *modFileSummary + err error + } + c := goModSummaryCache.Do(m, func() interface{} { + if cfg.BuildMod == "vendor" { + summary := &modFileSummary{ + module: module.Version{Path: m.Path}, + } + if vendorVersion[m.Path] != m.Version { + // This module is not vendored, so packages cannot be loaded from it and + // it cannot be relevant to the build. + return cached{summary, nil} + } + + // For every module other than the target, + // return the full list of modules from modules.txt. + readVendorList() + + // TODO(#36876): Load the "go" version from vendor/modules.txt and store it + // in rawGoVersion with the appropriate key. + + // We don't know what versions the vendored module actually relies on, + // so assume that it requires everything. + summary.require = vendorList + return cached{summary, nil} + } + + actual := Replacement(m) + if actual.Path == "" { + actual = m + } + summary, err := rawGoModSummary(actual) + if err != nil { + return cached{nil, err} + } + + if actual.Version == "" { + // The actual module is a filesystem-local replacement, for which we have + // unfortunately not enforced any sort of invariants about module lines or + // matching module paths. Anything goes. + // + // TODO(bcmills): Remove this special-case, update tests, and add a + // release note. + } else { + if summary.module.Path == "" { + return cached{nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line"))} + } + + // In theory we should only allow mpath to be unequal to m.Path here if the + // version that we fetched lacks an explicit go.mod file: if the go.mod file + // is explicit, then it should match exactly (to ensure that imports of other + // packages within the module are interpreted correctly). Unfortunately, we + // can't determine that information from the module proxy protocol: we'll have + // to leave that validation for when we load actual packages from within the + // module. + if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path { + return cached{nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod: + module declares its path as: %s + but was required as: %s`, mpath, m.Path))} + } + } + + if index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + nonExcluded := summary.require[:0] + for _, r := range summary.require { + if !index.exclude[r] { + nonExcluded = append(nonExcluded, r) + } + } + summary.require = nonExcluded + } + return cached{summary, nil} + }).(cached) + + return c.summary, c.err +} + +var goModSummaryCache par.Cache // module.Version → goModSummary result + +// rawGoModSummary returns a new summary of the go.mod file for module m, +// ignoring all replacements that may apply to m and excludes that may apply to +// its dependencies. +// +// rawGoModSummary cannot be used on the Target module. +func rawGoModSummary(m module.Version) (*modFileSummary, error) { + if m == Target { + panic("internal error: rawGoModSummary called on the Target module") + } + + summary := new(modFileSummary) + var f *modfile.File + if m.Version == "" { + // m is a replacement module with only a file path. + dir := m.Path + if !filepath.IsAbs(dir) { + dir = filepath.Join(ModRoot(), dir) + } + gomod := filepath.Join(dir, "go.mod") + + data, err := lockedfile.Read(gomod) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err)) + } + f, err = modfile.ParseLax(gomod, data, nil) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err)) + } + } else { + if !semver.IsValid(m.Version) { + // Disallow the broader queries supported by fetch.Lookup. + base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version) + } + + data, err := modfetch.GoMod(m.Path, m.Version) + if err != nil { + return nil, err + } + f, err = modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err)) + } + } + + if f.Module != nil { + summary.module = f.Module.Mod + } + if f.Go != nil && f.Go.Version != "" { + rawGoVersion.LoadOrStore(m, f.Go.Version) + summary.goVersionV = "v" + f.Go.Version + } + if len(f.Require) > 0 { + summary.require = make([]module.Version, 0, len(f.Require)) + for _, req := range f.Require { + summary.require = append(summary.require, req.Mod) + } + } + if len(f.Retract) > 0 { + summary.retract = make([]retraction, 0, len(f.Retract)) + for _, ret := range f.Retract { + summary.retract = append(summary.retract, retraction{ + VersionInterval: ret.VersionInterval, + Rationale: ret.Rationale, + }) + } + } + + return summary, nil +} diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 67eb2c2e19..24856260d4 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -11,16 +11,10 @@ import ( "os" "path/filepath" "sort" - "sync" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/lockedfile" "cmd/go/internal/modfetch" "cmd/go/internal/mvs" - "cmd/go/internal/par" - "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -29,8 +23,6 @@ import ( // with any exclusions or replacements applied internally. type mvsReqs struct { buildList []module.Version - cache par.Cache - versions sync.Map } // Reqs returns the current module requirement graph. @@ -44,118 +36,21 @@ func Reqs() mvs.Reqs { } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - type cached struct { - list []module.Version - err error - } - - c := r.cache.Do(mod, func() interface{} { - list, err := r.required(mod) - if err != nil { - return cached{nil, err} - } - for i, mv := range list { - if index != nil { - for index.exclude[mv] { - mv1, err := r.next(mv) - if err != nil { - return cached{nil, err} - } - if mv1.Version == "none" { - return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)} - } - mv = mv1 - } - } - list[i] = mv - } - - return cached{list, nil} - }).(cached) - - return c.list, c.err -} - -func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version { - list := make([]module.Version, 0, len(f.Require)) - for _, r := range f.Require { - list = append(list, r.Mod) - } - return list -} - -// required returns a unique copy of the requirements of mod. -func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) { if mod == Target { - if modFile != nil && modFile.Go != nil { - r.versions.LoadOrStore(mod, modFile.Go.Version) - } - return append([]module.Version(nil), r.buildList[1:]...), nil - } - - if cfg.BuildMod == "vendor" { - // For every module other than the target, - // return the full list of modules from modules.txt. - readVendorList() - return append([]module.Version(nil), vendorList...), nil - } - - origPath := mod.Path - if repl := Replacement(mod); repl.Path != "" { - if repl.Version == "" { - // TODO: need to slip the new version into the tags list etc. - dir := repl.Path - if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) - } - gomod := filepath.Join(dir, "go.mod") - data, err := lockedfile.Read(gomod) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - f, err := modfile.ParseLax(gomod, data, nil) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - return r.modFileToList(f), nil - } - mod = repl + // Use the build list as it existed when r was constructed, not the current + // global build list. + return r.buildList[1:], nil } if mod.Version == "none" { return nil, nil } - if !semver.IsValid(mod.Version) { - // Disallow the broader queries supported by fetch.Lookup. - base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version) - } - - data, err := modfetch.GoMod(mod.Path, mod.Version) + summary, err := goModSummary(mod) if err != nil { return nil, err } - f, err := modfile.ParseLax("go.mod", data, nil) - if err != nil { - return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err)) - } - - if f.Module == nil { - return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line")) - } - if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path { - return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod: - module declares its path as: %s - but was required as: %s`, mpath, origPath)) - } - if f.Go != nil { - r.versions.LoadOrStore(mod, f.Go.Version) - } - - return r.modFileToList(f), nil + return summary.require, nil } // Max returns the maximum of v1 and v2 according to semver.Compare. @@ -177,16 +72,29 @@ func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) { return m, nil } -func versions(path string) ([]string, error) { +func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) { // Note: modfetch.Lookup and repo.Versions are cached, // so there's no need for us to add extra caching here. var versions []string err := modfetch.TryProxies(func(proxy string) error { repo, err := modfetch.Lookup(proxy, path) - if err == nil { - versions, err = repo.Versions("") + if err != nil { + return err + } + allVersions, err := repo.Versions("") + if err != nil { + return err + } + allowedVersions := make([]string, 0, len(allVersions)) + for _, v := range allVersions { + if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil { + allowedVersions = append(allowedVersions, v) + } else if !errors.Is(err, ErrDisallowed) { + return err + } } - return err + versions = allowedVersions + return nil }) return versions, err } @@ -194,7 +102,8 @@ func versions(path string) ([]string, error) { // Previous returns the tagged version of m.Path immediately prior to // m.Version, or version "none" if no prior version is tagged. func (*mvsReqs) Previous(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { return module.Version{}, err } @@ -209,7 +118,8 @@ func (*mvsReqs) Previous(m module.Version) (module.Version, error) { // It is only used by the exclusion processing in the Required method, // not called directly by MVS. func (*mvsReqs) next(m module.Version) (module.Version, error) { - list, err := versions(m.Path) + // TODO(golang.org/issue/38714): thread tracing context through MVS. + list, err := versions(context.TODO(), m.Path, CheckAllowed) if err != nil { return module.Version{}, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e82eb1506f..f67a738677 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -52,12 +52,16 @@ import ( // version that would otherwise be chosen. This prevents accidental downgrades // from newer pre-release or development versions. // -// If the allowed function is non-nil, Query excludes any versions for which -// allowed returns false. +// The allowed function (which may be nil) is used to filter out unsuitable +// versions (see AllowedFunc documentation for details). If the query refers to +// a specific revision (for example, "master"; see IsRevisionQuery), and the +// revision is disallowed by allowed, Query returns the error. If the query +// does not refer to a specific revision (for example, "latest"), Query +// acts as if versions disallowed by allowed do not exist. // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { info, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -66,6 +70,17 @@ func Query(ctx context.Context, path, query, current string, allowed func(module return info, err } +// AllowedFunc is used by Query and other functions to filter out unsuitable +// versions, for example, those listed in exclude directives in the main +// module's go.mod file. +// +// An AllowedFunc returns an error equivalent to ErrDisallowed for an unsuitable +// version. Any other error indicates the function was unable to determine +// whether the version should be allowed, for example, the function was unable +// to fetch or parse a go.mod file containing retractions. Typically, errors +// other than ErrDisallowd may be ignored. +type AllowedFunc func(context.Context, module.Version) error + var errQueryDisabled error = queryDisabledError{} type queryDisabledError struct{} @@ -77,7 +92,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) { ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) defer span.Done() @@ -88,7 +103,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, errQueryDisabled } if allowed == nil { - allowed = func(module.Version) bool { return true } + allowed = func(context.Context, module.Version) error { return nil } } // Parse query to detect parse errors (and possibly handle query) @@ -104,7 +119,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return module.CheckPathMajor(v, pathMajor) == nil } var ( - ok func(module.Version) bool + match = func(m module.Version) bool { return true } + prefix string preferOlder bool mayUseLatest bool @@ -112,21 +128,18 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed ) switch { case query == "latest": - ok = allowed mayUseLatest = true case query == "upgrade": - ok = allowed mayUseLatest = true case query == "patch": if current == "" { - ok = allowed mayUseLatest = true } else { prefix = semver.MajorMinor(current) - ok = func(m module.Version) bool { - return matchSemverPrefix(prefix, m.Version) && allowed(m) + match = func(m module.Version) bool { + return matchSemverPrefix(prefix, m.Version) } } @@ -139,8 +152,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // Refuse to say whether <=v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) <= 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) <= 0 } if !matchesMajor(v) { preferIncompatible = true @@ -151,8 +164,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if !semver.IsValid(v) { return badVersion(v) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) < 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) < 0 } if !matchesMajor(v) { preferIncompatible = true @@ -163,8 +176,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if !semver.IsValid(v) { return badVersion(v) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) >= 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) >= 0 } preferOlder = true if !matchesMajor(v) { @@ -180,8 +193,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed // Refuse to say whether >v1.2 allows v1.2.3 (remember, @v1.2 might mean v1.2.3). return nil, fmt.Errorf("ambiguous semantic version %q in range %q", v, query) } - ok = func(m module.Version) bool { - return semver.Compare(m.Version, v) > 0 && allowed(m) + match = func(m module.Version) bool { + return semver.Compare(m.Version, v) > 0 } preferOlder = true if !matchesMajor(v) { @@ -189,8 +202,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } case semver.IsValid(query) && isSemverPrefix(query): - ok = func(m module.Version) bool { - return matchSemverPrefix(query, m.Version) && allowed(m) + match = func(m module.Version) bool { + return matchSemverPrefix(query, m.Version) } prefix = query + "." if !matchesMajor(query) { @@ -219,8 +232,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, queryErr } } - if !allowed(module.Version{Path: path, Version: info.Version}) { - return nil, fmt.Errorf("%s@%s excluded", path, info.Version) + if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) { + return nil, err } return info, nil } @@ -229,8 +242,8 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if query != "latest" { return nil, fmt.Errorf("can't query specific version (%q) for the main module (%s)", query, path) } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: main module version is not allowed") + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } return &modfetch.RevInfo{Version: Target.Version}, nil } @@ -248,7 +261,13 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if err != nil { return nil, err } - releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) + matchAndAllowed := func(ctx context.Context, m module.Version) error { + if !match(m) { + return ErrDisallowed + } + return allowed(ctx, m) + } + releases, prereleases, err := filterVersions(ctx, path, versions, matchAndAllowed, preferIncompatible) if err != nil { return nil, err } @@ -288,11 +307,12 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed } if mayUseLatest { - // Special case for "latest": if no tags match, use latest commit in repo, - // provided it is not excluded. + // Special case for "latest": if no tags match, use latest commit in repo + // if it is allowed. latest, err := repo.Latest() if err == nil { - if allowed(module.Version{Path: path, Version: latest.Version}) { + m := module.Version{Path: path, Version: latest.Version} + if err := allowed(ctx, m); !errors.Is(err, ErrDisallowed) { return lookup(latest.Version) } } else if !errors.Is(err, os.ErrNotExist) { @@ -303,6 +323,22 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed return nil, &NoMatchingVersionError{query: query, current: current} } +// IsRevisionQuery returns true if vers is a version query that may refer to +// a particular version or revision in a repository like "v1.0.0", "master", +// or "0123abcd". IsRevisionQuery returns false if vers is a query that +// chooses from among available versions like "latest" or ">v1.0.0". +func IsRevisionQuery(vers string) bool { + if vers == "latest" || + vers == "upgrade" || + vers == "patch" || + strings.HasPrefix(vers, "<") || + strings.HasPrefix(vers, ">") || + (semver.IsValid(vers) && isSemverPrefix(vers)) { + return false + } + return true +} + // isSemverPrefix reports whether v is a semantic version prefix: v1 or v1.2 (not v1.2.3). // The caller is assumed to have checked that semver.IsValid(v) is true. func isSemverPrefix(v string) bool { @@ -329,13 +365,16 @@ func matchSemverPrefix(p, v string) bool { // filterVersions classifies versions into releases and pre-releases, filtering // out: -// 1. versions that do not satisfy the 'ok' predicate, and +// 1. versions that do not satisfy the 'allowed' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +// +// If the allowed predicate returns an error not equivalent to ErrDisallowed, +// filterVersions returns that error. +func filterVersions(ctx context.Context, path string, versions []string, allowed AllowedFunc, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { - if !ok(module.Version{Path: path, Version: v}) { + if err := allowed(ctx, module.Version{Path: path, Version: v}); errors.Is(err, ErrDisallowed) { continue } @@ -385,7 +424,7 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed AllowedFunc) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) @@ -406,7 +445,7 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed AllowedFunc) ([]QueryResult, error) { ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) defer span.Done() @@ -450,8 +489,8 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul if query != "latest" { return nil, fmt.Errorf("can't query specific version for package %s in the main module (%s)", pattern, Target.Path) } - if !allowed(Target) { - return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed", pattern, Target.Path) + if err := allowed(ctx, Target); err != nil { + return nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) } return []QueryResult{{ Mod: Target, diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 77080e9b5b..351826f2ab 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -187,9 +187,11 @@ func TestQuery(t *testing.T) { if allow == "" { allow = "*" } - allowed := func(m module.Version) bool { - ok, _ := path.Match(allow, m.Version) - return ok + allowed := func(ctx context.Context, m module.Version) error { + if ok, _ := path.Match(allow, m.Version); !ok { + return ErrDisallowed + } + return nil } tt := tt t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 71f68efbcc..9f34b829fc 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -133,7 +133,7 @@ func checkVendorConsistency() { readVendorList() pre114 := false - if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 { + if semver.Compare(index.goVersionV, "v1.14") < 0 { // Go versions before 1.14 did not include enough information in // vendor/modules.txt to check for consistency. // If we know that we're on an earlier version, relax the consistency check. @@ -150,6 +150,8 @@ func checkVendorConsistency() { } } + // Iterate over the Require directives in their original (not indexed) order + // so that the errors match the original file. for _, r := range modFile.Require { if !vendorMeta[r.Mod].Explicit { if pre114 { diff --git a/src/cmd/go/internal/mvs/errors.go b/src/cmd/go/internal/mvs/errors.go new file mode 100644 index 0000000000..5564965fb5 --- /dev/null +++ b/src/cmd/go/internal/mvs/errors.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isUpgrade function reports whether a path step is due to an upgrade. +// A nil isUpgrade function indicates that none of the path steps are due to upgrades. +func NewBuildListError(err error, path []module.Version, isUpgrade func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isUpgrade != nil && isUpgrade(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} diff --git a/src/cmd/go/internal/mvs/mvs.go b/src/cmd/go/internal/mvs/mvs.go index 1f8eaa1f60..ea23a9f45e 100644 --- a/src/cmd/go/internal/mvs/mvs.go +++ b/src/cmd/go/internal/mvs/mvs.go @@ -9,7 +9,6 @@ package mvs import ( "fmt" "sort" - "strings" "sync" "sync/atomic" @@ -61,59 +60,6 @@ type Reqs interface { Previous(m module.Version) (module.Version, error) } -// BuildListError decorates an error that occurred gathering requirements -// while constructing a build list. BuildListError prints the chain -// of requirements to the module where the error occurred. -type BuildListError struct { - Err error - stack []buildListErrorElem -} - -type buildListErrorElem struct { - m module.Version - - // nextReason is the reason this module depends on the next module in the - // stack. Typically either "requires", or "upgraded to". - nextReason string -} - -// Module returns the module where the error occurred. If the module stack -// is empty, this returns a zero value. -func (e *BuildListError) Module() module.Version { - if len(e.stack) == 0 { - return module.Version{} - } - return e.stack[0].m -} - -func (e *BuildListError) Error() string { - b := &strings.Builder{} - stack := e.stack - - // Don't print modules at the beginning of the chain without a - // version. These always seem to be the main module or a - // synthetic module ("target@"). - for len(stack) > 0 && stack[len(stack)-1].m.Version == "" { - stack = stack[:len(stack)-1] - } - - for i := len(stack) - 1; i >= 1; i-- { - fmt.Fprintf(b, "%s@%s %s\n\t", stack[i].m.Path, stack[i].m.Version, stack[i].nextReason) - } - if len(stack) == 0 { - b.WriteString(e.Err.Error()) - } else { - // Ensure that the final module path and version are included as part of the - // error message. - if _, ok := e.Err.(*module.ModuleError); ok { - fmt.Fprintf(b, "%v", e.Err) - } else { - fmt.Fprintf(b, "%v", module.VersionError(stack[0].m, e.Err)) - } - } - return b.String() -} - // BuildList returns the build list for the target module. // // target is the root vertex of a module requirement graph. For cmd/go, this is @@ -202,18 +148,30 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m q = q[1:] if node.err != nil { - err := &BuildListError{ - Err: node.err, - stack: []buildListErrorElem{{m: node.m}}, - } + pathUpgrade := map[module.Version]module.Version{} + + // Construct the error path reversed (from the error to the main module), + // then reverse it to obtain the usual order (from the main module to + // the error). + errPath := []module.Version{node.m} for n, prev := neededBy[node], node; n != nil; n, prev = neededBy[n], n { - reason := "requires" if n.upgrade == prev.m { - reason = "updating to" + pathUpgrade[n.m] = prev.m } - err.stack = append(err.stack, buildListErrorElem{m: n.m, nextReason: reason}) + errPath = append(errPath, n.m) } - return nil, err + i, j := 0, len(errPath)-1 + for i < j { + errPath[i], errPath[j] = errPath[j], errPath[i] + i++ + j-- + } + + isUpgrade := func(from, to module.Version) bool { + return pathUpgrade[from] == to + } + + return nil, NewBuildListError(node.err, errPath, isUpgrade) } neighbors := node.required diff --git a/src/cmd/go/internal/par/queue.go b/src/cmd/go/internal/par/queue.go new file mode 100644 index 0000000000..180bc75e34 --- /dev/null +++ b/src/cmd/go/internal/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/src/cmd/go/internal/par/queue_test.go b/src/cmd/go/internal/par/queue_test.go new file mode 100644 index 0000000000..1331e65f98 --- /dev/null +++ b/src/cmd/go/internal/par/queue_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import ( + "sync" + "testing" +) + +func TestQueueIdle(t *testing.T) { + q := NewQueue(1) + select { + case <-q.Idle(): + default: + t.Errorf("NewQueue(1) is not initially idle.") + } + + started := make(chan struct{}) + unblock := make(chan struct{}) + q.Add(func() { + close(started) + <-unblock + }) + + <-started + idle := q.Idle() + select { + case <-idle: + t.Errorf("NewQueue(1) is marked idle while processing work.") + default: + } + + close(unblock) + <-idle // Should be closed as soon as the Add callback returns. +} + +func TestQueueBacklog(t *testing.T) { + const ( + maxActive = 2 + totalWork = 3 * maxActive + ) + + q := NewQueue(maxActive) + t.Logf("q = NewQueue(%d)", maxActive) + + var wg sync.WaitGroup + wg.Add(totalWork) + started := make([]chan struct{}, totalWork) + unblock := make(chan struct{}) + for i := range started { + started[i] = make(chan struct{}) + i := i + q.Add(func() { + close(started[i]) + <-unblock + wg.Done() + }) + } + + for i, c := range started { + if i < maxActive { + <-c // Work item i should be started immediately. + } else { + select { + case <-c: + t.Errorf("Work item %d started before previous items finished.", i) + default: + } + } + } + + close(unblock) + for _, c := range started[maxActive:] { + <-c + } + wg.Wait() +} diff --git a/src/cmd/go/internal/str/str_test.go b/src/cmd/go/internal/str/str_test.go new file mode 100644 index 0000000000..147ce1a63e --- /dev/null +++ b/src/cmd/go/internal/str/str_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package str + +import "testing" + +var foldDupTests = []struct { + list []string + f1, f2 string +}{ + {StringList("math/rand", "math/big"), "", ""}, + {StringList("math", "strings"), "", ""}, + {StringList("strings"), "", ""}, + {StringList("strings", "strings"), "strings", "strings"}, + {StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, +} + +func TestFoldDup(t *testing.T) { + for _, tt := range foldDupTests { + f1, f2 := FoldDup(tt.list) + if f1 != tt.f1 || f2 != tt.f2 { + t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) + } + } +} diff --git a/src/cmd/go/internal/test/flagdefs_test.go b/src/cmd/go/internal/test/flagdefs_test.go index 7562415298..ab5440b380 100644 --- a/src/cmd/go/internal/test/flagdefs_test.go +++ b/src/cmd/go/internal/test/flagdefs_test.go @@ -16,9 +16,14 @@ func TestPassFlagToTestIncludesAllTestFlags(t *testing.T) { return } name := strings.TrimPrefix(f.Name, "test.") - if name != "testlogfile" && !passFlagToTest[name] { - t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) - t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + switch name { + case "testlogfile", "paniconexit0": + // These are internal flags. + default: + if !passFlagToTest[name] { + t.Errorf("passFlagToTest missing entry for %q (flag test.%s)", name, name) + t.Logf("(Run 'go generate cmd/go/internal/test' if it should be added.)") + } } }) diff --git a/src/cmd/go/internal/test/genflags.go b/src/cmd/go/internal/test/genflags.go index 512fa1671e..5e83d53980 100644 --- a/src/cmd/go/internal/test/genflags.go +++ b/src/cmd/go/internal/test/genflags.go @@ -62,9 +62,10 @@ func testFlags() []string { } name := strings.TrimPrefix(f.Name, "test.") - if name == "testlogfile" { - // test.testlogfile is “for use only by cmd/go” - } else { + switch name { + case "testlogfile", "paniconexit0": + // These flags are only for use by cmd/go. + default: names = append(names, name) } }) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 3aee6939d2..1ea6d2881e 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1164,7 +1164,8 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. if !c.disableCache && len(execCmd) == 0 { testlogArg = []string{"-test.testlogfile=" + a.Objdir + "testlog.txt"} } - args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, testArgs) + panicArg := "-test.paniconexit0" + args := str.StringList(execCmd, a.Deps[0].BuiltTarget(), testlogArg, panicArg, testArgs) if testCoverProfile != "" { // Write coverage to temporary profile, for merging later. diff --git a/src/cmd/go/internal/get/discovery.go b/src/cmd/go/internal/vcs/discovery.go index afa6ef455f..327b44cb9a 100644 --- a/src/cmd/go/internal/get/discovery.go +++ b/src/cmd/go/internal/vcs/discovery.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/xml" diff --git a/src/cmd/go/internal/get/pkg_test.go b/src/cmd/go/internal/vcs/discovery_test.go index fc6a179c2e..eb99fdf64c 100644 --- a/src/cmd/go/internal/get/pkg_test.go +++ b/src/cmd/go/internal/vcs/discovery_test.go @@ -2,35 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( - "cmd/go/internal/str" "reflect" "strings" "testing" ) -var foldDupTests = []struct { - list []string - f1, f2 string -}{ - {str.StringList("math/rand", "math/big"), "", ""}, - {str.StringList("math", "strings"), "", ""}, - {str.StringList("strings"), "", ""}, - {str.StringList("strings", "strings"), "strings", "strings"}, - {str.StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, -} - -func TestFoldDup(t *testing.T) { - for _, tt := range foldDupTests { - f1, f2 := str.FoldDup(tt.list) - if f1 != tt.f1 || f2 != tt.f2 { - t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) - } - } -} - var parseMetaGoImportsTests = []struct { in string mod ModuleMode diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/vcs/vcs.go index fd37fcb76f..e535998d89 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "encoding/json" @@ -27,23 +27,23 @@ import ( // A vcsCmd describes how to use a version control system // like Mercurial, Git, or Subversion. -type vcsCmd struct { - name string - cmd string // name of binary to invoke command +type Cmd struct { + Name string + Cmd string // name of binary to invoke command - createCmd []string // commands to download a fresh copy of a repository - downloadCmd []string // commands to download updates into an existing repository + CreateCmd []string // commands to download a fresh copy of a repository + DownloadCmd []string // commands to download updates into an existing repository - tagCmd []tagCmd // commands to list tags - tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd []string // commands to sync to specific tag - tagSyncDefault []string // commands to sync to default tag + TagCmd []tagCmd // commands to list tags + TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + TagSyncCmd []string // commands to sync to specific tag + TagSyncDefault []string // commands to sync to default tag - scheme []string - pingCmd string + Scheme []string + PingCmd string - remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) - resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) + RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) + ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) } var defaultSecureScheme = map[string]bool{ @@ -54,7 +54,7 @@ var defaultSecureScheme = map[string]bool{ "ssh": true, } -func (v *vcsCmd) isSecure(repo string) bool { +func (v *Cmd) IsSecure(repo string) bool { u, err := urlpkg.Parse(repo) if err != nil { // If repo is not a URL, it's not secure. @@ -63,8 +63,8 @@ func (v *vcsCmd) isSecure(repo string) bool { return v.isSecureScheme(u.Scheme) } -func (v *vcsCmd) isSecureScheme(scheme string) bool { - switch v.cmd { +func (v *Cmd) isSecureScheme(scheme string) bool { + switch v.Cmd { case "git": // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a // colon-separated list of schemes that are allowed to be used with git @@ -89,7 +89,7 @@ type tagCmd struct { } // vcsList lists the known version control systems -var vcsList = []*vcsCmd{ +var vcsList = []*Cmd{ vcsHg, vcsGit, vcsSvn, @@ -97,11 +97,15 @@ var vcsList = []*vcsCmd{ vcsFossil, } +// vcsMod is a stub for the "mod" scheme. It's returned by +// repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. +var vcsMod = &Cmd{Name: "mod"} + // vcsByCmd returns the version control system for the given // command name (hg, git, svn, bzr). -func vcsByCmd(cmd string) *vcsCmd { +func vcsByCmd(cmd string) *Cmd { for _, vcs := range vcsList { - if vcs.cmd == cmd { + if vcs.Cmd == cmd { return vcs } } @@ -109,31 +113,31 @@ func vcsByCmd(cmd string) *vcsCmd { } // vcsHg describes how to use Mercurial. -var vcsHg = &vcsCmd{ - name: "Mercurial", - cmd: "hg", +var vcsHg = &Cmd{ + Name: "Mercurial", + Cmd: "hg", - createCmd: []string{"clone -U -- {repo} {dir}"}, - downloadCmd: []string{"pull"}, + CreateCmd: []string{"clone -U -- {repo} {dir}"}, + DownloadCmd: []string{"pull"}, // We allow both tag and branch names as 'tags' // for selecting a version. This lets people have // a go.release.r60 branch and a go1 branch // and make changes in both, without constantly // editing .hgtags. - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ {"tags", `^(\S+)`}, {"branches", `^(\S+)`}, }, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update default"}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update default"}, - scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify -- {scheme}://{repo}", - remoteRepo: hgRemoteRepo, + Scheme: []string{"https", "http", "ssh"}, + PingCmd: "identify -- {scheme}://{repo}", + RemoteRepo: hgRemoteRepo, } -func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { +func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsHg.runOutput(rootDir, "paths default") if err != nil { return "", err @@ -142,45 +146,45 @@ func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) } // vcsGit describes how to use Git. -var vcsGit = &vcsCmd{ - name: "Git", - cmd: "git", +var vcsGit = &Cmd{ + Name: "Git", + Cmd: "git", - createCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, - downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, - tagCmd: []tagCmd{ + TagCmd: []tagCmd{ // tags/xxx matches a git tag named xxx // origin/xxx matches a git branch named xxx on the default remote repository {"show-ref", `(?:tags|origin)/(\S+)$`}, }, - tagLookupCmd: []tagCmd{ + TagLookupCmd: []tagCmd{ {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, }, - tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, // both createCmd and downloadCmd update the working dir. // No need to do more here. We used to 'checkout master' // but that doesn't work if the default branch is not named master. // DO NOT add 'checkout master' here. // See golang.org/issue/9032. - tagSyncDefault: []string{"submodule update --init --recursive"}, + TagSyncDefault: []string{"submodule update --init --recursive"}, - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not // support such a separator for that command, and this use should be safe // without it because the {scheme} value comes from the predefined list above. // See golang.org/issue/33836. - pingCmd: "ls-remote {scheme}://{repo}", + PingCmd: "ls-remote {scheme}://{repo}", - remoteRepo: gitRemoteRepo, + RemoteRepo: gitRemoteRepo, } // scpSyntaxRe matches the SCP-like addresses used by Git to access // repositories by SSH. var scpSyntaxRe = lazyregexp.New(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) -func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { +func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { cmd := "config remote.origin.url" errParse := errors.New("unable to parse output of git " + cmd) errRemoteOriginNotFound := errors.New("remote origin not found") @@ -216,7 +220,7 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error // Iterate over insecure schemes too, because this function simply // reports the state of the repo. If we can't see insecure schemes then // we can't report the actual repo URL. - for _, s := range vcsGit.scheme { + for _, s := range vcsGit.Scheme { if repoURL.Scheme == s { return repoURL.String(), nil } @@ -225,27 +229,27 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error } // vcsBzr describes how to use Bazaar. -var vcsBzr = &vcsCmd{ - name: "Bazaar", - cmd: "bzr", +var vcsBzr = &Cmd{ + Name: "Bazaar", + Cmd: "bzr", - createCmd: []string{"branch -- {repo} {dir}"}, + CreateCmd: []string{"branch -- {repo} {dir}"}, // Without --overwrite bzr will not pull tags that changed. // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: []string{"pull --overwrite"}, + DownloadCmd: []string{"pull --overwrite"}, - tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update -r revno:-1"}, + TagCmd: []tagCmd{{"tags", `^(\S+)`}}, + TagSyncCmd: []string{"update -r {tag}"}, + TagSyncDefault: []string{"update -r revno:-1"}, - scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: bzrRemoteRepo, - resolveRepo: bzrResolveRepo, + Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: bzrRemoteRepo, + ResolveRepo: bzrResolveRepo, } -func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { +func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "config parent_location") if err != nil { return "", err @@ -253,7 +257,7 @@ func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error return strings.TrimSpace(string(outb)), nil } -func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { +func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) if err != nil { return "", err @@ -287,22 +291,22 @@ func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string } // vcsSvn describes how to use Subversion. -var vcsSvn = &vcsCmd{ - name: "Subversion", - cmd: "svn", +var vcsSvn = &Cmd{ + Name: "Subversion", + Cmd: "svn", - createCmd: []string{"checkout -- {repo} {dir}"}, - downloadCmd: []string{"update"}, + CreateCmd: []string{"checkout -- {repo} {dir}"}, + DownloadCmd: []string{"update"}, // There is no tag command in subversion. // The branch information is all in the path names. - scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info -- {scheme}://{repo}", - remoteRepo: svnRemoteRepo, + Scheme: []string{"https", "http", "svn", "svn+ssh"}, + PingCmd: "info -- {scheme}://{repo}", + RemoteRepo: svnRemoteRepo, } -func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { +func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { outb, err := vcsSvn.runOutput(rootDir, "info") if err != nil { return "", err @@ -337,22 +341,22 @@ func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error const fossilRepoName = ".fossil" // vcsFossil describes how to use Fossil (fossil-scm.org) -var vcsFossil = &vcsCmd{ - name: "Fossil", - cmd: "fossil", +var vcsFossil = &Cmd{ + Name: "Fossil", + Cmd: "fossil", - createCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, - downloadCmd: []string{"up"}, + CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, + DownloadCmd: []string{"up"}, - tagCmd: []tagCmd{{"tag ls", `(.*)`}}, - tagSyncCmd: []string{"up tag:{tag}"}, - tagSyncDefault: []string{"up trunk"}, + TagCmd: []tagCmd{{"tag ls", `(.*)`}}, + TagSyncCmd: []string{"up tag:{tag}"}, + TagSyncDefault: []string{"up trunk"}, - scheme: []string{"https", "http"}, - remoteRepo: fossilRemoteRepo, + Scheme: []string{"https", "http"}, + RemoteRepo: fossilRemoteRepo, } -func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err error) { +func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { out, err := vcsFossil.runOutput(rootDir, "remote-url") if err != nil { return "", err @@ -360,8 +364,8 @@ func fossilRemoteRepo(vcsFossil *vcsCmd, rootDir string) (remoteRepo string, err return strings.TrimSpace(string(out)), nil } -func (v *vcsCmd) String() string { - return v.name +func (v *Cmd) String() string { + return v.Name } // run runs the command line cmd in the given directory. @@ -371,24 +375,24 @@ func (v *vcsCmd) String() string { // If an error occurs, run prints the command line and the // command's combined stdout+stderr to standard error. // Otherwise run discards the command's output. -func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { +func (v *Cmd) run(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, true) return err } // runVerboseOnly is like run but only generates error output to standard error in verbose mode. -func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { +func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { _, err := v.run1(dir, cmd, keyval, false) return err } // runOutput is like run but returns the output of the command. -func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { +func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { return v.run1(dir, cmd, keyval, true) } // run1 is the generalized implementation of run and runOutput. -func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { +func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { m := make(map[string]string) for i := 0; i < len(keyval); i += 2 { m[keyval[i]] = keyval[i+1] @@ -420,25 +424,25 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) args = args[2:] } - _, err := exec.LookPath(v.cmd) + _, err := exec.LookPath(v.Cmd) if err != nil { fmt.Fprintf(os.Stderr, "go: missing %s command. See https://golang.org/s/gogetcmd\n", - v.name) + v.Name) return nil, err } - cmd := exec.Command(v.cmd, args...) + cmd := exec.Command(v.Cmd, args...) cmd.Dir = dir cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir) if cfg.BuildX { fmt.Fprintf(os.Stderr, "cd %s\n", dir) - fmt.Fprintf(os.Stderr, "%s %s\n", v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) } out, err := cmd.Output() if err != nil { if verbose || cfg.BuildV { - fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { os.Stderr.Write(ee.Stderr) } else { @@ -449,15 +453,15 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) return out, err } -// ping pings to determine scheme to use. -func (v *vcsCmd) ping(scheme, repo string) error { - return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) +// Ping pings to determine scheme to use. +func (v *Cmd) Ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo) } -// create creates a new copy of repo in dir. +// Create creates a new copy of repo in dir. // The parent of dir must exist; dir must not. -func (v *vcsCmd) create(dir, repo string) error { - for _, cmd := range v.createCmd { +func (v *Cmd) Create(dir, repo string) error { + for _, cmd := range v.CreateCmd { if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { return err } @@ -465,9 +469,9 @@ func (v *vcsCmd) create(dir, repo string) error { return nil } -// download downloads any new changes for the repo in dir. -func (v *vcsCmd) download(dir string) error { - for _, cmd := range v.downloadCmd { +// Download downloads any new changes for the repo in dir. +func (v *Cmd) Download(dir string) error { + for _, cmd := range v.DownloadCmd { if err := v.run(dir, cmd); err != nil { return err } @@ -475,10 +479,10 @@ func (v *vcsCmd) download(dir string) error { return nil } -// tags returns the list of available tags for the repo in dir. -func (v *vcsCmd) tags(dir string) ([]string, error) { +// Tags returns the list of available tags for the repo in dir. +func (v *Cmd) Tags(dir string) ([]string, error) { var tags []string - for _, tc := range v.tagCmd { + for _, tc := range v.TagCmd { out, err := v.runOutput(dir, tc.cmd) if err != nil { return nil, err @@ -493,12 +497,12 @@ func (v *vcsCmd) tags(dir string) ([]string, error) { // tagSync syncs the repo in dir to the named tag, // which either is a tag returned by tags or is v.tagDefault. -func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == nil { +func (v *Cmd) TagSync(dir, tag string) error { + if v.TagSyncCmd == nil { return nil } if tag != "" { - for _, tc := range v.tagLookupCmd { + for _, tc := range v.TagLookupCmd { out, err := v.runOutput(dir, tc.cmd, "tag", tag) if err != nil { return err @@ -512,8 +516,8 @@ func (v *vcsCmd) tagSync(dir, tag string) error { } } - if tag == "" && v.tagSyncDefault != nil { - for _, cmd := range v.tagSyncDefault { + if tag == "" && v.TagSyncDefault != nil { + for _, cmd := range v.TagSyncDefault { if err := v.run(dir, cmd); err != nil { return err } @@ -521,7 +525,7 @@ func (v *vcsCmd) tagSync(dir, tag string) error { return nil } - for _, cmd := range v.tagSyncCmd { + for _, cmd := range v.TagSyncCmd { if err := v.run(dir, cmd, "tag", tag); err != nil { return err } @@ -540,11 +544,11 @@ type vcsPath struct { schemelessRepo bool // if true, the repo pattern lacks a scheme } -// vcsFromDir inspects dir and its parents to determine the +// FromDir inspects dir and its parents to determine the // version control system and code repository to use. // On return, root is the import path // corresponding to the root of the repository. -func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { +func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { // Clean and double-check that dir is in (a subdirectory of) srcRoot. dir = filepath.Clean(dir) srcRoot = filepath.Clean(srcRoot) @@ -552,13 +556,13 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } - var vcsRet *vcsCmd + var vcsRet *Cmd var rootRet string origDir := dir for len(dir) > len(srcRoot) { for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { root := filepath.ToSlash(dir[len(srcRoot)+1:]) // Record first VCS we find, but keep looking, // to detect mistakes like one kind of VCS inside another. @@ -568,12 +572,12 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { continue } // Allow .git inside .git, which can arise due to submodules. - if vcsRet == vcs && vcs.cmd == "git" { + if vcsRet == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s", - filepath.Join(srcRoot, rootRet), vcsRet.cmd, filepath.Join(srcRoot, root), vcs.cmd) + filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd) } } @@ -593,9 +597,9 @@ func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) } -// checkNestedVCS checks for an incorrectly-nested VCS-inside-VCS +// CheckNested checks for an incorrectly-nested VCS-inside-VCS // situation for dir, checking parents up until srcRoot. -func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { +func CheckNested(vcs *Cmd, dir, srcRoot string) error { if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) } @@ -603,17 +607,17 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { otherDir := dir for len(otherDir) > len(srcRoot) { for _, otherVCS := range vcsList { - if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.cmd)); err == nil { + if _, err := os.Stat(filepath.Join(otherDir, "."+otherVCS.Cmd)); err == nil { // Allow expected vcs in original dir. if otherDir == dir && otherVCS == vcs { continue } // Allow .git inside .git, which can arise due to submodules. - if otherVCS == vcs && vcs.cmd == "git" { + if otherVCS == vcs && vcs.Cmd == "git" { continue } // Otherwise, we have one VCS inside a different VCS. - return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.cmd, otherDir, otherVCS.cmd) + return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) } } // Move to parent. @@ -633,9 +637,7 @@ type RepoRoot struct { Repo string // repository URL, including scheme Root string // import path corresponding to root of repo IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) - VCS string // vcs type ("mod", "git", ...) - - vcs *vcsCmd // internal: vcs command access + VCS *Cmd } func httpPrefix(s string) string { @@ -735,15 +737,15 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if !srv.schemelessRepo { repoURL = match["repo"] } else { - scheme := vcs.scheme[0] // default to first scheme + scheme := vcs.Scheme[0] // default to first scheme repo := match["repo"] - if vcs.pingCmd != "" { + if vcs.PingCmd != "" { // If we know how to test schemes, scan to find one. - for _, s := range vcs.scheme { + for _, s := range vcs.Scheme { if security == web.SecureOnly && !vcs.isSecureScheme(s) { continue } - if vcs.ping(s, repo) == nil { + if vcs.Ping(s, repo) == nil { scheme = s break } @@ -754,8 +756,7 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths rr := &RepoRoot{ Repo: repoURL, Root: match["root"], - VCS: vcs.cmd, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -846,17 +847,21 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se if err := validateRepoRoot(mmi.RepoRoot); err != nil { return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) } - vcs := vcsByCmd(mmi.VCS) - if vcs == nil && mmi.VCS != "mod" { - return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + var vcs *Cmd + if mmi.VCS == "mod" { + vcs = vcsMod + } else { + vcs = vcsByCmd(mmi.VCS) + if vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) + } } rr := &RepoRoot{ Repo: mmi.RepoRoot, Root: mmi.Prefix, IsCustom: true, - VCS: mmi.VCS, - vcs: vcs, + VCS: vcs, } return rr, nil } @@ -1027,7 +1032,7 @@ var vcsPaths = []*vcsPath{ // Github { prefix: "github.com/", - regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`), + regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`), vcs: "git", repo: "https://{root}", check: noVCSSuffix, @@ -1103,7 +1108,7 @@ var vcsPathsAfterDynamic = []*vcsPath{ func noVCSSuffix(match map[string]string) error { repo := match["repo"] for _, vcs := range vcsList { - if strings.HasSuffix(repo, "."+vcs.cmd) { + if strings.HasSuffix(repo, "."+vcs.Cmd) { return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) } } @@ -1133,7 +1138,7 @@ func bitbucketVCS(match map[string]string) error { // VCS it uses. See issue 5375. root := match["root"] for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).ping("https", root) == nil { + if vcsByCmd(vcs).Ping("https", root) == nil { resp.SCM = vcs break } diff --git a/src/cmd/go/internal/get/vcs_test.go b/src/cmd/go/internal/vcs/vcs_test.go index 91800baa83..5b874204f1 100644 --- a/src/cmd/go/internal/get/vcs_test.go +++ b/src/cmd/go/internal/vcs/vcs_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package get +package vcs import ( "errors" @@ -28,30 +28,27 @@ func TestRepoRootForImportPath(t *testing.T) { { "github.com/golang/groupcache", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://github.com/golang/groupcache", }, }, - // Unicode letters in directories (issue 18660). + // Unicode letters in directories are not valid. { "github.com/user/unicode/испытание", - &RepoRoot{ - vcs: vcsGit, - Repo: "https://github.com/user/unicode", - }, + nil, }, // IBM DevOps Services tests { "hub.jazz.net/git/user1/pkgname", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, { "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user1/pkgname", }, }, @@ -92,7 +89,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "hub.jazz.net/git/user/pkg.name", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://hub.jazz.net/git/user/pkg.name", }, }, @@ -105,7 +102,7 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -115,14 +112,14 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.openstack.org/openstack/swift.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift.git", }, }, { "git.openstack.org/openstack/swift/go/hummingbird", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.openstack.org/openstack/swift", }, }, @@ -151,21 +148,21 @@ func TestRepoRootForImportPath(t *testing.T) { { "git.apache.org/package-name.git", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name.git", }, }, { "git.apache.org/package-name_2.x.git/path/to/lib", &RepoRoot{ - vcs: vcsGit, + VCS: vcsGit, Repo: "https://git.apache.org/package-name_2.x.git", }, }, { "chiselapp.com/user/kyle/repository/fossilgg", &RepoRoot{ - vcs: vcsFossil, + VCS: vcsFossil, Repo: "https://chiselapp.com/user/kyle/repository/fossilgg", }, }, @@ -194,8 +191,8 @@ func TestRepoRootForImportPath(t *testing.T) { t.Errorf("RepoRootForImportPath(%q): %v", test.path, err) continue } - if got.vcs.name != want.vcs.name || got.Repo != want.Repo { - t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo) + if got.VCS.Name != want.VCS.Name || got.Repo != want.Repo { + t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.VCS, got.Repo, want.VCS, want.Repo) } } } @@ -209,7 +206,7 @@ func TestFromDir(t *testing.T) { defer os.RemoveAll(tempDir) for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) + dir := filepath.Join(tempDir, "example.com", vcs.Name, "."+vcs.Cmd) if j&1 == 0 { err := os.MkdirAll(dir, 0755) if err != nil { @@ -228,24 +225,24 @@ func TestFromDir(t *testing.T) { } want := RepoRoot{ - vcs: vcs, - Root: path.Join("example.com", vcs.name), + VCS: vcs, + Root: path.Join("example.com", vcs.Name), } var got RepoRoot - got.vcs, got.Root, err = vcsFromDir(dir, tempDir) + got.VCS, got.Root, err = FromDir(dir, tempDir) if err != nil { t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) continue } - if got.vcs.name != want.vcs.name || got.Root != want.Root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root) + if got.VCS.Name != want.VCS.Name || got.Root != want.Root { + t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.VCS, got.Root, want.VCS, want.Root) } } } func TestIsSecure(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -270,7 +267,7 @@ func TestIsSecure(t *testing.T) { } for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } @@ -279,7 +276,7 @@ func TestIsSecure(t *testing.T) { func TestIsSecureGitAllowProtocol(t *testing.T) { tests := []struct { - vcs *vcsCmd + vcs *Cmd url string secure bool }{ @@ -310,7 +307,7 @@ func TestIsSecureGitAllowProtocol(t *testing.T) { defer os.Unsetenv("GIT_ALLOW_PROTOCOL") os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") for _, test := range tests { - secure := test.vcs.isSecure(test.url) + secure := test.vcs.IsSecure(test.url) if secure != test.secure { t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index d020aa6e9f..e99982ed36 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -240,13 +240,12 @@ const ( // AddBuildFlags adds the flags common to the build, clean, get, // install, list, run, and test commands. func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { + base.AddBuildFlagsNX(&cmd.Flag) cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") - cmd.Flag.BoolVar(&cfg.BuildN, "n", false, "") cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") if mask&OmitVFlag == 0 { cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") } - cmd.Flag.BoolVar(&cfg.BuildX, "x", false, "") cmd.Flag.Var(&load.BuildAsmflags, "asmflags", "") cmd.Flag.Var(buildCompiler{}, "compiler", "") @@ -254,10 +253,10 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.Var(&load.BuildGcflags, "gcflags", "") cmd.Flag.Var(&load.BuildGccgoflags, "gccgoflags", "") if mask&OmitModFlag == 0 { - cmd.Flag.StringVar(&cfg.BuildMod, "mod", "", "") + base.AddModFlag(&cmd.Flag) } if mask&OmitModCommonFlags == 0 { - AddModCommonFlags(cmd) + base.AddModCommonFlags(&cmd.Flag) } cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") cmd.Flag.Var(&load.BuildLdflags, "ldflags", "") @@ -275,13 +274,6 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") } -// AddModCommonFlags adds the module-related flags common to build commands -// and 'go mod' subcommands. -func AddModCommonFlags(cmd *base.Command) { - cmd.Flag.BoolVar(&cfg.ModCacheRW, "modcacherw", false, "") - cmd.Flag.StringVar(&cfg.ModFile, "modfile", "", "") -} - // tagsFlag is the implementation of the -tags flag. type tagsFlag []string diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index f1d08e0268..6031897f88 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -259,6 +259,15 @@ func asmArgs(a *Action, p *load.Package) []interface{} { } } } + if p.ImportPath == "runtime" && objabi.Regabi_enabled != 0 { + // In order to make it easier to port runtime assembly + // to the register ABI, we introduce a macro + // indicating the experiment is enabled. + // + // TODO(austin): Remove this once we commit to the + // register ABI (#40724). + args = append(args, "-D=GOEXPERIMENT_REGABI=1") + } if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { // Define GOMIPS_value from cfg.GOMIPS. diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index dad3b10111..f78020032c 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -252,7 +252,7 @@ func buildModeInit() { switch cfg.BuildMod { case "": - // ok + // Behavior will be determined automatically, as if no flag were passed. case "readonly", "vendor", "mod": if !cfg.ModulesEnabled && !inGOFLAGS("-mod") { base.Fatalf("build flag -mod=%s only valid when using modules", cfg.BuildMod) diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index 3ee68ac1b4..d2a2697f0f 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -177,6 +177,7 @@ var validLinkerFlags = []*lazyregexp.Regexp{ re(`-Wl,-Bdynamic`), re(`-Wl,-berok`), re(`-Wl,-Bstatic`), + re(`-Wl,-Bsymbolic-functions`), re(`-WL,-O([^@,\-][^,]*)?`), re(`-Wl,-d[ny]`), re(`-Wl,--disable-new-dtags`), diff --git a/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.0.0.txt b/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.0.0.txt new file mode 100644 index 0000000000..1d8d81071e --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.0.0.txt @@ -0,0 +1,10 @@ +This version should be retracted, but the go.mod file for the version that would +contain the retraction is not available. +-- .mod -- +module example.com/retract/missingmod + +go 1.14 +-- .info -- +{"Version":"v1.0.0"} +-- missingmod.go -- +package missingmod diff --git a/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.9.0.txt new file mode 100644 index 0000000000..bba919ec21 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_missingmod_v1.9.0.txt @@ -0,0 +1,4 @@ +The go.mod file at this version will be loaded to check for retractions +of earlier versions. However, the .mod file is not available. +-- .info -- +{"Version":"v1.9.0"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-block.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-block.txt new file mode 100644 index 0000000000..c4a53e1d80 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-block.txt @@ -0,0 +1,6 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-block"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-blockwithcomment.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-blockwithcomment.txt new file mode 100644 index 0000000000..92573b62e3 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-blockwithcomment.txt @@ -0,0 +1,6 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-blockwithcomment"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-empty.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-empty.txt new file mode 100644 index 0000000000..1f0894aa8b --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-empty.txt @@ -0,0 +1,8 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-empty"} +-- empty.go -- +package empty diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-long.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-long.txt new file mode 100644 index 0000000000..1b5e753428 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-long.txt @@ -0,0 +1,8 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-long"} +-- empty.go -- +package empty diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline1.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline1.txt new file mode 100644 index 0000000000..b1ffe27225 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline1.txt @@ -0,0 +1,8 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-multiline1"} +-- empty.go -- +package empty diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline2.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline2.txt new file mode 100644 index 0000000000..72f80b3254 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-multiline2.txt @@ -0,0 +1,8 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-multiline2"} +-- empty.go -- +package empty diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-order.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-order.txt new file mode 100644 index 0000000000..1b0450462b --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-order.txt @@ -0,0 +1,6 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-order"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-unprintable.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-unprintable.txt new file mode 100644 index 0000000000..949612431e --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.0-unprintable.txt @@ -0,0 +1,8 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.0-unprintable"} +-- empty.go -- +package empty diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.1-order.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.1-order.txt new file mode 100644 index 0000000000..3be7d5b56e --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.0.1-order.txt @@ -0,0 +1,6 @@ +-- .mod -- +module example.com/retract/rationale + +go 1.14 +-- .info -- +{"Version":"v1.0.1-order"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.9.0.txt new file mode 100644 index 0000000000..6975d4ebd4 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_rationale_v1.9.0.txt @@ -0,0 +1,48 @@ +Module example.com/retract/description retracts all versions of itself. +The rationale comments have various problems. + +-- .mod -- +module example.com/retract/rationale + +go 1.14 + +retract ( + v1.0.0-empty + + // short description + // more + // + // detail + v1.0.0-multiline1 // suffix + // after not included +) + +// short description +// more +// +// detail +retract v1.0.0-multiline2 // suffix + +// loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong +retract v1.0.0-long + +// Ends with a BEL character. Beep! +retract v1.0.0-unprintable + +// block comment +retract ( + v1.0.0-block + + // inner comment + v1.0.0-blockwithcomment +) + +retract ( + [v1.0.0-order, v1.0.0-order] // degenerate range + v1.0.0-order // single version + + v1.0.1-order // single version + [v1.0.1-order, v1.0.1-order] // degenerate range +) +-- .info -- +{"Version":"v1.9.0"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_all_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_all_v1.9.0.txt new file mode 100644 index 0000000000..4dc486b599 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_all_v1.9.0.txt @@ -0,0 +1,14 @@ +Module example.com/retract/self/prev is a module that retracts its own +latest version. + +No unretracted versions are available. + +-- .mod -- +module example.com/retract/self/all + +go 1.15 + +retract v1.9.0 // bad + +-- .info -- +{"Version":"v1.9.0"} diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.0.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.0.0.txt new file mode 100644 index 0000000000..04c28455d7 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.0.0.txt @@ -0,0 +1,16 @@ +Module example.com/retract/self/prerelease is a module that retracts its own +latest version and all other release version. + +A pre-release version higher than the highest release version is still +available, and that should be matched by @latest. + +-- .mod -- +module example.com/retract/self/prerelease + +go 1.15 + +-- .info -- +{"Version":"v1.0.0"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.0.txt new file mode 100644 index 0000000000..7c1c047e69 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.0.txt @@ -0,0 +1,19 @@ +Module example.com/retract/self/prerelease is a module that retracts its own +latest version and all other release version. + +A pre-release version higher than the highest release version is still +available, and that should be matched by @latest. + +-- .mod -- +module example.com/retract/self/prerelease + +go 1.15 + +retract v1.0.0 // bad +retract v1.9.0 // self + +-- .info -- +{"Version":"v1.9.0"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.1-pre.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.1-pre.txt new file mode 100644 index 0000000000..abf44fdae1 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prerelease_v1.9.1-pre.txt @@ -0,0 +1,16 @@ +Module example.com/retract/self/prerelease is a module that retracts its own +latest version and all other release version. + +A pre-release version higher than the highest release version is still +available, and that should be matched by @latest. + +-- .mod -- +module example.com/retract/self/prerelease + +go 1.15 + +-- .info -- +{"Version":"v1.9.1-pre"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.0.0-bad.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.0.0-bad.txt new file mode 100644 index 0000000000..095063d69b --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.0.0-bad.txt @@ -0,0 +1,14 @@ +See example.com_retract_self_prev_v1.9.0.txt. + +This version is retracted. + +-- .mod -- +module example.com/retract/self/prev + +go 1.15 + +-- .info -- +{"Version":"v1.0.0-bad"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.1.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.1.0.txt new file mode 100644 index 0000000000..27c3a39065 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.1.0.txt @@ -0,0 +1,14 @@ +See example.com_retract_self_pref_v1.9.0.txt. + +This version is the latest (only) non-retracted version. + +-- .mod -- +module example.com/retract/self/prev + +go 1.15 + +-- .info -- +{"Version":"v1.1.0"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.9.0.txt new file mode 100644 index 0000000000..03d6168f0d --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_prev_v1.9.0.txt @@ -0,0 +1,18 @@ +Module example.com/retract/self/prev is a module that retracts its own +latest version, as well as an earlier version. + +A previous unretracted release version, v1.1.0, is still available. + +-- .mod -- +module example.com/retract/self/prev + +go 1.15 + +retract v1.0.0-bad // bad +retract v1.9.0 // self + +-- .info -- +{"Version":"v1.9.0"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v0.0.0-20200325131415-0123456789ab b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v0.0.0-20200325131415-0123456789ab new file mode 100644 index 0000000000..f9ab41e88f --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v0.0.0-20200325131415-0123456789ab @@ -0,0 +1,20 @@ +See example.com_retract_self_pseudo_v1.9.0.txt. + +This version is not retracted. It should be returned by the proxy's +@latest endpoint. It should match the @latest version query. + +TODO(golang.org/issue/24031): the proxy and proxy.golang.org both return +the highest release version from the @latest endpoint, even if that +version is retracted, so there is no way for the go command to +discover an unretracted pseudo-version. + +-- .mod -- +module example.com/retract/self/pseudo + +go 1.15 + +-- .info -- +{"Version":"v0.0.0-20200325131415-01234567890ab"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.0.0-bad.txt b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.0.0-bad.txt new file mode 100644 index 0000000000..d47eda0597 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.0.0-bad.txt @@ -0,0 +1,14 @@ +See example.com_retract_self_pseudo_v1.9.0.txt. + +This version is retracted. + +-- .mod -- +module example.com/retract/self/pseudo + +go 1.15 + +-- .info -- +{"Version":"v1.0.0-bad"} + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.9.0.txt b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.9.0.txt new file mode 100644 index 0000000000..db09cc6a5f --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_self_pseudo_v1.9.0.txt @@ -0,0 +1,16 @@ +Module example.com/retract/self/pseudo is a module that retracts its own +latest version, as well as an earlier version. + +An unretracted pseudo-version is available. + +-- .mod -- +module example.com/retract/self/pseudo + +go 1.15 + +retract v1.0.0-bad // bad +retract v1.9.0 // self + +-- .info -- +{"Version":"v1.9.0"} + diff --git a/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-bad.txt b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-bad.txt new file mode 100644 index 0000000000..2f996cfc36 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-bad.txt @@ -0,0 +1,10 @@ +-- .mod -- +module example.com/retract + +go 1.15 + +-- .info -- +{"Version":"v1.0.0-bad"} + +-- retract.go -- +package retract diff --git a/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-good.txt b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-good.txt new file mode 100644 index 0000000000..78152bba4f --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-good.txt @@ -0,0 +1,10 @@ +-- .mod -- +module example.com/retract + +go 1.15 + +-- .info -- +{"Version":"v1.0.0-good"} + +-- retract.go -- +package retract diff --git a/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-unused.txt b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-unused.txt new file mode 100644 index 0000000000..3bc9e35b7c --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_v1.0.0-unused.txt @@ -0,0 +1,10 @@ +-- .mod -- +module example.com/retract + +go 1.15 + +-- .info -- +{"Version":"v1.0.0-unused"} + +-- retract.go -- +package retract diff --git a/src/cmd/go/testdata/mod/example.com_retract_v1.1.0.txt b/src/cmd/go/testdata/mod/example.com_retract_v1.1.0.txt new file mode 100644 index 0000000000..18d6d832e2 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_retract_v1.1.0.txt @@ -0,0 +1,13 @@ +-- .mod -- +module example.com/retract + +go 1.15 + +retract v1.0.0-bad // bad +retract v1.0.0-unused // bad + +-- .info -- +{"Version":"v1.1.0"} + +-- retract.go -- +package retract diff --git a/src/cmd/go/testdata/mod/example.com_split-incompatible_subpkg_v0.1.0.txt b/src/cmd/go/testdata/mod/example.com_split-incompatible_subpkg_v0.1.0.txt new file mode 100644 index 0000000000..8f9e49176c --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_split-incompatible_subpkg_v0.1.0.txt @@ -0,0 +1,14 @@ +Written by hand. +Test case for getting a package that has been moved to a nested module, +with a +incompatible verison (and thus no go.mod file) at the root module. + +-- .mod -- +module example.com/split-incompatible/subpkg +-- .info -- +{"Version": "v0.1.0"} +-- go.mod -- +module example.com/split-incompatible/subpkg + +go 1.16 +-- subpkg.go -- +package subpkg diff --git a/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.0.0+incompatible.txt b/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.0.0+incompatible.txt new file mode 100644 index 0000000000..35c3f27710 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.0.0+incompatible.txt @@ -0,0 +1,10 @@ +Written by hand. +Test case for getting a package that has been moved to a nested module, +with a +incompatible verison (and thus no go.mod file) at the root module. + +-- .mod -- +module example.com/split-incompatible +-- .info -- +{"Version": "v2.0.0+incompatible"} +-- subpkg/subpkg.go -- +package subpkg diff --git a/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.1.0-pre+incompatible.txt b/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.1.0-pre+incompatible.txt new file mode 100644 index 0000000000..917fc0f559 --- /dev/null +++ b/src/cmd/go/testdata/mod/example.com_split-incompatible_v2.1.0-pre+incompatible.txt @@ -0,0 +1,10 @@ +Written by hand. +Test case for getting a package that has been moved to a nested module, +with a +incompatible verison (and thus no go.mod file) at the root module. + +-- .mod -- +module example.com/split-incompatible +-- .info -- +{"Version": "v2.1.0-pre+incompatible"} +-- README.txt -- +subpkg has moved to module example.com/split-incompatible/subpkg diff --git a/src/cmd/go/testdata/script/get_insecure_env.txt b/src/cmd/go/testdata/script/get_insecure_env.txt new file mode 100644 index 0000000000..8d88427c31 --- /dev/null +++ b/src/cmd/go/testdata/script/get_insecure_env.txt @@ -0,0 +1,29 @@ +[!net] skip +[!exec:git] skip + +# GOPATH: Set up +env GO111MODULE=off + +# GOPATH: Try go get -d of HTTP-only repo (should fail). +! go get -d insecure.go-get-issue-15410.appspot.com/pkg/p + +# GOPATH: Try again with invalid GOINSECURE (should fail). +env GOINSECURE=insecure.go-get-issue-15410.appspot.com/pkg/q +! go get -d insecure.go-get-issue-15410.appspot.com/pkg/p + +# GOPATH: Try with correct GOINSECURE (should succeed). +env GOINSECURE=insecure.go-get-issue-15410.appspot.com/pkg/p +go get -d insecure.go-get-issue-15410.appspot.com/pkg/p + +# GOPATH: Try updating without GOINSECURE (should fail). +env GOINSECURE= +! go get -d -u -f insecure.go-get-issue-15410.appspot.com/pkg/p + +# GOPATH: Try updating with GOINSECURE glob (should succeed). +env GOINSECURE=*.go-get-*.appspot.com +go get -d -u -f insecure.go-get-issue-15410.appspot.com/pkg/p + +# GOPATH: Try updating with GOINSECURE base URL (should succeed). +env GOINSECURE=insecure.go-get-issue-15410.appspot.com +go get -d -u -f insecure.go-get-issue-15410.appspot.com/pkg/p + diff --git a/src/cmd/go/testdata/script/get_unicode.txt b/src/cmd/go/testdata/script/get_unicode.txt deleted file mode 100644 index d3b82bdf25..0000000000 --- a/src/cmd/go/testdata/script/get_unicode.txt +++ /dev/null @@ -1,40 +0,0 @@ -env GO111MODULE=off - -[!exec:git] skip -[short] skip - -# Construct a repository that imports a non-ASCII path. -cd $WORK/_origin/example.com/unicode -exec git init -exec git config user.name 'Nameless Gopher' -exec git config user.email 'nobody@golang.org' -exec git add unicode.go -exec git commit -m 'add unicode.go' - -# Clone the repo into GOPATH so that 'go get -u' can find it. -mkdir $GOPATH/src/example.com/unicode -cd $GOPATH/src/example.com/unicode -exec git clone $WORK/_origin/example.com/unicode . - -# Construct the imported repository. -cd $WORK/_origin/example.com/испытание -exec git init -exec git config user.name 'Nameless Gopher' -exec git config user.email 'nobody@golang.org' -exec git add испытание.go -exec git commit -m 'add испытание.go' - -# Clone that repo into GOPATH too. -mkdir $GOPATH/src/example.com/испытание -cd $GOPATH/src/example.com/испытание -exec git clone $WORK/_origin/example.com/испытание . - -# Upgrading the importer should pull from the non-ASCII repo. -cd $GOPATH -go get -u example.com/unicode - --- $WORK/_origin/example.com/unicode/unicode.go -- -package unicode -import _ "example.com/испытание" --- $WORK/_origin/example.com/испытание/испытание.go -- -package испытание diff --git a/src/cmd/go/testdata/script/list_bad_import.txt b/src/cmd/go/testdata/script/list_bad_import.txt index b8f9d586f3..dbec35069c 100644 --- a/src/cmd/go/testdata/script/list_bad_import.txt +++ b/src/cmd/go/testdata/script/list_bad_import.txt @@ -15,10 +15,11 @@ stdout 'incomplete' stdout 'bad dep: .*example.com[/\\]notfound' # Listing with -deps should also fail. -# BUG: Today, it does not. -# ! go list -deps example.com/direct -# stderr example.com[/\\]notfound -go list -deps example.com/direct +! go list -deps example.com/direct +stderr example.com[/\\]notfound + +# But -e -deps should succeed. +go list -e -deps example.com/direct stdout example.com/notfound @@ -31,10 +32,11 @@ stdout incomplete stdout 'bad dep: .*example.com[/\\]notfound' # Again, -deps should fail. -# BUG: Again, it does not. -# ! go list -deps example.com/indirect -# stderr example.com[/\\]notfound -go list -deps example.com/indirect +! go list -deps example.com/indirect +stderr example.com[/\\]notfound + +# But -deps -e should succeed. +go list -e -deps example.com/indirect stdout example.com/notfound diff --git a/src/cmd/go/testdata/script/list_test_err.txt b/src/cmd/go/testdata/script/list_test_err.txt index a174b5e9ad..c6f1ecf400 100644 --- a/src/cmd/go/testdata/script/list_test_err.txt +++ b/src/cmd/go/testdata/script/list_test_err.txt @@ -22,6 +22,9 @@ go list -e -test -deps -f '{{.ImportPath}} {{.Error | printf "%q"}}' syntaxerr stdout 'pkgdep <nil>' stdout 'testdep_a <nil>' stdout 'testdep_b <nil>' +stdout 'syntaxerr <nil>' +stdout 'syntaxerr \[syntaxerr.test\] <nil>' +stdout 'syntaxerr_test \[syntaxerr.test\] <nil>' stdout 'syntaxerr\.test "[^"]*expected declaration' ! stderr 'expected declaration' diff --git a/src/cmd/go/testdata/script/mod_all.txt b/src/cmd/go/testdata/script/mod_all.txt new file mode 100644 index 0000000000..aac66292d6 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_all.txt @@ -0,0 +1,444 @@ +# This test illustrates the relationship between the 'all' pattern and +# the dependencies of the main module. + +# The package import graph used in this test looks like: +# +# main --------- a --------- b +# | | +# | a_test ---- c +# | | +# | c_test ---- d +# | +# main_test ---- t --------- u +# | +# t_test ---- w +# | +# w_test ---- x +# +# main/testonly_test ---- q --------- r +# | +# q_test ---- s +# +# And the module dependency graph looks like: +# +# main --- a.1 ---- b.1 +# \ \ \ +# \ \ c.1 -- d.1 +# \ \ +# \ t.1 ---- u.1 +# \ \ +# \ w.1 -- x.1 +# \ +# q.1 ---- r.1 +# \ +# s.1 + +env PKGFMT='{{if .Module}}{{.ImportPath}}{{end}}' +env MODFMT='{{.Path}}' + + +# 'go list -deps' lists packages and tests in the main module, +# along with their transitive dependencies. + +go list -f $PKGFMT -deps ./... +stdout -count=4 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly' + + +# 'go list -deps -test' lists transitive imports of tests and non-tests in the +# main module. + +go list -f $PKGFMT -deps -test ./... +stdout -count=13 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main.test$' +stdout '^example.com/main \[example.com/main.test\]$' +stdout '^example.com/main_test \[example.com/main.test\]$' +stdout '^example.com/main/testonly$' +stdout '^example.com/main/testonly.test$' +stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/t$' +stdout '^example.com/u$' + + +# 'go list all' lists the fixpoint of iterating 'go list -deps -test' starting +# with the packages in the main module, then reducing to only the non-test +# variants of those packages. + +go list -f $PKGFMT all +stdout -count=13 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/c$' +stdout '^example.com/d$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/s$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/w$' +stdout '^example.com/x$' + + +# 'go list -test all' is equivalent to 'go list -test $(go list all)' +# and both should include tests for every package in 'all'. + +go list -test -f $PKGFMT example.com/a example.com/b example.com/c example.com/d example.com/main example.com/main/testonly example.com/q example.com/r example.com/s example.com/t example.com/u example.com/w example.com/x +cp stdout list-test-explicit.txt + +go list -test -f $PKGFMT all +cmp stdout list-test-explicit.txt +stdout -count=36 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/c$' +stdout '^example.com/d$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/s$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/w$' +stdout '^example.com/x$' +stdout '^example.com/a.test$' +stdout '^example.com/a_test \[example.com/a.test\]$' +stdout '^example.com/b.test$' +stdout '^example.com/b_test \[example.com/b.test\]$' +stdout '^example.com/c.test$' +stdout '^example.com/c_test \[example.com/c.test\]$' +stdout '^example.com/main.test$' +stdout '^example.com/main \[example.com/main.test\]$' +stdout '^example.com/main_test \[example.com/main.test\]$' +stdout '^example.com/main/testonly.test$' +stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$' +stdout '^example.com/q.test$' +stdout '^example.com/q_test \[example.com/q.test\]$' +stdout '^example.com/r.test$' +stdout '^example.com/r_test \[example.com/r.test\]$' +stdout '^example.com/s.test$' +stdout '^example.com/s_test \[example.com/s.test\]$' +stdout '^example.com/t.test$' +stdout '^example.com/t_test \[example.com/t.test\]$' +stdout '^example.com/u.test$' +stdout '^example.com/u_test \[example.com/u.test\]$' +stdout '^example.com/w.test$' +stdout '^example.com/w_test \[example.com/w.test\]$' + + +# 'go list -m all' covers the packages in 'go list -test -deps all'. + +go list -m -f $MODFMT all +stdout -count=12 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/c$' +stdout '^example.com/d$' +stdout '^example.com/main$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/s$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/w$' +stdout '^example.com/x$' + + +# 'go mod vendor' copies in only the packages transitively imported by the main +# module, and omits their tests. As a result, the 'all' and '...' patterns +# report fewer packages when using '-mod=vendor'. + +go mod vendor + +go list -f $PKGFMT -mod=vendor all +stdout -count=8 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/t$' +stdout '^example.com/u$' + +go list -test -f $PKGFMT -mod=vendor all +stdout -count=13 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/main.test$' +stdout '^example.com/main \[example.com/main.test\]$' +stdout '^example.com/main_test \[example.com/main.test\]$' +stdout '^example.com/main/testonly.test$' +stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$' + +rm vendor + +# Convert all modules to go 1.16 to enable lazy loading. +go mod edit -go=1.16 a/go.mod +go mod edit -go=1.16 b/go.mod +go mod edit -go=1.16 c/go.mod +go mod edit -go=1.16 d/go.mod +go mod edit -go=1.16 q/go.mod +go mod edit -go=1.16 r/go.mod +go mod edit -go=1.16 s/go.mod +go mod edit -go=1.16 t/go.mod +go mod edit -go=1.16 u/go.mod +go mod edit -go=1.16 w/go.mod +go mod edit -go=1.16 x/go.mod +go mod edit -go=1.16 + +# With lazy loading, 'go list all' with neither -mod=vendor nor -test should +# match -mod=vendor without -test in 1.15. + +go list -f $PKGFMT all +stdout -count=8 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/t$' +stdout '^example.com/u$' + +# 'go list -test all' should expand that to include the test variants of the +# packages in 'all', but not the dependencies of outside tests. + +go list -test -f $PKGFMT all +stdout -count=25 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/a.test$' +stdout '^example.com/a_test \[example.com/a.test\]$' +stdout '^example.com/b.test$' +stdout '^example.com/b_test \[example.com/b.test\]$' +stdout '^example.com/main.test$' +stdout '^example.com/main \[example.com/main.test\]$' +stdout '^example.com/main_test \[example.com/main.test\]$' +stdout '^example.com/main/testonly.test$' +stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$' +stdout '^example.com/q.test$' +stdout '^example.com/q_test \[example.com/q.test\]$' +stdout '^example.com/r.test$' +stdout '^example.com/r_test \[example.com/r.test\]$' +stdout '^example.com/t.test$' +stdout '^example.com/t_test \[example.com/t.test\]$' +stdout '^example.com/u.test$' +stdout '^example.com/u_test \[example.com/u.test\]$' + +# 'go list -test -deps all' should include the dependencies of those tests, +# but not the tests of the dependencies of outside tests. + +go list -test -deps -f $PKGFMT all +stdout -count=28 '^.' +stdout '^example.com/a$' +stdout '^example.com/b$' +stdout '^example.com/c$' +stdout '^example.com/main$' +stdout '^example.com/main/testonly$' +stdout '^example.com/q$' +stdout '^example.com/r$' +stdout '^example.com/s$' +stdout '^example.com/t$' +stdout '^example.com/u$' +stdout '^example.com/w$' +stdout '^example.com/a.test$' +stdout '^example.com/a_test \[example.com/a.test\]$' +stdout '^example.com/b.test$' +stdout '^example.com/b_test \[example.com/b.test\]$' +stdout '^example.com/main.test$' +stdout '^example.com/main \[example.com/main.test\]$' +stdout '^example.com/main_test \[example.com/main.test\]$' +stdout '^example.com/main/testonly.test$' +stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$' +stdout '^example.com/q.test$' +stdout '^example.com/q_test \[example.com/q.test\]$' +stdout '^example.com/r.test$' +stdout '^example.com/r_test \[example.com/r.test\]$' +stdout '^example.com/t.test$' +stdout '^example.com/t_test \[example.com/t.test\]$' +stdout '^example.com/u.test$' +stdout '^example.com/u_test \[example.com/u.test\]$' + + +# TODO(#36460): +# 'go list -m all' should exactly cover the packages in 'go list -test all'. + +-- go.mod -- +module example.com/main + +go 1.15 + +require ( + example.com/a v0.1.0 + example.com/b v0.1.0 + example.com/q v0.1.0 + example.com/t v0.1.0 +) + +replace ( + example.com/a v0.1.0 => ./a + example.com/b v0.1.0 => ./b + example.com/c v0.1.0 => ./c + example.com/d v0.1.0 => ./d + example.com/q v0.1.0 => ./q + example.com/r v0.1.0 => ./r + example.com/s v0.1.0 => ./s + example.com/t v0.1.0 => ./t + example.com/u v0.1.0 => ./u + example.com/w v0.1.0 => ./w + example.com/x v0.1.0 => ./x +) +-- main.go -- +package main + +import _ "example.com/a" + +func main() {} +-- main_test.go -- +package main_test + +import _ "example.com/t" +-- testonly/testonly_test.go -- +package testonly_test + +import _ "example.com/q" +-- a/go.mod -- +module example.com/a + +go 1.15 + +require ( + example.com/b v0.1.0 + example.com/c v0.1.0 +) +-- a/a.go -- +package a + +import _ "example.com/b" +-- a/a_test.go -- +package a_test + +import _ "example.com/c" +-- b/go.mod -- +module example.com/b + +go 1.15 +-- b/b.go -- +package b +-- b/b_test.go -- +package b_test +-- c/go.mod -- +module example.com/c + +go 1.15 + +require example.com/d v0.1.0 +-- c/c.go -- +package c +-- c/c_test.go -- +package c_test + +import _ "example.com/d" +-- d/go.mod -- +module example.com/d + +go 1.15 +-- d/d.go -- +package d +-- q/go.mod -- +module example.com/q + +go 1.15 + +require ( + example.com/r v0.1.0 + example.com/s v0.1.0 +) +-- q/q.go -- +package q +import _ "example.com/r" +-- q/q_test.go -- +package q_test +import _ "example.com/s" +-- r/go.mod -- +module example.com/r + +go 1.15 +-- r/r.go -- +package r +-- r/r_test.go -- +package r_test +-- s/go.mod -- +module example.com/s + +go 1.15 +-- s/s.go -- +package s +-- s/s_test.go -- +package s_test +-- t/go.mod -- +module example.com/t + +go 1.15 + +require ( + example.com/u v0.1.0 + example.com/w v0.1.0 +) +-- t/t.go -- +package t + +import _ "example.com/u" +-- t/t_test.go -- +package t_test + +import _ "example.com/w" +-- u/go.mod -- +module example.com/u + +go 1.15 +-- u/u.go -- +package u +-- u/u_test.go -- +package u_test +-- w/go.mod -- +module example.com/w + +go 1.15 + +require example.com/x v0.1.0 +-- w/w.go -- +package w +-- w/w_test.go -- +package w_test + +import _ "example.com/x" +-- x/go.mod -- +module example.com/x + +go 1.15 +-- x/x.go -- +package x diff --git a/src/cmd/go/testdata/script/mod_auth.txt b/src/cmd/go/testdata/script/mod_auth.txt index 5bcbcd1a18..544acbc1f8 100644 --- a/src/cmd/go/testdata/script/mod_auth.txt +++ b/src/cmd/go/testdata/script/mod_auth.txt @@ -7,7 +7,7 @@ env GOSUMDB=off # Without credentials, downloading a module from a path that requires HTTPS # basic auth should fail. env NETRC=$WORK/empty -! go list all +! go mod tidy stderr '^\tserver response: ACCESS DENIED, buddy$' stderr '^\tserver response: File\? What file\?$' diff --git a/src/cmd/go/testdata/script/mod_bad_filenames.txt b/src/cmd/go/testdata/script/mod_bad_filenames.txt index 6e0c8bd302..eb556f4c7c 100644 --- a/src/cmd/go/testdata/script/mod_bad_filenames.txt +++ b/src/cmd/go/testdata/script/mod_bad_filenames.txt @@ -3,9 +3,9 @@ env GO111MODULE=on ! go get rsc.io/badfile1 rsc.io/badfile2 rsc.io/badfile3 rsc.io/badfile4 rsc.io/badfile5 ! stderr 'unzip.*badfile1' stderr 'unzip.*badfile2[\\/]@v[\\/]v1.0.0.zip:.*malformed file path "☺.go": invalid char ''☺''' -stderr 'unzip.*badfile3[\\/]@v[\\/]v1.0.0.zip: malformed file path "x\?y.go": invalid char ''\?''' -stderr 'unzip.*badfile4[\\/]@v[\\/]v1.0.0.zip: case-insensitive file name collision: "x/Y.go" and "x/y.go"' -stderr 'unzip.*badfile5[\\/]@v[\\/]v1.0.0.zip: case-insensitive file name collision: "x/y" and "x/Y"' +stderr 'unzip.*badfile3[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile3@v1.0.0[\\/]x\?y.go: malformed file path "x\?y.go": invalid char ''\?''' +stderr 'unzip.*badfile4[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile4@v1.0.0[\\/]x[\\/]y.go: case-insensitive file name collision: "x/Y.go" and "x/y.go"' +stderr 'unzip.*badfile5[\\/]@v[\\/]v1.0.0.zip: rsc.io[\\/]badfile5@v1.0.0[\\/]x[\\/]Y[\\/]zz[\\/]ww.go: case-insensitive file name collision: "x/y" and "x/Y"' -- go.mod -- module x diff --git a/src/cmd/go/testdata/script/mod_build_info_err.txt b/src/cmd/go/testdata/script/mod_build_info_err.txt index 87a099b219..a6853b5c86 100644 --- a/src/cmd/go/testdata/script/mod_build_info_err.txt +++ b/src/cmd/go/testdata/script/mod_build_info_err.txt @@ -2,7 +2,7 @@ # Verifies golang.org/issue/34393. go list -e -deps -f '{{with .Error}}{{.Pos}}: {{.Err}}{{end}}' ./main -stdout 'bad[/\\]bad.go:3:8: malformed module path "🐧.example.com/string": invalid char ''🐧''' +stdout 'bad[/\\]bad.go:3:8: malformed import path "🐧.example.com/string": invalid char ''🐧''' -- go.mod -- module m diff --git a/src/cmd/go/testdata/script/mod_case.txt b/src/cmd/go/testdata/script/mod_case.txt index ee818c2c07..6f8d869c44 100644 --- a/src/cmd/go/testdata/script/mod_case.txt +++ b/src/cmd/go/testdata/script/mod_case.txt @@ -1,6 +1,6 @@ env GO111MODULE=on -go get rsc.io/QUOTE +go get -d go list -m all stdout '^rsc.io/quote v1.5.2' stdout '^rsc.io/QUOTE v1.5.2' @@ -18,3 +18,8 @@ stdout '!q!u!o!t!e@v1.5.3-!p!r!e' -- go.mod -- module x + +-- use.go -- +package use + +import _ "rsc.io/QUOTE/QUOTE" diff --git a/src/cmd/go/testdata/script/mod_concurrent.txt b/src/cmd/go/testdata/script/mod_concurrent.txt index e03e5e5edb..8c21525158 100644 --- a/src/cmd/go/testdata/script/mod_concurrent.txt +++ b/src/cmd/go/testdata/script/mod_concurrent.txt @@ -1,6 +1,7 @@ env GO111MODULE=on # Concurrent builds should succeed, even if they need to download modules. +go get -d ./x ./y go build ./x & go build ./y wait diff --git a/src/cmd/go/testdata/script/mod_doc.txt b/src/cmd/go/testdata/script/mod_doc.txt index aac3db00be..595ad679fc 100644 --- a/src/cmd/go/testdata/script/mod_doc.txt +++ b/src/cmd/go/testdata/script/mod_doc.txt @@ -1,6 +1,7 @@ # go doc should find module documentation env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # Check when module x is inside GOPATH/src. @@ -48,6 +49,7 @@ stderr '^doc: cannot find module providing package example.com/hello: module loo # path used in source code, not to the absolute path relative to GOROOT. cd $GOROOT/src +env GOFLAGS= go doc cryptobyte stdout '// import "golang.org/x/crypto/cryptobyte"' diff --git a/src/cmd/go/testdata/script/mod_domain_root.txt b/src/cmd/go/testdata/script/mod_domain_root.txt index e34cc29fa6..14745b5812 100644 --- a/src/cmd/go/testdata/script/mod_domain_root.txt +++ b/src/cmd/go/testdata/script/mod_domain_root.txt @@ -2,7 +2,7 @@ # (example.com not example.com/something) env GO111MODULE=on -go build +go get -d -- go.mod -- module x diff --git a/src/cmd/go/testdata/script/mod_download.txt b/src/cmd/go/testdata/script/mod_download.txt index bb5c4627db..c53bbe4567 100644 --- a/src/cmd/go/testdata/script/mod_download.txt +++ b/src/cmd/go/testdata/script/mod_download.txt @@ -1,13 +1,15 @@ env GO111MODULE=on -# download with version should print nothing +# download with version should print nothing. +# It should not load retractions from the .mod file from the latest version. go mod download rsc.io/quote@v1.5.0 ! stdout . ! stderr . - exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.info exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.mod exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.zip +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod # download of an invalid path should report the error [short] skip @@ -31,53 +33,59 @@ stdout '^\t"GoModSum": "h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe\+TKr0="' go list -m all ! stdout rsc.io -# add to go.mod so we can test non-query downloads -go mod edit -require rsc.io/quote@v1.5.2 -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip - -# module loading will page in the info and mod files -go list -m all +# download query should have downloaded go.mod for the highest release version +# in order to find retractions when resolving the query '@<=v1.5.0'. exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip +# add to go.mod so we can test non-query downloads +go mod edit -require rsc.io/quote@v1.5.3-pre1 +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip + +# module loading will page in the info and mod files +go list -m -mod=mod all +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip + # download will fetch and unpack the zip file go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip -exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 # download repopulates deleted files and directories independently. -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.info +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod -rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.mod +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip go mod download -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip -rm -r $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +rm -r $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 go mod download -exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 +exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.3-pre1 # download reports the locations of downloaded files go mod download -json stdout '^\t"Path": "rsc.io/quote"' -stdout '^\t"Version": "v1.5.2"' -stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.info"' -stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.mod"' -stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.zip"' -stdout '^\t"Dir": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)rsc.io(\\\\|/)quote@v1.5.2"' +stdout '^\t"Version": "v1.5.3-pre1"' +stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.info"' +stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.mod"' +stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.3-pre1.zip"' +stdout '^\t"Dir": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)rsc.io(\\\\|/)quote@v1.5.3-pre1"' # download will follow replacements -go mod edit -require rsc.io/quote@v1.5.1 -replace rsc.io/quote@v1.5.1=rsc.io/quote@v1.5.3-pre1 +go mod edit -require rsc.io/quote@v1.5.1 -replace rsc.io/quote@v1.5.1=rsc.io/quote@v1.5.2 go mod download ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.zip -exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip +exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip # download will not follow replacements for explicit module queries go mod download -json rsc.io/quote@v1.5.1 @@ -99,6 +107,14 @@ stderr '^go mod download: skipping argument m that resolves to the main module\n go mod download m@latest stderr '^go mod download: skipping argument m@latest that resolves to the main module\n' +# download updates go.mod and populates go.sum +cd update +! exists go.sum +go mod download +grep '^rsc.io/sampler v1.3.0 ' go.sum +go list -m rsc.io/sampler +stdout '^rsc.io/sampler v1.3.0$' + # allow go mod download without go.mod env GO111MODULE=auto rm go.mod @@ -114,3 +130,13 @@ stderr 'get '$GOPROXY -- go.mod -- module m + +-- update/go.mod -- +module m + +go 1.16 + +require ( + rsc.io/quote v1.5.2 + rsc.io/sampler v1.2.1 // older version than in build list +) diff --git a/src/cmd/go/testdata/script/mod_download_json.txt b/src/cmd/go/testdata/script/mod_download_json.txt index 26291681ce..9555adf8c4 100644 --- a/src/cmd/go/testdata/script/mod_download_json.txt +++ b/src/cmd/go/testdata/script/mod_download_json.txt @@ -3,7 +3,7 @@ env GOSUMDB=$sumdb' '$proxy/sumdb-wrong # download -json with version should print JSON on sumdb failure ! go mod download -json 'rsc.io/quote@<=v1.5.0' -stdout '"Error": ".*verifying module' +stdout '"Error": ".*verifying (module|go.mod)' -- go.mod -- module m diff --git a/src/cmd/go/testdata/script/mod_download_partial.txt b/src/cmd/go/testdata/script/mod_download_partial.txt index 4978982dab..8d31970160 100644 --- a/src/cmd/go/testdata/script/mod_download_partial.txt +++ b/src/cmd/go/testdata/script/mod_download_partial.txt @@ -1,5 +1,5 @@ -# Download a module -go mod download -modcacherw rsc.io/quote +# Download modules and populate go.sum. +go get -d -modcacherw exists $GOPATH/pkg/mod/rsc.io/quote@v1.5.2/go.mod # 'go mod verify' should fail if we delete a file. @@ -61,4 +61,9 @@ go 1.14 require rsc.io/quote v1.5.2 +-- use.go -- +package use + +import _ "rsc.io/quote" + -- empty -- diff --git a/src/cmd/go/testdata/script/mod_edit.txt b/src/cmd/go/testdata/script/mod_edit.txt index 898d8524ac..78485eb86a 100644 --- a/src/cmd/go/testdata/script/mod_edit.txt +++ b/src/cmd/go/testdata/script/mod_edit.txt @@ -16,15 +16,19 @@ cmpenv go.mod $WORK/go.mod.init cmpenv go.mod $WORK/go.mod.init # go mod edits -go mod edit -droprequire=x.1 -require=x.1@v1.0.0 -require=x.2@v1.1.0 -droprequire=x.2 -exclude='x.1 @ v1.2.0' -exclude=x.1@v1.2.1 -replace=x.1@v1.3.0=y.1@v1.4.0 -replace='x.1@v1.4.0 = ../z' +go mod edit -droprequire=x.1 -require=x.1@v1.0.0 -require=x.2@v1.1.0 -droprequire=x.2 -exclude='x.1 @ v1.2.0' -exclude=x.1@v1.2.1 -replace=x.1@v1.3.0=y.1@v1.4.0 -replace='x.1@v1.4.0 = ../z' -retract=v1.6.0 -retract=[v1.1.0,v1.2.0] -retract=[v1.3.0,v1.4.0] -retract=v1.0.0 cmpenv go.mod $WORK/go.mod.edit1 -go mod edit -droprequire=x.1 -dropexclude=x.1@v1.2.1 -dropreplace=x.1@v1.3.0 -require=x.3@v1.99.0 +go mod edit -droprequire=x.1 -dropexclude=x.1@v1.2.1 -dropreplace=x.1@v1.3.0 -require=x.3@v1.99.0 -dropretract=v1.0.0 -dropretract=[v1.1.0,v1.2.0] cmpenv go.mod $WORK/go.mod.edit2 # go mod edit -json go mod edit -json cmpenv stdout $WORK/go.mod.json +# go mod edit -json (retractions with rationales) +go mod edit -json $WORK/go.mod.retractrationale +cmp stdout $WORK/go.mod.retractrationale.json + # go mod edit -json (empty mod file) go mod edit -json $WORK/go.mod.empty cmp stdout $WORK/go.mod.empty.json @@ -40,11 +44,11 @@ cmpenv go.mod $WORK/go.mod.edit5 # go mod edit -fmt cp $WORK/go.mod.badfmt go.mod go mod edit -fmt -print # -print should avoid writing file -cmpenv stdout $WORK/go.mod.edit6 +cmpenv stdout $WORK/go.mod.goodfmt cmp go.mod $WORK/go.mod.badfmt go mod edit -fmt # without -print, should write file (and nothing to stdout) ! stdout . -cmpenv go.mod $WORK/go.mod.edit6 +cmpenv go.mod $WORK/go.mod.goodfmt # go mod edit -module cd $WORK/m @@ -84,6 +88,13 @@ replace ( x.1 v1.3.0 => y.1 v1.4.0 x.1 v1.4.0 => ../z ) + +retract ( + v1.6.0 + [v1.3.0, v1.4.0] + [v1.1.0, v1.2.0] + v1.0.0 +) -- $WORK/go.mod.edit2 -- module x.x/y/z @@ -93,6 +104,11 @@ exclude x.1 v1.2.0 replace x.1 v1.4.0 => ../z +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.json -- { @@ -122,6 +138,16 @@ require x.3 v1.99.0 "Path": "../z" } } + ], + "Retract": [ + { + "Low": "v1.6.0", + "High": "v1.6.0" + }, + { + "Low": "v1.3.0", + "High": "v1.4.0" + } ] } -- $WORK/go.mod.edit3 -- @@ -136,6 +162,11 @@ replace ( x.1 v1.4.0 => y.1/v2 v2.3.5 ) +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.edit4 -- module x.x/y/z @@ -146,6 +177,11 @@ exclude x.1 v1.2.0 replace x.1 => y.1/v2 v2.3.6 +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) + require x.3 v1.99.0 -- $WORK/go.mod.edit5 -- module x.x/y/z @@ -154,15 +190,10 @@ go $goversion exclude x.1 v1.2.0 -require x.3 v1.99.0 --- $WORK/go.mod.edit6 -- -module x.x/y/z - -go 1.10 - -exclude x.1 v1.2.0 - -replace x.1 => y.1/v2 v2.3.6 +retract ( + v1.6.0 + [v1.3.0, v1.4.0] +) require x.3 v1.99.0 -- $WORK/local/go.mod.edit -- @@ -183,10 +214,64 @@ exclude x.1 v1.2.0 replace x.1 => y.1/v2 v2.3.6 require x.3 v1.99.0 + +retract [ "v1.8.1" , "v1.8.2" ] +-- $WORK/go.mod.goodfmt -- +module x.x/y/z + +go 1.10 + +exclude x.1 v1.2.0 + +replace x.1 => y.1/v2 v2.3.6 + +require x.3 v1.99.0 + +retract [v1.8.1, v1.8.2] -- $WORK/m/go.mod.edit -- module x.x/y/z go $goversion +-- $WORK/go.mod.retractrationale -- +module x.x/y/z + +go 1.15 + +// a +retract v1.0.0 + +// b +retract ( + v1.0.1 + v1.0.2 // c +) +-- $WORK/go.mod.retractrationale.json -- +{ + "Module": { + "Path": "x.x/y/z" + }, + "Go": "1.15", + "Require": null, + "Exclude": null, + "Replace": null, + "Retract": [ + { + "Low": "v1.0.0", + "High": "v1.0.0", + "Rationale": "a" + }, + { + "Low": "v1.0.1", + "High": "v1.0.1", + "Rationale": "b" + }, + { + "Low": "v1.0.2", + "High": "v1.0.2", + "Rationale": "c" + } + ] +} -- $WORK/go.mod.empty -- -- $WORK/go.mod.empty.json -- { @@ -195,5 +280,6 @@ go $goversion }, "Require": null, "Exclude": null, - "Replace": null + "Replace": null, + "Retract": null } diff --git a/src/cmd/go/testdata/script/mod_find.txt b/src/cmd/go/testdata/script/mod_find.txt index 7fbe9fb7fe..9468acfd33 100644 --- a/src/cmd/go/testdata/script/mod_find.txt +++ b/src/cmd/go/testdata/script/mod_find.txt @@ -19,6 +19,11 @@ go mod init stderr 'module example.com/x/y$' rm go.mod +# go mod init rejects a zero-length go.mod file +cp $devnull go.mod # can't use touch to create it because Windows +! go mod init +stderr 'go.mod already exists' + # Module path from Godeps/Godeps.json overrides GOPATH. cd $GOPATH/src/example.com/x/y/z go mod init diff --git a/src/cmd/go/testdata/script/mod_get_incompatible.txt b/src/cmd/go/testdata/script/mod_get_incompatible.txt index b210715a5d..b28718a694 100644 --- a/src/cmd/go/testdata/script/mod_get_incompatible.txt +++ b/src/cmd/go/testdata/script/mod_get_incompatible.txt @@ -1,6 +1,6 @@ env GO111MODULE=on -go list x +go get -d x go list -m all stdout 'rsc.io/breaker v2.0.0\+incompatible' diff --git a/src/cmd/go/testdata/script/mod_get_indirect.txt b/src/cmd/go/testdata/script/mod_get_indirect.txt index f25e170a49..e1cc1ab411 100644 --- a/src/cmd/go/testdata/script/mod_get_indirect.txt +++ b/src/cmd/go/testdata/script/mod_get_indirect.txt @@ -27,7 +27,7 @@ grep 'golang.org/x/text v0.3.0 // indirect$' go.mod # indirect tag should be removed upon seeing direct import. cp $WORK/tmp/uselang.go x.go -go list +go get -d grep 'rsc.io/quote v1.5.2$' go.mod grep 'golang.org/x/text [v0-9a-f\.-]+$' go.mod diff --git a/src/cmd/go/testdata/script/mod_get_latest_pseudo.txt b/src/cmd/go/testdata/script/mod_get_latest_pseudo.txt index 825ee8cf89..241a0c2f0d 100644 --- a/src/cmd/go/testdata/script/mod_get_latest_pseudo.txt +++ b/src/cmd/go/testdata/script/mod_get_latest_pseudo.txt @@ -5,6 +5,6 @@ env GO111MODULE=on go mod init m -go list example.com/notags +go get -d example.com/notags go list -m all stdout '^example.com/notags v0.0.0-20190507143103-cc8cbe209b64$' diff --git a/src/cmd/go/testdata/script/mod_get_retract.txt b/src/cmd/go/testdata/script/mod_get_retract.txt new file mode 100644 index 0000000000..da6c25523f --- /dev/null +++ b/src/cmd/go/testdata/script/mod_get_retract.txt @@ -0,0 +1,49 @@ +# 'go get pkg' should not upgrade to a retracted version. +cp go.mod.orig go.mod +go mod edit -require example.com/retract/self/prev@v1.1.0 +go get -d example.com/retract/self/prev +go list -m example.com/retract/self/prev +stdout '^example.com/retract/self/prev v1.1.0$' + +# 'go get pkg' should not downgrade from a retracted version when no higher +# version is available. +cp go.mod.orig go.mod +go mod edit -require example.com/retract/self/prev@v1.9.0 +go get -d example.com/retract/self/prev +stderr '^go: warning: example.com/retract/self/prev@v1.9.0 is retracted: self$' +go list -m example.com/retract/self/prev +stdout '^example.com/retract/self/prev v1.9.0$' + +# 'go get pkg@latest' should downgrade from a retracted version. +cp go.mod.orig go.mod +go mod edit -require example.com/retract/self/prev@v1.9.0 +go get -d example.com/retract/self/prev@latest +go list -m example.com/retract/self/prev +stdout '^example.com/retract/self/prev v1.1.0$' + +# 'go get pkg@version' should update to a specific version, even if that +# version is retracted. +cp go.mod.orig go.mod +go get -d example.com/retract@v1.0.0-bad +stderr '^go: warning: example.com/retract@v1.0.0-bad is retracted: bad$' +go list -m example.com/retract +stdout '^example.com/retract v1.0.0-bad$' + +# 'go get -u' should not downgrade from a retracted version when no higher +# version is available. +cp go.mod.orig go.mod +go mod edit -require example.com/retract/self/prev@v1.9.0 +go get -d -u . +stderr '^go: warning: example.com/retract/self/prev@v1.9.0 is retracted: self$' +go list -m example.com/retract/self/prev +stdout '^example.com/retract/self/prev v1.9.0$' + +-- go.mod.orig -- +module example.com/use + +go 1.15 + +-- use.go -- +package use + +import _ "example.com/retract/self/prev" diff --git a/src/cmd/go/testdata/script/mod_get_sum_noroot.txt b/src/cmd/go/testdata/script/mod_get_sum_noroot.txt new file mode 100644 index 0000000000..0d9a840e77 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_get_sum_noroot.txt @@ -0,0 +1,11 @@ +# When 'go get' is invoked on a module without a package in the root directory, +# it should add sums for the module's go.mod file and its content to go.sum. +# Verifies golang.org/issue/41103. +go mod init m +go get rsc.io/QUOTE +grep '^rsc.io/QUOTE v1.5.2/go.mod ' go.sum +grep '^rsc.io/QUOTE v1.5.2 ' go.sum + +# Double-check rsc.io/QUOTE does not have a root package. +! go list -mod=readonly rsc.io/QUOTE +stderr '^cannot find module providing package rsc.io/QUOTE: import lookup disabled by -mod=readonly$' diff --git a/src/cmd/go/testdata/script/mod_get_trailing_slash.txt b/src/cmd/go/testdata/script/mod_get_trailing_slash.txt index 7b5d90c50b..3b38d8ba7d 100644 --- a/src/cmd/go/testdata/script/mod_get_trailing_slash.txt +++ b/src/cmd/go/testdata/script/mod_get_trailing_slash.txt @@ -1,3 +1,6 @@ +# Populate go.sum +go mod download + # go list should succeed to load a package ending with ".go" if the path does # not correspond to an existing local file. Listing a pattern ending with # ".go/" should try to list a package regardless of whether a file exists at the diff --git a/src/cmd/go/testdata/script/mod_import.txt b/src/cmd/go/testdata/script/mod_import.txt index 3985b43144..28358b5b0c 100644 --- a/src/cmd/go/testdata/script/mod_import.txt +++ b/src/cmd/go/testdata/script/mod_import.txt @@ -1,7 +1,7 @@ env GO111MODULE=on # latest rsc.io/quote should be v1.5.2 not v1.5.3-pre1 -go list +go get -d go list -m all stdout 'rsc.io/quote v1.5.2' diff --git a/src/cmd/go/testdata/script/mod_import_issue41113.txt b/src/cmd/go/testdata/script/mod_import_issue41113.txt new file mode 100644 index 0000000000..e98ac63d48 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_import_issue41113.txt @@ -0,0 +1,28 @@ +# Regression test for https://golang.org/issue/41113. +# +# When resolving a missing import path, the inability to add the package from +# one module path should not interfere with adding a nested path. + +# Initially, our module depends on split-incompatible v2.1.0-pre+incompatible, +# from which an imported package has been removed (and relocated to the nested +# split-incompatible/subpkg module). modload.QueryPackage will suggest +# split-incompatible v2.0.0+incompatible, which we cannot use (because it would +# be an implicit downgrade), and split-incompatible/subpkg v0.1.0, which we +# *should* use. + +go mod tidy + +go list -m all +stdout '^example.com/split-incompatible/subpkg v0\.1\.0$' +! stdout '^example.com/split-incompatible .*' + +-- go.mod -- +module golang.org/issue/41113 + +go 1.16 + +require example.com/split-incompatible v2.1.0-pre+incompatible +-- x.go -- +package issue41113 + +import _ "example.com/split-incompatible/subpkg" diff --git a/src/cmd/go/testdata/script/mod_in_testdata_dir.txt b/src/cmd/go/testdata/script/mod_in_testdata_dir.txt index f582569798..66f79faa6d 100644 --- a/src/cmd/go/testdata/script/mod_in_testdata_dir.txt +++ b/src/cmd/go/testdata/script/mod_in_testdata_dir.txt @@ -8,8 +8,8 @@ env GO111MODULE=on cd $WORK/testdata go mod init testdata.tld/foo -# Building a package within that module should resolve its dependencies. -go build +# Getting a package within that module should resolve its dependencies. +go get -d grep 'rsc.io/quote' go.mod # Tidying the module should preserve those dependencies. @@ -26,7 +26,7 @@ exists vendor/rsc.io/quote cd $WORK/_ignored go mod init testdata.tld/foo -go build +go get grep 'rsc.io/quote' go.mod go mod tidy diff --git a/src/cmd/go/testdata/script/mod_init_dep.txt b/src/cmd/go/testdata/script/mod_init_dep.txt index 755076eae8..f8cf1d563a 100644 --- a/src/cmd/go/testdata/script/mod_init_dep.txt +++ b/src/cmd/go/testdata/script/mod_init_dep.txt @@ -1,24 +1,14 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod # modconv uses git directly to examine what old 'go get' would [!net] skip [!exec:git] skip -# go build should populate go.mod from Gopkg.lock -cp go.mod1 go.mod -go build +# go mod init should populate go.mod from Gopkg.lock +go mod init x stderr 'copying requirements from Gopkg.lock' go list -m all -! stderr 'copying requirements from Gopkg.lock' -stdout 'rsc.io/sampler v1.0.0' - -# go list should populate go.mod from Gopkg.lock -cp go.mod1 go.mod -go list -stderr 'copying requirements from Gopkg.lock' -go list -! stderr 'copying requirements from Gopkg.lock' -go list -m all stdout 'rsc.io/sampler v1.0.0' # test dep replacement @@ -26,9 +16,6 @@ cd y go mod init cmpenv go.mod go.mod.replace --- go.mod1 -- -module x - -- x.go -- package x @@ -54,4 +41,4 @@ go $goversion replace z v1.0.0 => rsc.io/quote v1.0.0 -require rsc.io/quote v1.0.0
\ No newline at end of file +require rsc.io/quote v1.0.0 diff --git a/src/cmd/go/testdata/script/mod_install_versioned.txt b/src/cmd/go/testdata/script/mod_install_versioned.txt index 03986d06a0..c6bce418b4 100644 --- a/src/cmd/go/testdata/script/mod_install_versioned.txt +++ b/src/cmd/go/testdata/script/mod_install_versioned.txt @@ -1,9 +1,11 @@ env GO111MODULE=on +go get -d rsc.io/fortune go list -f '{{.Target}}' rsc.io/fortune ! stdout fortune@v1 stdout 'fortune(\.exe)?$' +go get -d rsc.io/fortune/v2 go list -f '{{.Target}}' rsc.io/fortune/v2 ! stdout v2 stdout 'fortune(\.exe)?$' diff --git a/src/cmd/go/testdata/script/mod_internal.txt b/src/cmd/go/testdata/script/mod_internal.txt index 1193d528ec..687269d18f 100644 --- a/src/cmd/go/testdata/script/mod_internal.txt +++ b/src/cmd/go/testdata/script/mod_internal.txt @@ -3,30 +3,34 @@ env GO111MODULE=on # golang.org/x/internal should be importable from other golang.org/x modules. go mod edit -module=golang.org/x/anything -go build . +go get -d . # ...and their tests... go test stdout PASS # ...but that should not leak into other modules. +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' # Internal packages in the standard library should not leak into modules. +go get -d ./fromstd ! go build ./fromstd stderr 'use of internal package internal/testenv not allowed' # Dependencies should be able to use their own internal modules... go mod edit -module=golang.org/notx -go build ./throughdep +go get -d ./throughdep # ... but other modules should not, even if they have transitive dependencies. +go get -d . ! go build . stderr 'use of internal package golang.org/x/.* not allowed' # And transitive dependencies still should not leak. +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' @@ -34,15 +38,17 @@ stderr 'use of internal package golang.org/x/.* not allowed' # Replacing an internal module should keep it internal to the same paths. go mod edit -module=golang.org/notx go mod edit -replace golang.org/x/internal=./replace/golang.org/notx/internal -go build ./throughdep +go get -d ./throughdep +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' go mod edit -replace golang.org/x/internal=./vendor/golang.org/x/internal -go build ./throughdep +go get -d ./throughdep +go get -d ./baddep ! go build ./baddep stderr golang.org[/\\]notx[/\\]useinternal stderr 'use of internal package golang.org/x/.* not allowed' diff --git a/src/cmd/go/testdata/script/mod_invalid_path.txt b/src/cmd/go/testdata/script/mod_invalid_path.txt new file mode 100644 index 0000000000..05a5133571 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_invalid_path.txt @@ -0,0 +1,40 @@ +# Test that mod files with invalid or missing paths produce an error. + +# Test that go list fails on a go.mod with no module declaration. +cd $WORK/gopath/src/mod +! go list . +stderr '^go: no module declaration in go.mod.\n\tRun ''go mod edit -module=example.com/mod'' to specify the module path.$' + +# Test that go mod init in GOPATH doesn't add a module declaration +# with a path that can't possibly be a module path, because +# it isn't even a valid import path. +# The single quote and backtick are the only characters we don't allow +# in checkModulePathLax, but is allowed in a Windows file name. +# TODO(matloob): choose a different character once +# module.CheckImportPath is laxened and replaces +# checkModulePathLax. +cd $WORK/'gopath/src/m''d' +! go mod init +stderr 'cannot determine module path' + +# Test that a go.mod file is rejected when its module declaration has a path that can't +# possibly be a module path, because it isn't even a valid import path +cd $WORK/gopath/src/badname +! go list . +stderr 'invalid module path' + +-- mod/go.mod -- + +-- mod/foo.go -- +package foo + +-- m'd/foo.go -- +package mad + +-- badname/go.mod -- + +module .\. + +-- badname/foo.go -- +package badname + diff --git a/src/cmd/go/testdata/script/mod_invalid_version.txt b/src/cmd/go/testdata/script/mod_invalid_version.txt index 7e1bc9ea4f..f9dfdd6346 100644 --- a/src/cmd/go/testdata/script/mod_invalid_version.txt +++ b/src/cmd/go/testdata/script/mod_invalid_version.txt @@ -4,6 +4,7 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOFLAGS=-mod=mod # Regression test for golang.org/issue/27173: if the user (or go.mod file) # requests a pseudo-version that does not match both the module path and commit @@ -18,7 +19,7 @@ cp go.mod.orig go.mod go mod edit -require golang.org/x/text@14c0d48ead0c cd outside ! go list -m golang.org/x/text -stderr 'go: example.com@v0.0.0: parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "14c0d48ead0c" invalid: must be of the form v1.2.3' +stderr 'go: example.com@v0.0.0 \(replaced by \./\..\): parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "14c0d48ead0c" invalid: must be of the form v1.2.3' cd .. go list -m golang.org/x/text stdout 'golang.org/x/text v0.1.1-0.20170915032832-14c0d48ead0c' @@ -46,7 +47,7 @@ cp go.mod.orig go.mod go mod edit -require golang.org/x/text@v2.1.1-0.20170915032832-14c0d48ead0c cd outside ! go list -m golang.org/x/text -stderr 'go: example.com@v0.0.0: parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' +stderr 'go: example.com@v0.0.0 \(replaced by \./\.\.\): parsing ../go.mod: '$WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' cd .. ! go list -m golang.org/x/text stderr $WORK'/gopath/src/go.mod:5: require golang.org/x/text: version "v2.1.1-0.20170915032832-14c0d48ead0c" invalid: should be v0 or v1, not v2' diff --git a/src/cmd/go/testdata/script/mod_lazy_import_allmod.txt b/src/cmd/go/testdata/script/mod_lazy_import_allmod.txt new file mode 100644 index 0000000000..4ad8cbf8ee --- /dev/null +++ b/src/cmd/go/testdata/script/mod_lazy_import_allmod.txt @@ -0,0 +1,172 @@ +# This test demonstrates dependency resolution when the main module imports a +# new package from a previously-test-only dependency. +# +# When lazy loading is active, the loader will not load dependencies of any +# module whose packages are *only* imported by tests outside the main module. If +# the main module is changed to import a package from such a module, the +# dependencies of that module will need to be reloaded. + +# The import graph used in this test looks like: +# +# m ---- a +# \ | +# \ a_test ---- b/x +# \ +# --------------b/y (new) ---- c +# +# Where b/x and b/y are disjoint packages, but both contained in module b. +# +# The module dependency graph initially looks like: +# +# m ---- a.1 ---- b.1 ---- c.1 +# +# This configuration is similar to that used in mod_lazy_new_import, +# but the new import is from what is initially a test-only dependency. + +# Control case: in Go 1.14, the original go.mod is tidy, +# and the dependency on c is eagerly loaded. + +cp go.mod go.mod.orig +go mod tidy +cmp go.mod.orig go.mod + +go list -m all +stdout '^a v0.1.0 ' +stdout '^b v0.1.0 ' +stdout '^c v0.1.0 ' + +# After adding a new import of b/y, +# the import of c from b/y should resolve to the version required by b. + +cp m.go m.go.orig +cp m.go.new m.go +go mod tidy +cmp go.mod.new go.mod + +go list -m all +stdout '^a v0.1.0 ' +stdout '^b v0.1.0 ' +stdout '^c v0.1.0 ' + +# With lazy loading, the go.mod requirements are the same, +# but the dependency on c is initially pruned out. + +cp m.go.orig m.go +cp go.mod.orig go.mod +go mod edit -go=1.16 +go mod edit -go=1.16 go.mod.new + +cp go.mod go.mod.orig +go mod tidy +cmp go.mod.orig go.mod + +go list -m all +stdout '^a v0.1.0 ' +stdout '^b v0.1.0 ' +stdout '^c v0.1.0 ' # TODO(#36460): This should be pruned out. + +# After adding a new import of b/y, +# the import of c from b/y should again resolve to the version required by b. + +cp m.go.new m.go +go mod tidy +cmp go.mod.new go.mod + +go list -m all +stdout '^a v0.1.0 ' +stdout '^b v0.1.0 ' +stdout '^c v0.1.0 ' + +-- m.go -- +package main + +import ( + "fmt" + + _ "a" // a_test imports b/x. +) + +func main() { +} +-- m.go.new -- +package main + +import ( + "fmt" + + _ "a" // a_test imports b/x. + "b/y" // This is a new import, not yet reflected in the go.mod file. +) + +func main() { + fmt.Println(b.CVersion()) +} +-- go.mod -- +module m + +go 1.14 + +require a v0.1.0 + +replace ( + a v0.1.0 => ./a1 + b v0.1.0 => ./b1 + c v0.1.0 => ./c1 + c v0.2.0 => ./c2 +) +-- go.mod.new -- +module m + +go 1.14 + +require ( + a v0.1.0 + b v0.1.0 +) + +replace ( + a v0.1.0 => ./a1 + b v0.1.0 => ./b1 + c v0.1.0 => ./c1 + c v0.2.0 => ./c2 +) +-- a1/go.mod -- +module a + +go 1.16 + +require b v0.1.0 +-- a1/a.go -- +package a +-- a1/a_test.go -- +package a_test + +import _ "b/x" +-- b1/go.mod -- +module b + +go 1.16 + +require c v0.1.0 +-- b1/x/x.go -- +package x +-- b1/y/y.go -- +package y + +import "c" + +func CVersion() string { + return c.Version +} +-- c1/go.mod -- +module c + +go 1.16 +-- c1/c.go -- +package c + +const Version = "v0.1.0" +-- c2/go.mod -- +This file should be unused. +-- c2/c.go -- +This file should be unused. diff --git a/src/cmd/go/testdata/script/mod_lazy_new_import.txt b/src/cmd/go/testdata/script/mod_lazy_new_import.txt new file mode 100644 index 0000000000..02935bf236 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_lazy_new_import.txt @@ -0,0 +1,107 @@ +# This test illustrates the use of a deepening scan to resolve transitive +# imports of imports of new packages from within existing dependencies. + +# The package import graph used in this test looks like: +# +# lazy ---- a/x ---- b +# \ +# ---- a/y ---- c +# +# Where a/x and x/y are disjoint packages, but both contained in module a. +# +# The module dependency graph initially looks like: +# +# lazy ---- a.1 ---- b.1 +# \ +# c.1 + + +cp go.mod go.mod.old +cp lazy.go lazy.go.old +go mod tidy +cmp go.mod go.mod.old + +# Before adding a new import, the go.mod file should +# enumerate modules for all packages already imported. +go list all +cmp go.mod go.mod.old + +# When we add a new import of a package in an existing dependency, +# and that dependency is already tidy, its transitive dependencies +# should already be present. +cp lazy.go.new lazy.go +go list all +go list -m all +stdout '^example.com/c v0.1.0' # not v0.2.0 as would be be resolved by 'latest' +cmp go.mod go.mod.old + +# TODO(#36460): +cp lazy.go.old lazy.go +cp go.mod.old go.mod +go mod edit -go=1.16 + +# When a new import is found, we should perform a deepening scan of the existing +# dependencies and add a requirement on the version required by those +# dependencies — not re-resolve 'latest'. + + +-- go.mod -- +module example.com/lazy + +go 1.15 + +require example.com/a v0.1.0 + +replace ( + example.com/a v0.1.0 => ./a + example.com/b v0.1.0 => ./b + example.com/c v0.1.0 => ./c1 + example.com/c v0.2.0 => ./c2 +) +-- lazy.go -- +package lazy + +import ( + _ "example.com/a/x" +) +-- lazy.go.new -- +package lazy + +import ( + _ "example.com/a/x" + _ "example.com/a/y" +) +-- a/go.mod -- +module example.com/a + +go 1.15 + +require ( + example.com/b v0.1.0 + example.com/c v0.1.0 +) +-- a/x/x.go -- +package x +import _ "example.com/b" +-- a/y/y.go -- +package y +import _ "example.com/c" +-- b/go.mod -- +module example.com/b + +go 1.15 +-- b/b.go -- +package b +-- c1/go.mod -- +module example.com/c + +go 1.15 +-- c1/c.go -- +package c +-- c2/go.mod -- +module example.com/c + +go 1.15 +-- c2/c.go -- +package c +This file should not be used, so this syntax error should be ignored. diff --git a/src/cmd/go/testdata/script/mod_lazy_test_horizon.txt b/src/cmd/go/testdata/script/mod_lazy_test_horizon.txt new file mode 100644 index 0000000000..9cdfad79f6 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_lazy_test_horizon.txt @@ -0,0 +1,131 @@ +# This file demonstrates the effect of lazy loading on the selected +# versions of test dependencies. + +# The package import graph used in this test looks like: +# +# m ---- a +# \ | +# \ a_test ---- b +# \ | +# x b_test +# | \ +# x_test -------------- c +# +# And the module dependency graph looks like: +# +# m -- a.1 -- b.1 -- c.2 +# \ +# x.1 ------------ c.1 + +# Control case: in Go 1.15, the version of c imported by 'go test x' is the +# version required by module b, even though b_test is not relevant to the main +# module. (The main module imports a, and a_test imports b, but all of the +# packages and tests in the main module can be built without b.) + +go list -m c +stdout '^c v0.2.0 ' + +[!short] go test -v x +[!short] stdout ' c v0.2.0$' + +# With lazy loading, the go.mod requirements are the same, +# but the irrelevant dependency on c v0.2.0 should be pruned out, +# leaving only the relevant dependency on c v0.1.0. + +go mod edit -go=1.16 +go list -m c +stdout '^c v0.2.0' # TODO(#36460): v0.1.0 + +[!short] go test -v x +[!short] stdout ' c v0.2.0$' # TODO(#36460): v0.1.0 + +-- m.go -- +package m + +import ( + _ "a" + _ "x" +) +-- go.mod -- +module m + +go 1.15 + +require ( + a v0.1.0 + x v0.1.0 +) + +replace ( + a v0.1.0 => ./a1 + b v0.1.0 => ./b1 + c v0.1.0 => ./c1 + c v0.2.0 => ./c2 + x v0.1.0 => ./x1 +) +-- a1/go.mod -- +module a + +go 1.16 + +require b v0.1.0 +-- a1/a.go -- +package a +-- a1/a_test.go -- +package a_test + +import _ "b" +-- b1/go.mod -- +module b + +go 1.16 + +require c v0.2.0 +-- b1/b.go -- +package b +-- b1/b_test.go -- +package b_test + +import ( + "c" + "testing" +) + +func TestCVersion(t *testing.T) { + t.Log(c.Version) +} +-- c1/go.mod -- +module c + +go 1.16 +-- c1/c.go -- +package c + +const Version = "v0.1.0" +-- c2/go.mod -- +module c + +go 1.16 +-- c2/c.go -- +package c + +const Version = "v0.2.0" +-- x1/go.mod -- +module x + +go 1.16 + +require c v0.1.0 +-- x1/x.go -- +package x +-- x1/x_test.go -- +package x_test + +import ( + "c" + "testing" +) + +func TestCVersion(t *testing.T) { + t.Log("c", c.Version) +} diff --git a/src/cmd/go/testdata/script/mod_lazy_test_of_test_dep.txt b/src/cmd/go/testdata/script/mod_lazy_test_of_test_dep.txt new file mode 100644 index 0000000000..ca6c55040e --- /dev/null +++ b/src/cmd/go/testdata/script/mod_lazy_test_of_test_dep.txt @@ -0,0 +1,145 @@ +# This file demonstrates the effect of lazy loading on the reproducibility of +# tests (and tests of test dependencies) outside the main module. +# +# It is similar to the cases in mod_all.txt and mod_lazy_test_horizon.txt, but +# focuses on the effect of "go test" on specific packages instead of the "all" +# pattern. + +# The package import graph used in this test looks like: +# +# lazy ---- a +# | +# a_test ---- b +# | +# b_test ---- c +# +# And the non-lazy module dependency graph looks like: +# +# lazy ---- a.1 ---- b.1 ---- c.1 + +cp go.mod go.mod.old +go mod tidy +cmp go.mod go.mod.old + +# In Go 1.15 mode, 'go list -m all' includes modules needed by the +# transitive closure of tests of dependencies of tests of dependencies of …. + +go list -m all +stdout 'example.com/b v0.1.0' +stdout 'example.com/c v0.1.0' +cmp go.mod go.mod.old + +# 'go test' (or equivalent) of any such dependency, no matter how remote, does +# not update the go.mod file. + +go list -test -deps example.com/a +stdout example.com/b +! stdout example.com/c + +[!short] go test -c example.com/a +[!short] cmp go.mod go.mod.old + +go list -test -deps example.com/b +stdout example.com/c + +[!short] go test -c example.com/b +[!short] cmp go.mod go.mod.old + +# TODO(#36460): + +# After changing to 'go 1.16` uniformly, 'go list -m all' should prune out +# example.com/c, because it is not imported by any package (or test of a package) +# transitively imported by the main module. +# +# example.com/a is imported, +# and example.com/b is needed in order to run 'go test example.com/a', +# but example.com/c is not needed because we don't expect the user to need to run +# 'go test example.com/b'. + +# If we skip directly to adding a new import of c, the dependency is too far +# away for a deepening scan to find, which is fine because the package whose +# test imported it wasn't even it "all". It should resolve from the latest +# version of its module. + +# However, if we reach c by running successive tests starting from the main +# module, we should end up with exactly the version require by c, with an update +# to the go.mod file as soon as we test a test dependency that is not itself in +# "all". + +-- go.mod -- +module example.com/lazy + +go 1.15 + +require example.com/a v0.1.0 + +replace ( + example.com/a v0.1.0 => ./a + example.com/b v0.1.0 => ./b1 + example.com/b v0.2.0 => ./b2 + example.com/c v0.1.0 => ./c1 + example.com/c v0.2.0 => ./c2 +) +-- lazy.go -- +package lazy + +import ( + _ "example.com/a" +) +-- a/go.mod -- +module example.com/a + +go 1.15 + +require example.com/b v0.1.0 +-- a/a.go -- +package a +-- a/a_test.go -- +package a + +import ( + "testing" + + _ "example.com/b" +) + +func TestUsingB(t *testing.T) { + // … +} +-- b1/go.mod -- +module example.com/b + +go 1.15 + +require example.com/c v0.1.0 +-- b1/b.go -- +package b +-- b1/b_test.go -- +package b + +import _ "example.com/c" +-- b2/go.mod -- +module example.com/b + +go 1.15 + +require example.com/c v0.1.0 +-- b2/b.go -- +package b +This file should not be used, so this syntax error should be ignored. +-- b2/b_test.go -- +package b +This file should not be used, so this syntax error should be ignored. +-- c1/go.mod -- +module example.com/c + +go 1.15 +-- c1/c.go -- +package c +-- c2/go.mod -- +module example.com/c + +go 1.15 +-- c2/c.go -- +package c +This file should not be used, so this syntax error should be ignored. diff --git a/src/cmd/go/testdata/script/mod_list.txt b/src/cmd/go/testdata/script/mod_list.txt index 17b33fcc7b..1ba6d7c910 100644 --- a/src/cmd/go/testdata/script/mod_list.txt +++ b/src/cmd/go/testdata/script/mod_list.txt @@ -2,12 +2,12 @@ env GO111MODULE=on [short] skip # list {{.Dir}} shows main module and go.mod but not not-yet-downloaded dependency dir. -go list -m -f '{{.Path}} {{.Main}} {{.GoMod}} {{.Dir}}' all +go list -mod=mod -m -f '{{.Path}} {{.Main}} {{.GoMod}} {{.Dir}}' all stdout '^x true .*[\\/]src[\\/]go.mod .*[\\/]src$' stdout '^rsc.io/quote false .*[\\/]v1.5.2.mod $' # list {{.Dir}} shows dependency after download (and go list without -m downloads it) -go list -f '{{.Dir}}' rsc.io/quote +go list -mod=mod -f '{{.Dir}}' rsc.io/quote stdout '.*mod[\\/]rsc.io[\\/]quote@v1.5.2$' # downloaded dependencies are read-only @@ -20,7 +20,7 @@ go clean -modcache # list {{.Dir}} shows replaced directories cp go.mod2 go.mod -go list -f {{.Dir}} rsc.io/quote +go list -mod=mod -f {{.Dir}} rsc.io/quote go list -m -f '{{.Path}} {{.Version}} {{.Dir}}{{with .Replace}} {{.GoMod}} => {{.Version}} {{.Dir}} {{.GoMod}}{{end}}' all stdout 'mod[\\/]rsc.io[\\/]quote@v1.5.1' stdout 'v1.3.0.*mod[\\/]rsc.io[\\/]sampler@v1.3.1 .*[\\/]v1.3.1.mod => v1.3.1.*sampler@v1.3.1 .*[\\/]v1.3.1.mod' @@ -30,7 +30,7 @@ go list std stdout ^math/big # rsc.io/quote/buggy should be listable as a package -go list rsc.io/quote/buggy +go list -mod=mod rsc.io/quote/buggy # rsc.io/quote/buggy should not be listable as a module go list -m -e -f '{{.Error.Err}}' nonexist rsc.io/quote/buggy diff --git a/src/cmd/go/testdata/script/mod_list_bad_import.txt b/src/cmd/go/testdata/script/mod_list_bad_import.txt index 8a66e0b72a..b3e2fff67d 100644 --- a/src/cmd/go/testdata/script/mod_list_bad_import.txt +++ b/src/cmd/go/testdata/script/mod_list_bad_import.txt @@ -12,10 +12,11 @@ stdout 'incomplete' stdout 'bad dep: .*example.com/notfound' # Listing with -deps should also fail. -# BUG: Today, it does not. -# ! go list -deps example.com/direct -# stderr example.com/notfound -go list -deps example.com/direct +! go list -deps example.com/direct +stderr example.com/notfound + +# But -e -deps should succeed. +go list -e -deps example.com/direct stdout example.com/notfound @@ -28,10 +29,11 @@ stdout incomplete stdout 'bad dep: .*example.com/notfound' # Again, -deps should fail. -# BUG: Again, it does not. -# ! go list -deps example.com/indirect -# stderr example.com/notfound -go list -deps example.com/indirect +! go list -deps example.com/indirect +stderr example.com/notfound + +# But -e -deps should succeed. +go list -e -deps example.com/indirect stdout example.com/notfound diff --git a/src/cmd/go/testdata/script/mod_list_dir.txt b/src/cmd/go/testdata/script/mod_list_dir.txt index 6653435a06..1adab8f027 100644 --- a/src/cmd/go/testdata/script/mod_list_dir.txt +++ b/src/cmd/go/testdata/script/mod_list_dir.txt @@ -2,6 +2,9 @@ # go list with path to directory should work +# populate go.sum +go get -d + env GO111MODULE=off go list -f '{{.ImportPath}}' $GOROOT/src/math stdout ^math$ @@ -29,3 +32,5 @@ require rsc.io/quote v1.5.2 -- x.go -- package x + +import _ "rsc.io/quote" diff --git a/src/cmd/go/testdata/script/mod_list_direct.txt b/src/cmd/go/testdata/script/mod_list_direct.txt index 8f85871189..62a472f475 100644 --- a/src/cmd/go/testdata/script/mod_list_direct.txt +++ b/src/cmd/go/testdata/script/mod_list_direct.txt @@ -10,7 +10,7 @@ env GOSUMDB=off # For a while, (*modfetch.codeRepo).Stat was not checking for a go.mod file, # which would produce a hard error at the subsequent call to GoMod. -go list all +go get -d -- go.mod -- module example.com diff --git a/src/cmd/go/testdata/script/mod_list_pseudo.txt b/src/cmd/go/testdata/script/mod_list_pseudo.txt index 3a10b3a040..056c093128 100644 --- a/src/cmd/go/testdata/script/mod_list_pseudo.txt +++ b/src/cmd/go/testdata/script/mod_list_pseudo.txt @@ -10,30 +10,25 @@ go mod download github.com/dmitshur-test/modtest5@v0.5.0-alpha go mod download github.com/dmitshur-test/modtest5@v0.5.0-alpha.0.20190619023908-3da23a9deb9e cmp $GOPATH/pkg/mod/cache/download/github.com/dmitshur-test/modtest5/@v/list $WORK/modtest5.list +env GOSUMDB=off # don't verify go.mod files when loading retractions env GOPROXY=file:///$GOPATH/pkg/mod/cache/download env GOPATH=$WORK/gopath2 mkdir $GOPATH -go list -m -json github.com/dmitshur-test/modtest5@latest -cmp stdout $WORK/modtest5.json +go list -m -f '{{.Path}} {{.Version}} {{.Time.Format "2006-01-02"}}' github.com/dmitshur-test/modtest5@latest +stdout '^github.com/dmitshur-test/modtest5 v0.5.0-alpha 2019-06-18$' # If the module proxy contains only pseudo-versions, 'latest' should stat # the version with the most recent timestamp — not the highest semantic # version — and return its metadata. env GOPROXY=file:///$WORK/tinyproxy -go list -m -json dmitri.shuralyov.com/test/modtest3@latest -cmp stdout $WORK/modtest3.json +go list -m -f '{{.Path}} {{.Version}} {{.Time.Format "2006-01-02"}}' dmitri.shuralyov.com/test/modtest3@latest +stdout '^dmitri.shuralyov.com/test/modtest3 v0.0.0-20181023043359-a85b471d5412 2018-10-22$' -- $WORK/modtest5.list -- v0.0.0-20190619020302-197a620e0c9a v0.5.0-alpha v0.5.0-alpha.0.20190619023908-3da23a9deb9e --- $WORK/modtest5.json -- -{ - "Path": "github.com/dmitshur-test/modtest5", - "Version": "v0.5.0-alpha", - "Time": "2019-06-18T19:04:46-07:00" -} -- $WORK/tinyproxy/dmitri.shuralyov.com/test/modtest3/@v/list -- v0.1.0-0.20161023043300-000000000000 v0.0.0-20181023043359-a85b471d5412 @@ -42,9 +37,3 @@ v0.0.0-20181023043359-a85b471d5412 "Version": "v0.0.0-20181023043359-a85b471d5412", "Time": "2018-10-22T21:33:59-07:00" } --- $WORK/modtest3.json -- -{ - "Path": "dmitri.shuralyov.com/test/modtest3", - "Version": "v0.0.0-20181023043359-a85b471d5412", - "Time": "2018-10-22T21:33:59-07:00" -} diff --git a/src/cmd/go/testdata/script/mod_list_replace_dir.txt b/src/cmd/go/testdata/script/mod_list_replace_dir.txt index cad7fe2528..f2f2d2b2bb 100644 --- a/src/cmd/go/testdata/script/mod_list_replace_dir.txt +++ b/src/cmd/go/testdata/script/mod_list_replace_dir.txt @@ -2,8 +2,11 @@ # module within the module cache. # Verifies golang.org/issue/29548 -env GO111MODULE=on -go mod download rsc.io/quote@v1.5.1 rsc.io/quote@v1.5.2 +# Populate go.sum and download dependencies. +go get -d + +# Ensure v1.5.2 is also in the cache so we can list it. +go mod download rsc.io/quote@v1.5.2 ! go list $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 stderr '^directory ..[/\\]pkg[/\\]mod[/\\]rsc.io[/\\]quote@v1.5.2 outside available modules$' @@ -17,3 +20,8 @@ module example.com/quoter require rsc.io/quote v1.5.2 replace rsc.io/quote => rsc.io/quote v1.5.1 + +-- use.go -- +package use + +import _ "rsc.io/quote" diff --git a/src/cmd/go/testdata/script/mod_list_retract.txt b/src/cmd/go/testdata/script/mod_list_retract.txt new file mode 100644 index 0000000000..4e177b3f54 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_list_retract.txt @@ -0,0 +1,108 @@ +# 'go list -mod=vendor -retracted' reports an error. +go mod vendor +! go list -m -retracted -mod=vendor +stderr '^go list -retracted cannot be used when vendoring is enabled$' +rm vendor + +# 'go list -retracted' reports an error in GOPATH mode. +env GO111MODULE=off +! go list -retracted +stderr '^go list -retracted can only be used in module-aware mode$' +env GO111MODULE= + +# 'go list pkg' does not show retraction. +go list -f '{{with .Module}}{{with .Retracted}}retracted{{end}}{{end}}' example.com/retract +! stdout . + +# 'go list -retracted pkg' shows retraction. +go list -retracted -f '{{with .Module}}{{with .Retracted}}retracted{{end}}{{end}}' example.com/retract +stdout retracted + +# 'go list -m' does not show retraction. +go list -m -f '{{with .Retracted}}retracted{{end}}' example.com/retract +! stdout . + +# 'go list -m -retracted' shows retraction. +go list -m -retracted -f '{{with .Retracted}}retracted{{end}}' example.com/retract + +# 'go list -m mod@version' does not show retraction. +go list -m -f '{{with .Retracted}}retracted{{end}}' example.com/retract@v1.0.0-unused +! stdout . + +# 'go list -m -retracted mod@version' shows an error if the go.mod that should +# contain the retractions is not available. +! go list -m -retracted example.com/retract/missingmod@v1.0.0 +stderr '^go list -m: loading module retractions: example.com/retract/missingmod@v1.9.0:.*404 Not Found$' +go list -e -m -retracted -f '{{.Error.Err}}' example.com/retract/missingmod@v1.0.0 +stdout '^loading module retractions: example.com/retract/missingmod@v1.9.0:.*404 Not Found$' + +# 'go list -m -retracted mod@version' shows retractions. +go list -m -retracted example.com/retract@v1.0.0-unused +stdout '^example.com/retract v1.0.0-unused \(retracted\)$' +go list -m -retracted -f '{{with .Retracted}}retracted{{end}}' example.com/retract@v1.0.0-unused +stdout retracted + +# 'go list -m mod@latest' selects a previous release version, not self-retracted latest. +go list -m -f '{{.Version}}{{with .Retracted}} retracted{{end}}' example.com/retract/self/prev@latest +stdout '^v1.1.0$' + +# 'go list -m -retracted mod@latest' selects the self-retracted latest version. +go list -m -retracted -f '{{.Version}}{{with .Retracted}} retracted{{end}}' example.com/retract/self/prev@latest +stdout '^v1.9.0 retracted$' + +# 'go list -m mod@latest' selects a pre-release version if all release versions are retracted. +go list -m -f '{{.Version}}{{with .Retracted}} retracted{{end}}' example.com/retract/self/prerelease@latest +stdout '^v1.9.1-pre$' + +# 'go list -m -retracted mod@latest' selects the self-retracted latest version. +go list -m -retracted -f '{{.Version}}{{with .Retracted}} retracted{{end}}' example.com/retract/self/prerelease@latest +stdout '^v1.9.0 retracted$' + +# 'go list -m mod@latest' selects a pseudo-version if all versions are retracted. +# TODO(golang.org/issue/24031): the proxy does not expose the pseudo-version, +# even if all release versions are retracted. +go list -m -e -f '{{.Error.Err}}' example.com/retract/self/pseudo@latest +stdout '^module example.com/retract/self/pseudo: no matching versions for query "latest"$' + +# 'go list -m mod@latest' reports an error if all versions are retracted. +go list -m -e -f '{{.Error.Err}}' example.com/retract/self/all@latest +stdout '^module example.com/retract/self/all: no matching versions for query "latest"$' + +# 'go list -m mod@<v1.10' selects a previous release version, not self-retracted latest. +# The @latest query is not special with respect to retractions. +go list -m -f '{{.Version}}{{with .Retracted}} retracted{{end}}' example.com/retract/self/prev@<v1.10 +stdout '^v1.1.0$' + +# 'go list -m -versions' hides retracted versions. +go list -m -versions example.com/retract +stdout '^example.com/retract v1.0.0-good v1.1.0$' + +# 'go list -m -retracted -versions' shows retracted versions. +go list -m -retracted -versions example.com/retract +stdout '^example.com/retract v1.0.0-bad v1.0.0-good v1.0.0-unused v1.1.0$' + +# 'go list -m -u -versions' loads retractions and does not show retracted versions. +go list -m -u -versions example.com/retract +stdout '^example.com/retract v1.0.0-good v1.1.0$' +go list -m -u -versions -f '{{with .Retracted}}retracted{{end}}' example.com/retract +stdout retracted + +# 'go list -m -u' shows retraction. +go list -m -u -f '{{with .Retracted}}retracted{{end}}' example.com/retract +stdout retracted + +# 'go list -m -u' does not suggest an update to a self-retracted latest version. +go list -m -u -f '{{with .Update}}{{.Version}}{{with .Retracted}} retracted{{end}}{{end}}' example.com/retract/self/prev@v1.0.0-bad +stdout '^v1.1.0$' + +-- go.mod -- +module example.com/use + +go 1.15 + +require example.com/retract v1.0.0-bad + +-- use.go -- +package use + +import _ "example.com/retract" diff --git a/src/cmd/go/testdata/script/mod_list_std.txt b/src/cmd/go/testdata/script/mod_list_std.txt index 76a3b00d1c..baf7908ab9 100644 --- a/src/cmd/go/testdata/script/mod_list_std.txt +++ b/src/cmd/go/testdata/script/mod_list_std.txt @@ -6,8 +6,13 @@ env GOPROXY=off # Outside of GOROOT, our vendored packages should be reported as part of the standard library. go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' std cmd -stdout ^vendor/golang.org/x/net/http2/hpack +stdout ^vendor/golang\.org/x/net/http2/hpack stdout ^cmd/vendor/golang\.org/x/arch/x86/x86asm +! stdout ^golang\.org/x/ + +# The dependencies of those packages should also be vendored. +go list -deps vendor/golang.org/x/crypto/chacha20 +stdout ^vendor/golang\.org/x/crypto/internal/subtle # cmd/... should match the same packages it used to match in GOPATH mode. go list cmd/... @@ -23,40 +28,57 @@ stdout ^bytes$ ! stdout ^builtin$ ! stdout ^cmd/ ! stdout ^vendor/ +! stdout ^golang\.org/x/ + +# Vendored dependencies should appear with their 'vendor/' paths in std (they're +# in GOROOT/src, but not in the 'std' module following the usual module-boundary +# rules). -# Within the std module, listing ./... should omit the 'std' prefix: -# the package paths should be the same via ./... or the 'std' meta-pattern. -# TODO(golang.org/issue/30241): Make that work. -# Today, they are listed in 'std' but not './...'. cd $GOROOT/src -go list ./... -! stdout ^vendor/golang.org/x # TODO: should be included, or should be omitted from 'std'. -cp stdout $WORK/listdot.txt go list std -stdout ^vendor/golang.org/x # TODO: remove vendor/ prefix -# TODO: cmp stdout $WORK/listdot.txt +stdout ^vendor/golang.org/x/net/http2/hpack +! stdout ^golang\.org/x + +# The dependencies of packages with an explicit 'vendor/' prefix should +# still themselves resolve to vendored packages. +go list -deps vendor/golang.org/x/crypto/chacha20 +stdout ^vendor/golang.org/x/crypto/internal/subtle +! stdout ^golang\.org/x + +# Within the std module, the dependencies of the non-vendored packages within +# std should appear to come from modules, but they should be loaded from the +# vendor directory (just like ordinary vendored module dependencies). go list all -stdout ^vendor/golang.org/x # TODO: remove vendor/ prefix. +stdout ^golang.org/x/ ! stdout ^std/ +! stdout ^cmd/ +! stdout ^vendor/ +go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' std +! stdout ^vendor/golang.org/x/net/http2/hpack +stdout ^golang.org/x/net/http2/hpack -# Within the std module, the vendored dependencies of std should appear -# to come from the actual modules. -# TODO(golang.org/issue/30241): Make that work. -# Today, they still have the vendor/ prefix. -go list std -stdout ^vendor/golang.org/x/net/http2/hpack # TODO -! stdout ^golang.org/x/net/http2/hpack # TODO +go list -f '{{.Dir}}' golang.org/x/net/http2/hpack +stdout $GOROOT[/\\]src[/\\]vendor -go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' std -# ! stdout ^vendor/golang.org/x/net/http2/hpack # TODO -! stdout ^golang.org/x/net/http2/hpack # TODO +# Within the std module, the packages within the module should omit the 'std/' +# prefix (they retain their own identities), but should respect normal module +# boundaries (vendored packages are not included in the module, even though they +# are included in the 'std' pattern). + +go list ./... +stdout ^bytes$ +! stdout ^builtin$ +! stdout ^cmd/ +! stdout ^vendor/ +! stdout ^golang\.org/x/ # Within std, the vendored dependencies of cmd should still appear to be part of cmd. + go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' cmd stdout ^cmd/vendor/golang\.org/x/arch/x86/x86asm diff --git a/src/cmd/go/testdata/script/mod_list_test.txt b/src/cmd/go/testdata/script/mod_list_test.txt index a99e4f36cd..f697af6c92 100644 --- a/src/cmd/go/testdata/script/mod_list_test.txt +++ b/src/cmd/go/testdata/script/mod_list_test.txt @@ -3,9 +3,19 @@ env GO111MODULE=on # go list -compiled -test must handle test-only packages # golang.org/issue/27097. go list -compiled -test +stdout -count=4 '^.' # 4 lines stdout '^m$' stdout '^m\.test$' stdout '^m \[m\.test\]$' +stdout '^m_test \[m\.test\]$' + +# https://golang.org/issue/39974: test packages should have the Module field populated. +go list -test -f '{{.ImportPath}}{{with .Module}}: {{.Path}}{{end}}' +stdout -count=4 '^.' # 4 lines +stdout '^m: m$' +stdout '^m\.test: m$' +stdout '^m \[m\.test\]: m$' +stdout '^m_test \[m\.test\]: m$' -- go.mod -- module m @@ -14,3 +24,7 @@ module m package x import "testing" func Test(t *testing.T) {} +-- x_x_test.go -- +package x_test +import "testing" +func Test(t *testing.T) {} diff --git a/src/cmd/go/testdata/script/mod_list_upgrade.txt b/src/cmd/go/testdata/script/mod_list_upgrade.txt index 474df0dc26..0cef04b89a 100644 --- a/src/cmd/go/testdata/script/mod_list_upgrade.txt +++ b/src/cmd/go/testdata/script/mod_list_upgrade.txt @@ -1,5 +1,9 @@ env GO111MODULE=on +# Populate go.sum +go list -m -mod=mod all + +# Check for upgrades. go list -m -u all stdout 'rsc.io/quote v1.2.0 \[v1\.5\.2\]' diff --git a/src/cmd/go/testdata/script/mod_load_badchain.txt b/src/cmd/go/testdata/script/mod_load_badchain.txt index 67d9a1584f..e943179c54 100644 --- a/src/cmd/go/testdata/script/mod_load_badchain.txt +++ b/src/cmd/go/testdata/script/mod_load_badchain.txt @@ -28,10 +28,10 @@ cmp stderr list-expected # Try listing a package that imports a package # in a module without a requirement. go mod edit -droprequire example.com/badchain/a -! go list m/use +! go list -mod=mod m/use cmp stderr list-missing-expected -! go list -test m/testuse +! go list -mod=mod -test m/testuse cmp stderr list-missing-test-expected -- go.mod.orig -- diff --git a/src/cmd/go/testdata/script/mod_load_badmod.txt b/src/cmd/go/testdata/script/mod_load_badmod.txt index 68c8b3792b..fa22e1808b 100644 --- a/src/cmd/go/testdata/script/mod_load_badmod.txt +++ b/src/cmd/go/testdata/script/mod_load_badmod.txt @@ -1,14 +1,13 @@ # Unknown lines should be ignored in dependency go.mod files. -env GO111MODULE=on -go list -m all +go list -m -mod=mod all # ... and in replaced dependency go.mod files. cp go.mod go.mod.usesub -go list -m all +go list -m -mod=mod all # ... but not in the main module. cp go.mod.bad go.mod -! go list -m all +! go list -m -mod=mod all stderr 'unknown directive: hello' -- go.mod -- diff --git a/src/cmd/go/testdata/script/mod_load_badzip.txt b/src/cmd/go/testdata/script/mod_load_badzip.txt index c5ba18e9f0..65374d2a6d 100644 --- a/src/cmd/go/testdata/script/mod_load_badzip.txt +++ b/src/cmd/go/testdata/script/mod_load_badzip.txt @@ -5,10 +5,8 @@ env GO111MODULE=on stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt' ! grep rsc.io/badzip go.mod -# TODO(golang.org/issue/31730): 'go build' should print the error below if the -# requirement is not present. go mod edit -require rsc.io/badzip@v1.0.0 -! go build rsc.io/badzip +! go build -mod=mod rsc.io/badzip stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt' -- go.mod -- diff --git a/src/cmd/go/testdata/script/mod_load_replace_mismatch.txt b/src/cmd/go/testdata/script/mod_load_replace_mismatch.txt index 74dbb34b8a..067e209b01 100644 --- a/src/cmd/go/testdata/script/mod_load_replace_mismatch.txt +++ b/src/cmd/go/testdata/script/mod_load_replace_mismatch.txt @@ -18,6 +18,6 @@ package use import _ "rsc.io/quote" -- want -- -go: example.com/quote@v1.5.2: parsing go.mod: +go: rsc.io/quote@v1.5.2 (replaced by example.com/quote@v1.5.2): parsing go.mod: module declares its path as: rsc.io/Quote but was required as: rsc.io/quote diff --git a/src/cmd/go/testdata/script/mod_missingpkg_prerelease.txt b/src/cmd/go/testdata/script/mod_missingpkg_prerelease.txt index 319ff85587..9c250e7d1c 100644 --- a/src/cmd/go/testdata/script/mod_missingpkg_prerelease.txt +++ b/src/cmd/go/testdata/script/mod_missingpkg_prerelease.txt @@ -1,7 +1,7 @@ env GO111MODULE=on -! go list use.go -stderr 'example.com/missingpkg/deprecated: package provided by example.com/missingpkg at latest version v1.0.0 but not at required version v1.0.1-beta' +! go list -mod=mod -deps use.go +stderr '^use.go:4:2: package example.com/missingpkg/deprecated provided by example.com/missingpkg at latest version v1.0.0 but not at required version v1.0.1-beta$' -- go.mod -- module m diff --git a/src/cmd/go/testdata/script/mod_modinfo.txt b/src/cmd/go/testdata/script/mod_modinfo.txt index fb31f9e43b..d9e9fdec21 100644 --- a/src/cmd/go/testdata/script/mod_modinfo.txt +++ b/src/cmd/go/testdata/script/mod_modinfo.txt @@ -6,6 +6,7 @@ env GO111MODULE=on cd x go mod edit -require=rsc.io/quote@v1.5.2 go mod edit -replace=rsc.io/quote@v1.5.2=rsc.io/quote@v1.0.0 +go mod tidy # populate go.sum # Build a binary and ensure that it can output its own debug info. # The debug info should be accessible before main starts (golang.org/issue/29628). diff --git a/src/cmd/go/testdata/script/mod_multirepo.txt b/src/cmd/go/testdata/script/mod_multirepo.txt index 7f977e80f6..0f335a11f0 100644 --- a/src/cmd/go/testdata/script/mod_multirepo.txt +++ b/src/cmd/go/testdata/script/mod_multirepo.txt @@ -7,6 +7,7 @@ go list -deps -f {{.Dir}} # v2 import should use a downloaded module # both without an explicit go.mod entry ... cp tmp/use_v2.go x.go +go get -d . go list -deps -f {{.Dir}} stdout 'pkg[\\/]mod[\\/]rsc.io[\\/]quote[\\/]v2@v2.0.1$' diff --git a/src/cmd/go/testdata/script/mod_notall.txt b/src/cmd/go/testdata/script/mod_notall.txt new file mode 100644 index 0000000000..1657c8d2d0 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_notall.txt @@ -0,0 +1,99 @@ +# This test demonstrates go commands that combine the 'all' pattern +# with packages outside of 'all'. + +# With -deps, 'all' should include test dependencies of packages in the main +# module, but not should not include test dependencies of packages imported only +# by other root patterns. + +env GOFLAGS=-mod=mod +cp go.mod go.mod.orig + +go list -deps all x/otherroot + +stdout '^x/inall$' +stdout '^x/inall/fromtest$' +stdout '^x/inall/fromtestinall$' +stdout '^x/otherroot$' +stdout '^x/otherdep$' + +! stdout '^x/fromotherroottest$' +! stdout '^y/fromotherdeptest$' + +cmp go.mod go.mod.orig + +# With -deps -test, test dependencies of other roots should be included, +# but test dependencies of non-roots should not. + +go list -deps -test all x/otherroot +stdout '^x/inall$' +stdout '^x/inall/fromtest$' +stdout '^x/inall/fromtestinall$' +stdout '^x/otherroot$' +stdout '^x/otherdep$' + +stdout '^x/fromotherroottest$' +! stdout '^y/fromotherdeptest$' + +cmp go.mod go.mod.orig + +-- m.go -- +package m + +import _ "x/inall" +-- m_test.go -- +package m_test + +import _ "x/inall/fromtest" +-- go.mod -- +module m + +go 1.15 + +require x v0.1.0 + +replace ( + x v0.1.0 => ./x + y v0.1.0 => ./y +) +-- x/go.mod -- +module x + +go 1.15 +-- x/inall/inall.go -- +package inall +-- x/inall/inall_test.go -- +package inall_test + +import _ "x/inall/fromtestinall" +-- x/inall/fromtest/fromtest.go -- +package fromtest +-- x/inall/fromtestinall/fromtestinall.go -- +package fromtestinall +-- x/otherroot/otherroot.go -- +package otherroot + +import _ "x/otherdep" +-- x/otherroot/otherroot_test.go -- +package otherroot_test + +import _ "x/fromotherroottest" +-- x/fromotherroottest/fromotherroottest.go -- +package fromotherroottest +-- x/otherdep/otherdep.go -- +package otherdep +-- x/otherdep/otherdep_test.go -- +package otherdep_test + +import _ "y/fromotherdeptest" +-- x/otherroot/testonly/testonly.go -- +package testonly +-- y/go.mod -- +module y + +go 1.15 +-- y/fromotherdeptest/fromotherdeptest.go -- +// Package fromotherdeptest is a test dependency of x/otherdep that is +// not declared in x/go.mod. If the loader resolves this package, +// it will add this module to the main module's go.mod file, +// and we can detect the mistake. +package fromotherdeptest diff --git a/src/cmd/go/testdata/script/mod_permissions.txt b/src/cmd/go/testdata/script/mod_permissions.txt index 11fb4754f8..2d32dcd10f 100644 --- a/src/cmd/go/testdata/script/mod_permissions.txt +++ b/src/cmd/go/testdata/script/mod_permissions.txt @@ -12,7 +12,7 @@ chmod 0640 go.mod chmod 0604 go.sum go mod edit -module=golang.org/issue/34634 -go build . +go get -d cmp go.mod go.mod.want cmp go.sum go.sum.want diff --git a/src/cmd/go/testdata/script/mod_query.txt b/src/cmd/go/testdata/script/mod_query.txt index e87ca302f0..e10185709d 100644 --- a/src/cmd/go/testdata/script/mod_query.txt +++ b/src/cmd/go/testdata/script/mod_query.txt @@ -1,5 +1,10 @@ env GO111MODULE=on +# Populate go.sum. +# TODO(golang.org/issue/41297): we shouldn't need go.sum. None of the commands +# below depend on the build list. +go mod download + go list -m -versions rsc.io/quote stdout '^rsc.io/quote v1.0.0 v1.1.0 v1.2.0 v1.2.1 v1.3.0 v1.4.0 v1.5.0 v1.5.1 v1.5.2 v1.5.3-pre1$' @@ -30,3 +35,8 @@ stdout 'no matching versions for query ">v1.5.3"' -- go.mod -- module x require rsc.io/quote v1.0.0 + +-- use.go -- +package use + +import _ "rsc.io/quote" diff --git a/src/cmd/go/testdata/script/mod_query_exclude.txt b/src/cmd/go/testdata/script/mod_query_exclude.txt index a64a8e1086..742c6f17e3 100644 --- a/src/cmd/go/testdata/script/mod_query_exclude.txt +++ b/src/cmd/go/testdata/script/mod_query_exclude.txt @@ -1,23 +1,43 @@ env GO111MODULE=on +# list excluded version +go list -modfile=go.exclude.mod -m rsc.io/quote@v1.5.0 +stdout '^rsc.io/quote v1.5.0$' + +# list versions should not print excluded versions +go list -m -versions rsc.io/quote +stdout '\bv1.5.0\b' +go list -modfile=go.exclude.mod -m -versions rsc.io/quote +! stdout '\bv1.5.0\b' + +# list query with excluded version +go list -m rsc.io/quote@>=v1.5 +stdout '^rsc.io/quote v1.5.0$' +go list -modfile=go.exclude.mod -m rsc.io/quote@>=v1.5 +stdout '^rsc.io/quote v1.5.1$' + # get excluded version -cp go.mod1 go.mod -! go get rsc.io/quote@v1.5.0 -stderr 'rsc.io/quote@v1.5.0 excluded' +cp go.exclude.mod go.exclude.mod.orig +! go get -modfile=go.exclude.mod -d rsc.io/quote@v1.5.0 +stderr '^go get rsc.io/quote@v1.5.0: rsc.io/quote@v1.5.0: excluded by go.mod$' # get non-excluded version -cp go.mod1 go.mod -go get rsc.io/quote@v1.5.1 +cp go.exclude.mod.orig go.exclude.mod +go get -modfile=go.exclude.mod -d rsc.io/quote@v1.5.1 stderr 'rsc.io/quote v1.5.1' -# get range with excluded version -cp go.mod1 go.mod -go get rsc.io/quote@>=v1.5 -go list -m ...quote +# get query with excluded version +cp go.exclude.mod.orig go.exclude.mod +go get -modfile=go.exclude.mod -d rsc.io/quote@>=v1.5 +go list -modfile=go.exclude.mod -m ...quote stdout 'rsc.io/quote v1.5.[1-9]' --- go.mod1 -- +-- go.mod -- module x + +-- go.exclude.mod -- +module x + exclude rsc.io/quote v1.5.0 -- x.go -- diff --git a/src/cmd/go/testdata/script/mod_replace.txt b/src/cmd/go/testdata/script/mod_replace.txt index c21f172002..dc9667f1d0 100644 --- a/src/cmd/go/testdata/script/mod_replace.txt +++ b/src/cmd/go/testdata/script/mod_replace.txt @@ -4,7 +4,7 @@ env GO111MODULE=on cp go.mod go.mod.orig # Make sure the test builds without replacement. -go build -o a1.exe . +go build -mod=mod -o a1.exe . exec ./a1.exe stdout 'Don''t communicate by sharing memory' @@ -32,7 +32,7 @@ stderr 'rsc.io/quote/v3@v3.0.0 used for two different module paths \(not-rsc.io/ # Modules that do not (yet) exist upstream can be replaced too. cp go.mod.orig go.mod go mod edit -replace=not-rsc.io/quote/v3@v3.1.0=./local/rsc.io/quote/v3 -go build -o a5.exe ./usenewmodule +go build -mod=mod -o a5.exe ./usenewmodule ! stderr 'finding not-rsc.io/quote/v3' grep 'not-rsc.io/quote/v3 v3.1.0' go.mod exec ./a5.exe diff --git a/src/cmd/go/testdata/script/mod_replace_gopkgin.txt b/src/cmd/go/testdata/script/mod_replace_gopkgin.txt index 28c1196284..df752d9716 100644 --- a/src/cmd/go/testdata/script/mod_replace_gopkgin.txt +++ b/src/cmd/go/testdata/script/mod_replace_gopkgin.txt @@ -11,6 +11,7 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOFLAGS=-mod=mod # Replacing gopkg.in/[…].vN with a repository with a root go.mod file # specifying […].vN and a compatible version should succeed, even if @@ -34,7 +35,7 @@ go list -m gopkg.in/src-d/go-git.v4 # A mismatched gopkg.in path should not be able to replace a different major version. cd ../3-to-gomod-4 ! go list -m gopkg.in/src-d/go-git.v3 -stderr '^go: gopkg\.in/src-d/go-git\.v3@v3.0.0-20190801152248-0d1a009cbb60: invalid version: go\.mod has non-\.\.\.\.v3 module path "gopkg\.in/src-d/go-git\.v4" at revision 0d1a009cbb60$' +stderr '^go: gopkg\.in/src-d/go-git\.v3@v3\.2\.0 \(replaced by gopkg\.in/src-d/go-git\.v3@v3\.0\.0-20190801152248-0d1a009cbb60\): version "v3\.0\.0-20190801152248-0d1a009cbb60" invalid: go\.mod has non-\.\.\.\.v3 module path "gopkg\.in/src-d/go-git\.v4" at revision 0d1a009cbb60$' -- 4-to-4/go.mod -- module golang.org/issue/34254 diff --git a/src/cmd/go/testdata/script/mod_replace_import.txt b/src/cmd/go/testdata/script/mod_replace_import.txt index 54b1a12448..b4de5c50f7 100644 --- a/src/cmd/go/testdata/script/mod_replace_import.txt +++ b/src/cmd/go/testdata/script/mod_replace_import.txt @@ -7,6 +7,7 @@ cp go.mod go.mod.orig cmp go.mod go.mod.orig # 'go list' should resolve imports using replacements. +go get -d go list all stdout 'example.com/a/b$' stdout 'example.com/x/v3$' diff --git a/src/cmd/go/testdata/script/mod_require_exclude.txt b/src/cmd/go/testdata/script/mod_require_exclude.txt index 60f7e3fa91..9156d4ce5d 100644 --- a/src/cmd/go/testdata/script/mod_require_exclude.txt +++ b/src/cmd/go/testdata/script/mod_require_exclude.txt @@ -1,16 +1,51 @@ # build with no newer version to satisfy exclude env GO111MODULE=on -! go list -m all -stderr 'no newer version available' +cp go.mod go.mod.orig + +# With the selected version excluded, commands that query that version without +# updating go.mod should fail. + +! go list -mod=readonly -m all +stderr '^go: ignoring requirement on excluded version rsc.io/sampler v1\.99\.99$' +stderr '^go: updates to go.mod needed, disabled by -mod=readonly$' +! stdout '^rsc.io/sampler v1.99.99' +cmp go.mod go.mod.orig + +! go list -mod=vendor -m rsc.io/sampler +stderr '^go: ignoring requirement on excluded version rsc.io/sampler v1\.99\.99$' +stderr '^go list -m: module rsc.io/sampler: can''t resolve module using the vendor directory\n\t\(Use -mod=mod or -mod=readonly to bypass\.\)$' +! stdout '^rsc.io/sampler v1.99.99' +cmp go.mod go.mod.orig + +# With the selected version excluded, commands that load only modules should +# drop the excluded module. + +go list -m -mod=mod all +stderr '^go: dropping requirement on excluded version rsc.io/sampler v1\.99\.99$' +stdout '^x$' +! stdout '^rsc.io/sampler' +cmp go.mod go.moddrop + +# With the latest version excluded, 'go list' should resolve needed packages +# from the next-highest version. + +cp go.mod.orig go.mod +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +stderr '^go: dropping requirement on excluded version rsc.io/sampler v1\.99\.99$' +stdout '^x $' +! stdout '^rsc.io/sampler v1.99.99' +stdout '^rsc.io/sampler v1.3.0' # build with newer version available cp go.mod2 go.mod -go list -m all +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +stderr '^go: dropping requirement on excluded version rsc.io/quote v1\.5\.1$' stdout 'rsc.io/quote v1.5.2' # build with excluded newer version cp go.mod3 go.mod -go list -m all +go list -mod=mod -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all +! stderr '^go: dropping requirement' stdout 'rsc.io/quote v1.5.1' -- x.go -- @@ -19,15 +54,28 @@ import _ "rsc.io/quote" -- go.mod -- module x -exclude rsc.io/sampler latest -require rsc.io/sampler latest +go 1.13 + +exclude rsc.io/sampler v1.99.99 +require rsc.io/sampler v1.99.99 +-- go.moddrop -- +module x + +go 1.13 + +exclude rsc.io/sampler v1.99.99 -- go.mod2 -- module x + +go 1.13 + exclude rsc.io/quote v1.5.1 require rsc.io/quote v1.5.1 - -- go.mod3 -- module x + +go 1.13 + exclude rsc.io/quote v1.5.2 require rsc.io/quote v1.5.1 diff --git a/src/cmd/go/testdata/script/mod_retention.txt b/src/cmd/go/testdata/script/mod_retention.txt index 1d83e6c07e..a4441c4b3c 100644 --- a/src/cmd/go/testdata/script/mod_retention.txt +++ b/src/cmd/go/testdata/script/mod_retention.txt @@ -7,7 +7,7 @@ env GO111MODULE=on # Control case: verify that go.mod.tidy is actually tidy. cp go.mod.tidy go.mod -go list all +go list -mod=mod all cmp go.mod go.mod.tidy @@ -35,7 +35,7 @@ cmp go.mod go.mod.tidy # "// indirect" comments should be removed if direct dependencies are seen. # changes. cp go.mod.indirect go.mod -go list all +go list -mod=mod all cmp go.mod go.mod.tidy # "// indirect" comments should be added if appropriate. @@ -63,7 +63,7 @@ cmp go.mod go.mod.tidy # A missing "go" version directive should be added. # However, that should not remove other redundant requirements. cp go.mod.nogo go.mod -go list all +go list -mod=mod all cmpenv go.mod go.mod.currentgo diff --git a/src/cmd/go/testdata/script/mod_retract.txt b/src/cmd/go/testdata/script/mod_retract.txt new file mode 100644 index 0000000000..a52e05bc72 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_retract.txt @@ -0,0 +1,43 @@ +cp go.mod go.mod.orig + +# Populate go.sum. +go mod download + +# 'go list pkg' does not report an error when a retracted version is used. +go list -e -f '{{if .Error}}{{.Error}}{{end}}' ./use +! stdout . +cmp go.mod go.mod.orig + +# Nor does 'go build'. +[!short] go build ./use +[!short] ! stderr . +[!short] cmp go.mod go.mod.orig + +# Neither 'go list' nor 'go build' should download go.mod from the version +# that would list retractions. +exists $GOPATH/pkg/mod/cache/download/example.com/retract/@v/v1.0.0-bad.mod +! exists $GOPATH/pkg/mod/cache/download/example.com/retract/@v/v1.1.0.mod + +# Importing a package from a module with a retracted latest version will +# select the latest non-retracted version. +go get -d ./use_self_prev +go list -m example.com/retract/self/prev +stdout '^example.com/retract/self/prev v1.1.0$' +exists $GOPATH/pkg/mod/cache/download/example.com/retract/self/prev/@v/v1.9.0.mod + +-- go.mod -- +module example.com/use + +go 1.15 + +require example.com/retract v1.0.0-bad + +-- use/use.go -- +package use + +import _ "example.com/retract" + +-- use_self_prev/use.go -- +package use_self_prev + +import _ "example.com/retract/self/prev" diff --git a/src/cmd/go/testdata/script/mod_retract_rationale.txt b/src/cmd/go/testdata/script/mod_retract_rationale.txt new file mode 100644 index 0000000000..584c3a3849 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_retract_rationale.txt @@ -0,0 +1,79 @@ +# When there is no rationale, 'go get' should print a hard-coded message. +go get -d example.com/retract/rationale@v1.0.0-empty +stderr '^go: warning: example.com/retract/rationale@v1.0.0-empty is retracted: retracted by module author$' + +# 'go list' should print the same hard-coded message. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale +stdout '^\[retracted by module author\]$' + + +# When there is a multi-line message, 'go get' should print the first line. +go get -d example.com/retract/rationale@v1.0.0-multiline1 +stderr '^go: warning: example.com/retract/rationale@v1.0.0-multiline1 is retracted: short description$' +! stderr 'detail' + +# 'go list' should show the full message. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale +cmp stdout multiline + +# 'go get' output should be the same whether the retraction appears at top-level +# or in a block. +go get -d example.com/retract/rationale@v1.0.0-multiline2 +stderr '^go: warning: example.com/retract/rationale@v1.0.0-multiline2 is retracted: short description$' +! stderr 'detail' + +# Same for 'go list'. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale +cmp stdout multiline + + +# 'go get' should omit long messages. +go get -d example.com/retract/rationale@v1.0.0-long +stderr '^go: warning: example.com/retract/rationale@v1.0.0-long is retracted: \(rationale omitted: too long\)' + +# 'go list' should show the full message. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale +stdout '^\[lo{500}ng\]$' + + +# 'go get' should omit messages with unprintable characters. +go get -d example.com/retract/rationale@v1.0.0-unprintable +stderr '^go: warning: example.com/retract/rationale@v1.0.0-unprintable is retracted: \(rationale omitted: contains non-printable characters\)' + +# 'go list' should show the full message. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale +stdout '^\[Ends with a BEL character. Beep!\x07\]$' + + +# When there is a comment on a block, but not on individual retractions within +# the block, the rationale should come from the block comment. +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale@v1.0.0-block +stdout '^\[block comment\]$' +go list -m -retracted -f '{{.Retracted}}' example.com/retract/rationale@v1.0.0-blockwithcomment +stdout '^\[inner comment\]$' + + +# When a version is covered by multiple retractions, all retractions should +# be reported in the order they appear in the file. +go list -m -retracted -f '{{range .Retracted}}{{.}},{{end}}' example.com/retract/rationale@v1.0.0-order +stdout '^degenerate range,single version,$' +go list -m -retracted -f '{{range .Retracted}}{{.}},{{end}}' example.com/retract/rationale@v1.0.1-order +stdout '^single version,degenerate range,$' + +# 'go get' will only report the first retraction to avoid being too verbose. +go get -d example.com/retract/rationale@v1.0.0-order +stderr '^go: warning: example.com/retract/rationale@v1.0.0-order is retracted: degenerate range$' +go get -d example.com/retract/rationale@v1.0.1-order +stderr '^go: warning: example.com/retract/rationale@v1.0.1-order is retracted: single version$' + +-- go.mod -- +module m + +go 1.14 + +-- multiline -- +[short description +more + +detail +suffix] diff --git a/src/cmd/go/testdata/script/mod_retract_replace.txt b/src/cmd/go/testdata/script/mod_retract_replace.txt new file mode 100644 index 0000000000..7aec438dda --- /dev/null +++ b/src/cmd/go/testdata/script/mod_retract_replace.txt @@ -0,0 +1,61 @@ +# If the latest unretracted version of a module is replaced, 'go list' should +# obtain retractions from the replacement. + +# Populate go.sum. +go get -d + +# The latest version, v1.9.0, is not available on the proxy. +! go list -m -retracted example.com/retract/missingmod +stderr '^go list -m: loading module retractions: example.com/retract/missingmod@v1.9.0:.*404 Not Found$' + +# If we replace that version, we should see retractions. +go mod edit -replace=example.com/retract/missingmod@v1.9.0=./missingmod-v1.9.0 +go list -m -retracted -f '{{range .Retracted}}{{.}}{{end}}' example.com/retract/missingmod +stdout '^bad version$' + +# If we replace the retracted version, we should not see a retraction. +go mod edit -replace=example.com/retract/missingmod=./missingmod-v1.9.0 +go list -m -retracted -f '{{if not .Retracted}}good version{{end}}' example.com/retract/missingmod +stdout '^good version$' + + +# If a replacement version is retracted, we should see a retraction. +# It should appear in both the replaced module and the replacement, as other +# fields like GoMod do. +go list -m -retracted -f '{{range .Retracted}}{{.}}{{end}}' example.com/retract +! stdout . +go list -m -retracted -f '{{if .Replace}}replaced{{end}}' example.com/retract +! stdout . +go mod edit -replace example.com/retract@v1.0.0-good=example.com/retract@v1.0.0-bad +go list -m -mod=mod -retracted -f '{{range .Retracted}}{{.}}{{end}}' example.com/retract +stdout '^bad$' +go list -m -mod=mod -retracted -f '{{with .Replace}}{{range .Retracted}}{{.}}{{end}}{{end}}' example.com/retract +stdout '^bad$' + +-- go.mod -- +module m + +go 1.14 + +require ( + example.com/retract v1.0.0-good + example.com/retract/missingmod v1.0.0 +) +-- use.go -- +package use + +import ( + _ "example.com/retract" + _ "example.com/retract/missingmod" +) +-- missingmod-v1.0.0/go.mod -- +module example.com/retract/missingmod + +go 1.14 +-- missingmod-v1.9.0/go.mod -- +module example.com/retract/missingmod + +go 1.14 + +// bad version +retract v1.0.0 diff --git a/src/cmd/go/testdata/script/mod_std_vendor.txt b/src/cmd/go/testdata/script/mod_std_vendor.txt index 5986cff594..fb954d74ed 100644 --- a/src/cmd/go/testdata/script/mod_std_vendor.txt +++ b/src/cmd/go/testdata/script/mod_std_vendor.txt @@ -37,12 +37,10 @@ stderr 'use of vendored package' # When run within the 'std' module, 'go list -test' should report vendored # transitive dependencies at their original module paths. -# TODO(golang.org/issue/30241): Make that work. -# Today, they're standard packages as long as they exist. cd $GOROOT/src go list -test -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' net/http -stdout ^vendor/golang.org/x/net/http2/hpack # TODO: remove vendor/ prefix -! stdout ^golang.org/x/net/http2/hpack +stdout ^golang.org/x/net/http2/hpack +! stdout ^vendor/golang.org/x/net/http2/hpack -- go.mod -- module m diff --git a/src/cmd/go/testdata/script/mod_sum_lookup.txt b/src/cmd/go/testdata/script/mod_sum_lookup.txt index ed80a44984..e021921380 100644 --- a/src/cmd/go/testdata/script/mod_sum_lookup.txt +++ b/src/cmd/go/testdata/script/mod_sum_lookup.txt @@ -1,13 +1,14 @@ # When we attempt to resolve an import that doesn't exist, we should not save # hashes for downloaded modules. # Verifies golang.org/issue/36260. -go list -e -tags=ignore ./noexist +# TODO(golang.org/issue/26603): use 'go mod tidy -e' when implemented. +go list -e -mod=mod -tags=ignore ./noexist ! exists go.sum # When an import is resolved successfully, we should only save hashes for # the module that provides the package, not for other modules looked up. # Verifies golang.org/issue/31580. -go list ./exist +go get -d ./exist grep '^example.com/join v1.1.0 h1:' go.sum ! grep '^example.com/join/subpkg' go.sum cp go.sum go.list.sum diff --git a/src/cmd/go/testdata/script/mod_sumdb.txt b/src/cmd/go/testdata/script/mod_sumdb.txt index caf97e9699..68bbd9c274 100644 --- a/src/cmd/go/testdata/script/mod_sumdb.txt +++ b/src/cmd/go/testdata/script/mod_sumdb.txt @@ -15,6 +15,12 @@ stderr 'localhost.localdev/sumdb: h1:wrong' stderr 'SECURITY ERROR\nThis download does NOT match the one reported by the checksum server.' ! go get -d rsc.io/sampler ! go get -d golang.org/x/text + +go mod edit -require rsc.io/quote@v1.5.2 +! go list all +stderr 'go: rsc.io/quote@v1.5.2: verifying go.mod: checksum mismatch' +stderr 'SECURITY ERROR\n' + rm go.sum # switching to truthful sumdb detects timeline inconsistency diff --git a/src/cmd/go/testdata/script/mod_sumdb_golang.txt b/src/cmd/go/testdata/script/mod_sumdb_golang.txt index 40a07fc7e9..cc0b0da474 100644 --- a/src/cmd/go/testdata/script/mod_sumdb_golang.txt +++ b/src/cmd/go/testdata/script/mod_sumdb_golang.txt @@ -9,7 +9,7 @@ env GOPROXY=https://proxy.golang.org go env GOSUMDB stdout '^sum.golang.org$' -# download direct from github +# Download direct from github. [!net] skip [!exec:git] skip env GOSUMDB=sum.golang.org @@ -17,11 +17,13 @@ env GOPROXY=direct go get -d rsc.io/quote@v1.5.2 cp go.sum saved.sum -# download from proxy.golang.org with go.sum entry already +# Download from proxy.golang.org with go.sum entry already. +# Use 'go list' instead of 'go get' since the latter may download extra go.mod +# files not listed in go.sum. go clean -modcache env GOSUMDB= env GOPROXY= -go get -x -d rsc.io/quote@v1.5.2 +go list -x -deps rsc.io/quote ! stderr github stderr proxy.golang.org/rsc.io/quote ! stderr sum.golang.org/tile @@ -32,7 +34,7 @@ cmp go.sum saved.sum # Should use the checksum database to validate new go.sum lines, # but not need to fetch any new data from the proxy. rm go.sum -go get -x -d rsc.io/quote@v1.5.2 +go list -mod=mod -x rsc.io/quote ! stderr github ! stderr proxy.golang.org/rsc.io/quote stderr sum.golang.org/tile @@ -43,7 +45,7 @@ cmp go.sum saved.sum env TESTGOPROXY404=1 go clean -modcache rm go.sum -go get -x -d rsc.io/quote@v1.5.2 +go list -mod=mod -x rsc.io/quote stderr 'proxy.golang.org.*404 testing' stderr github.com/rsc cmp go.sum saved.sum diff --git a/src/cmd/go/testdata/script/mod_symlink.txt b/src/cmd/go/testdata/script/mod_symlink.txt index 49bece2b84..dbc23fb8f0 100644 --- a/src/cmd/go/testdata/script/mod_symlink.txt +++ b/src/cmd/go/testdata/script/mod_symlink.txt @@ -1,16 +1,19 @@ env GO111MODULE=on [!symlink] skip -# 'go list' should resolve modules of imported packages. +# 'go get -d' should resolve modules of imported packages. +go get -d go list -deps -f '{{.Module}}' . stdout golang.org/x/text +go get -d ./subpkg go list -deps -f '{{.Module}}' ./subpkg stdout golang.org/x/text # Create a copy of the module using symlinks in src/links. mkdir links symlink links/go.mod -> $GOPATH/src/go.mod +symlink links/go.sum -> $GOPATH/src/go.sum symlink links/issue.go -> $GOPATH/src/issue.go mkdir links/subpkg symlink links/subpkg/issue.go -> $GOPATH/src/subpkg/issue.go diff --git a/src/cmd/go/testdata/script/mod_symlink_dotgo.txt b/src/cmd/go/testdata/script/mod_symlink_dotgo.txt new file mode 100644 index 0000000000..d4cc143a36 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_symlink_dotgo.txt @@ -0,0 +1,17 @@ +env GO111MODULE=on +[!symlink] skip + +symlink dir.go -> dir + +# Issue #39841: symlinks to directories should be ignored, not treated as source files. +go list -f '{{range .GoFiles}}{{.}}{{"\n"}}{{end}}' . +stdout 'p\.go$' +! stdout 'dir\.go$' + +-- go.mod -- +module example.com +go 1.15 +-- p.go -- +package p +-- dir/README.txt -- +This file exists to ensure that dir is a directory. diff --git a/src/cmd/go/testdata/script/mod_test.txt b/src/cmd/go/testdata/script/mod_test.txt index 8f2da2f2a5..50f00355c1 100644 --- a/src/cmd/go/testdata/script/mod_test.txt +++ b/src/cmd/go/testdata/script/mod_test.txt @@ -1,4 +1,5 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # TODO(bcmills): Convert the 'go test' calls below to 'go list -test' once 'go diff --git a/src/cmd/go/testdata/script/mod_tidy_replace.txt b/src/cmd/go/testdata/script/mod_tidy_replace.txt index c3158f8610..7b00bf1384 100644 --- a/src/cmd/go/testdata/script/mod_tidy_replace.txt +++ b/src/cmd/go/testdata/script/mod_tidy_replace.txt @@ -1,4 +1,5 @@ env GO111MODULE=on +env GOFLAGS=-mod=mod [short] skip # golang.org/issue/30166: 'go mod tidy' should not crash if a replaced module is diff --git a/src/cmd/go/testdata/script/mod_upgrade_patch.txt b/src/cmd/go/testdata/script/mod_upgrade_patch.txt index 3939e54c1b..1ef25b9aef 100644 --- a/src/cmd/go/testdata/script/mod_upgrade_patch.txt +++ b/src/cmd/go/testdata/script/mod_upgrade_patch.txt @@ -2,6 +2,7 @@ env GO111MODULE=on [short] skip # Initially, we are at v1.0.0 for all dependencies. +go get -d cp go.mod go.mod.orig go list -m all stdout '^patch.example.com/direct v1.0.0' diff --git a/src/cmd/go/testdata/script/mod_vcs_missing.txt b/src/cmd/go/testdata/script/mod_vcs_missing.txt index a755935b53..f8be43cf4c 100644 --- a/src/cmd/go/testdata/script/mod_vcs_missing.txt +++ b/src/cmd/go/testdata/script/mod_vcs_missing.txt @@ -5,14 +5,14 @@ env GO111MODULE=on env GOPROXY=direct cd empty -! go list launchpad.net/gocheck +! go get -d launchpad.net/gocheck stderr '"bzr": executable file not found' cd .. # 1.11 used to give the cryptic error "cannot find module for path" here, but # only for a main package. cd main -! go build +! go build -mod=mod stderr '"bzr": executable file not found' cd .. diff --git a/src/cmd/go/testdata/script/mod_vendor_build.txt b/src/cmd/go/testdata/script/mod_vendor_build.txt index 0c359cea6e..4efda55e08 100644 --- a/src/cmd/go/testdata/script/mod_vendor_build.txt +++ b/src/cmd/go/testdata/script/mod_vendor_build.txt @@ -1,6 +1,9 @@ env GO111MODULE=on [short] skip +# Populate go.mod and go.sum. +go mod tidy + # initial conditions: using sampler v1.3.0, not listed in go.mod. go list -deps stdout rsc.io/sampler diff --git a/src/cmd/go/testdata/script/mod_verify.txt b/src/cmd/go/testdata/script/mod_verify.txt index 3918400435..43812d069f 100644 --- a/src/cmd/go/testdata/script/mod_verify.txt +++ b/src/cmd/go/testdata/script/mod_verify.txt @@ -56,7 +56,7 @@ go mod tidy # Packages below module root should not be mentioned in go.sum. rm go.sum go mod edit -droprequire rsc.io/quote -go list rsc.io/quote/buggy # re-resolves import path and updates go.mod +go get -d rsc.io/quote/buggy grep '^rsc.io/quote v1.5.2/go.mod ' go.sum ! grep buggy go.sum diff --git a/src/cmd/go/testdata/script/mod_why.txt b/src/cmd/go/testdata/script/mod_why.txt index 10a4f9fbea..c0ff4647a7 100644 --- a/src/cmd/go/testdata/script/mod_why.txt +++ b/src/cmd/go/testdata/script/mod_why.txt @@ -1,6 +1,9 @@ env GO111MODULE=on [short] skip +# Populate go.sum. +go mod tidy + go list -test all stdout rsc.io/quote stdout golang.org/x/text/language diff --git a/src/cmd/go/testdata/script/modfile_flag.txt b/src/cmd/go/testdata/script/modfile_flag.txt index f05bf03fbf..0ad0880817 100644 --- a/src/cmd/go/testdata/script/modfile_flag.txt +++ b/src/cmd/go/testdata/script/modfile_flag.txt @@ -37,10 +37,10 @@ go mod why rsc.io/quote # 'go list' and other commands with build flags should work. # They should update the alternate go.mod when a dependency is missing. go mod edit -droprequire rsc.io/quote -go list . +go list -mod=mod . grep rsc.io/quote go.alt.mod -go build -n . -go test -n . +go build -n -mod=mod . +go test -n -mod=mod . go get -d rsc.io/quote diff --git a/src/cmd/go/testdata/script/test_example_goexit.txt b/src/cmd/go/testdata/script/test_example_goexit.txt new file mode 100644 index 0000000000..59219e3366 --- /dev/null +++ b/src/cmd/go/testdata/script/test_example_goexit.txt @@ -0,0 +1,25 @@ +# For issue golang.org/issue/41084 +[short] skip + +! go test -v examplegoexit +stdout '(?s)--- PASS.*--- FAIL.*' +stdout 'panic: test executed panic\(nil\) or runtime\.Goexit' + +-- examplegoexit/example_test.go -- +package main + +import ( + "fmt" + "runtime" +) + +func ExamplePass() { + fmt.Println("pass") + // Output: + // pass +} + +func ExampleGoexit() { + runtime.Goexit() + // Output: +} diff --git a/src/cmd/go/testdata/script/test_exit.txt b/src/cmd/go/testdata/script/test_exit.txt new file mode 100644 index 0000000000..23a2429d1e --- /dev/null +++ b/src/cmd/go/testdata/script/test_exit.txt @@ -0,0 +1,114 @@ +# Builds and runs test binaries, so skip in short mode. +[short] skip + +env GO111MODULE=on + +# If a test invoked by 'go test' exits with a zero status code, +# it will panic. +! go test ./zero +! stdout ^ok +! stdout 'exit status' +stdout 'panic' +stdout ^FAIL + +# If a test exits with a non-zero status code, 'go test' fails normally. +! go test ./one +! stdout ^ok +stdout 'exit status' +! stdout 'panic' +stdout ^FAIL + +# Ensure that other flags still do the right thing. +go test -list=. ./zero +stdout ExitZero + +! go test -bench=. ./zero +stdout 'panic' + +# 'go test' with no args streams output without buffering. Ensure that it still +# catches a zero exit with missing output. +cd zero +! go test +stdout 'panic' +cd ../normal +go test +stdout ^ok +cd .. + +# If a TestMain exits with a zero status code, 'go test' shouldn't +# complain about that. It's a common way to skip testing a package +# entirely. +go test ./main_zero +! stdout 'skipping all tests' +stdout ^ok + +# With -v, we'll see the warning from TestMain. +go test -v ./main_zero +stdout 'skipping all tests' +stdout ^ok + +# Listing all tests won't actually give a result if TestMain exits. That's okay, +# because this is how TestMain works. If we decide to support -list even when +# TestMain is used to skip entire packages, we can change this test case. +go test -list=. ./main_zero +stdout 'skipping all tests' +! stdout TestNotListed + +-- go.mod -- +module m + +-- ./normal/normal.go -- +package normal +-- ./normal/normal_test.go -- +package normal + +import "testing" + +func TestExitZero(t *testing.T) { +} + +-- ./zero/zero.go -- +package zero +-- ./zero/zero_test.go -- +package zero + +import ( + "os" + "testing" +) + +func TestExitZero(t *testing.T) { + os.Exit(0) +} + +-- ./one/one.go -- +package one +-- ./one/one_test.go -- +package one + +import ( + "os" + "testing" +) + +func TestExitOne(t *testing.T) { + os.Exit(1) +} + +-- ./main_zero/zero.go -- +package zero +-- ./main_zero/zero_test.go -- +package zero + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + fmt.Println("skipping all tests") + os.Exit(0) +} + +func TestNotListed(t *testing.T) {} diff --git a/src/cmd/go/testdata/script/testing_issue40908.txt b/src/cmd/go/testdata/script/testing_issue40908.txt new file mode 100644 index 0000000000..4939de080c --- /dev/null +++ b/src/cmd/go/testdata/script/testing_issue40908.txt @@ -0,0 +1,21 @@ +[short] skip +[!race] skip + +go test -race testrace + +-- testrace/race_test.go -- +package testrace + +import "testing" + +func TestRace(t *testing.T) { + helperDone := make(chan struct{}) + go func() { + t.Logf("Something happened before cleanup.") + close(helperDone) + }() + + t.Cleanup(func() { + <-helperDone + }) +} diff --git a/src/cmd/go/testdata/script/version.txt b/src/cmd/go/testdata/script/version.txt index 0123ac6d53..81ca698620 100644 --- a/src/cmd/go/testdata/script/version.txt +++ b/src/cmd/go/testdata/script/version.txt @@ -14,6 +14,7 @@ env GO111MODULE=on [short] skip # Check that 'go version' and 'go version -m' work on a binary built in module mode. +go get -d rsc.io/fortune go build -o fortune.exe rsc.io/fortune go version fortune.exe stdout '^fortune.exe: .+' diff --git a/src/cmd/go/testdata/script/version_replace.txt b/src/cmd/go/testdata/script/version_replace.txt index b657086f09..ec98f4e3f3 100644 --- a/src/cmd/go/testdata/script/version_replace.txt +++ b/src/cmd/go/testdata/script/version_replace.txt @@ -1,7 +1,7 @@ [short] skip go mod download example.com/printversion@v0.1.0 example.com/printversion@v1.0.0 - +go get -d example.com/printversion@v0.1.0 go install example.com/printversion go run example.com/printversion |