aboutsummaryrefslogtreecommitdiff
path: root/src/cmd/go/internal/modfetch
diff options
context:
space:
mode:
Diffstat (limited to 'src/cmd/go/internal/modfetch')
-rw-r--r--src/cmd/go/internal/modfetch/cache.go77
-rw-r--r--src/cmd/go/internal/modfetch/codehost/codehost.go14
-rw-r--r--src/cmd/go/internal/modfetch/codehost/git.go27
-rw-r--r--src/cmd/go/internal/modfetch/codehost/git_test.go6
-rw-r--r--src/cmd/go/internal/modfetch/codehost/shell.go3
-rw-r--r--src/cmd/go/internal/modfetch/codehost/vcs.go5
-rw-r--r--src/cmd/go/internal/modfetch/coderepo.go87
-rw-r--r--src/cmd/go/internal/modfetch/coderepo_test.go30
-rw-r--r--src/cmd/go/internal/modfetch/fetch.go162
-rw-r--r--src/cmd/go/internal/modfetch/proxy.go11
-rw-r--r--src/cmd/go/internal/modfetch/pseudo.go12
-rw-r--r--src/cmd/go/internal/modfetch/repo.go52
-rw-r--r--src/cmd/go/internal/modfetch/sumdb.go11
13 files changed, 291 insertions, 206 deletions
diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go
index e3074b775e..b7aa670250 100644
--- a/src/cmd/go/internal/modfetch/cache.go
+++ b/src/cmd/go/internal/modfetch/cache.go
@@ -10,10 +10,12 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
"strings"
+ "sync"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
@@ -59,7 +61,7 @@ func CachePath(m module.Version, suffix string) (string, error) {
// DownloadDir returns the directory to which m should have been downloaded.
// An error will be returned if the module path or version cannot be escaped.
-// An error satisfying errors.Is(err, os.ErrNotExist) will be returned
+// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned
// along with the directory if the directory does not exist or if the directory
// is not completely populated.
func DownloadDir(m module.Version) (string, error) {
@@ -106,14 +108,14 @@ func DownloadDir(m module.Version) (string, error) {
// DownloadDirPartialError is returned by DownloadDir if a module directory
// exists but was not completely populated.
//
-// DownloadDirPartialError is equivalent to os.ErrNotExist.
+// DownloadDirPartialError is equivalent to fs.ErrNotExist.
type DownloadDirPartialError struct {
Dir string
Err error
}
func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
-func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist }
+func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
// lockVersion locks a file within the module cache that guards the downloading
// and extraction of the zipfile for the given module version.
@@ -155,16 +157,30 @@ func SideLock() (unlock func(), err error) {
type cachingRepo struct {
path string
cache par.Cache // cache for all operations
- r Repo
+
+ once sync.Once
+ initRepo func() (Repo, error)
+ r Repo
}
-func newCachingRepo(r Repo) *cachingRepo {
+func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo {
return &cachingRepo{
- r: r,
- path: r.ModulePath(),
+ path: path,
+ initRepo: initRepo,
}
}
+func (r *cachingRepo) repo() Repo {
+ r.once.Do(func() {
+ var err error
+ r.r, err = r.initRepo()
+ if err != nil {
+ r.r = errRepo{r.path, err}
+ }
+ })
+ return r.r
+}
+
func (r *cachingRepo) ModulePath() string {
return r.path
}
@@ -175,7 +191,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) {
err error
}
c := r.cache.Do("versions:"+prefix, func() interface{} {
- list, err := r.r.Versions(prefix)
+ list, err := r.repo().Versions(prefix)
return cached{list, err}
}).(cached)
@@ -197,7 +213,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
return cachedInfo{info, nil}
}
- info, err = r.r.Stat(rev)
+ info, err = r.repo().Stat(rev)
if err == nil {
// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
// then save the information under the proper version, for future use.
@@ -224,7 +240,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
func (r *cachingRepo) Latest() (*RevInfo, error) {
c := r.cache.Do("latest:", func() interface{} {
- info, err := r.r.Latest()
+ info, err := r.repo().Latest()
// Save info for likely future Stat call.
if err == nil {
@@ -258,7 +274,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) {
return cached{text, nil}
}
- text, err = r.r.GoMod(version)
+ text, err = r.repo().GoMod(version)
if err == nil {
if err := checkGoMod(r.path, version, text); err != nil {
return cached{text, err}
@@ -277,26 +293,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) {
}
func (r *cachingRepo) Zip(dst io.Writer, version string) error {
- return r.r.Zip(dst, version)
-}
-
-// Stat is like Lookup(path).Stat(rev) but avoids the
-// repository path resolution in Lookup if the result is
-// already cached on local disk.
-func Stat(proxy, path, rev string) (*RevInfo, error) {
- _, info, err := readDiskStat(path, rev)
- if err == nil {
- return info, nil
- }
- repo, err := Lookup(proxy, path)
- if err != nil {
- return nil, err
- }
- return repo.Stat(rev)
+ return r.repo().Zip(dst, version)
}
-// InfoFile is like Stat but returns the name of the file containing
-// the cached information.
+// InfoFile is like Lookup(path).Stat(version) but returns the name of the file
+// containing the cached information.
func InfoFile(path, version string) (string, error) {
if !semver.IsValid(version) {
return "", fmt.Errorf("invalid version %q", version)
@@ -307,10 +308,7 @@ func InfoFile(path, version string) (string, error) {
}
err := TryProxies(func(proxy string) error {
- repo, err := Lookup(proxy, path)
- if err == nil {
- _, err = repo.Stat(version)
- }
+ _, err := Lookup(proxy, path).Stat(version)
return err
})
if err != nil {
@@ -336,11 +334,7 @@ func GoMod(path, rev string) ([]byte, error) {
rev = info.Version
} else {
err := TryProxies(func(proxy string) error {
- repo, err := Lookup(proxy, path)
- if err != nil {
- return err
- }
- info, err := repo.Stat(rev)
+ info, err := Lookup(proxy, path).Stat(rev)
if err == nil {
rev = info.Version
}
@@ -357,11 +351,8 @@ func GoMod(path, rev string) ([]byte, error) {
return data, nil
}
- err = TryProxies(func(proxy string) error {
- repo, err := Lookup(proxy, path)
- if err == nil {
- data, err = repo.GoMod(rev)
- }
+ err = TryProxies(func(proxy string) (err error) {
+ data, err = Lookup(proxy, path).GoMod(rev)
return err
})
return data, err
diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go
index d85eddf767..c5fbb31b2b 100644
--- a/src/cmd/go/internal/modfetch/codehost/codehost.go
+++ b/src/cmd/go/internal/modfetch/codehost/codehost.go
@@ -11,6 +11,7 @@ import (
"crypto/sha256"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"os/exec"
@@ -79,9 +80,8 @@ type Repo interface {
ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
// RecentTag returns the most recent tag on rev or one of its predecessors
- // with the given prefix and major version.
- // An empty major string matches any major version.
- RecentTag(rev, prefix, major string) (tag string, err error)
+ // with the given prefix. allowed may be used to filter out unwanted versions.
+ RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error)
// DescendsFrom reports whether rev or any of its ancestors has the given tag.
//
@@ -106,7 +106,7 @@ type FileRev struct {
Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
}
-// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a
+// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a
// revision rather than a file.
type UnknownRevisionError struct {
Rev string
@@ -116,10 +116,10 @@ func (e *UnknownRevisionError) Error() string {
return "unknown revision " + e.Rev
}
func (UnknownRevisionError) Is(err error) bool {
- return err == os.ErrNotExist
+ return err == fs.ErrNotExist
}
-// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given
+// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given
// repository or module contains no commits.
var ErrNoCommits error = noCommitsError{}
@@ -129,7 +129,7 @@ func (noCommitsError) Error() string {
return "no commits"
}
func (noCommitsError) Is(err error) bool {
- return err == os.ErrNotExist
+ return err == fs.ErrNotExist
}
// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index 31921324a7..8abc039e7f 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -9,7 +9,7 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
+ "io/fs"
"net/url"
"os"
"os/exec"
@@ -34,13 +34,13 @@ func LocalGitRepo(remote string) (Repo, error) {
}
// A notExistError wraps another error to retain its original text
-// but makes it opaquely equivalent to os.ErrNotExist.
+// but makes it opaquely equivalent to fs.ErrNotExist.
type notExistError struct {
err error
}
func (e notExistError) Error() string { return e.err.Error() }
-func (notExistError) Is(err error) bool { return err == os.ErrNotExist }
+func (notExistError) Is(err error) bool { return err == fs.ErrNotExist }
const gitWorkDirType = "git3"
@@ -188,7 +188,7 @@ func (r *gitRepo) loadRefs() {
// For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL
// ourselves and see what code it serves.
if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
- if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) {
+ if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) {
gitErr = notExistError{gitErr}
}
}
@@ -505,7 +505,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
}
out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file)
if err != nil {
- return nil, os.ErrNotExist
+ return nil, fs.ErrNotExist
}
return out, nil
}
@@ -629,9 +629,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F
case "tag", "commit":
switch fileType {
default:
- f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
+ f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
case "missing":
- f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist}
+ f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist}
case "blob":
f.Data = fileData
}
@@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F
return missing, nil
}
-func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
+func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
info, err := r.Stat(rev)
if err != nil {
return "", err
@@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
// NOTE: Do not replace the call to semver.Compare with semver.Max.
// We want to return the actual tag, not a canonicalized version of it,
// and semver.Max currently canonicalizes (see golang.org/issue/32700).
- if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 {
+ if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) {
+ continue
+ }
+ if semver.Compare(semtag, highest) > 0 {
highest = semtag
}
}
@@ -823,12 +826,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args)
if err != nil {
if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) {
- return nil, os.ErrNotExist
+ return nil, fs.ErrNotExist
}
return nil, err
}
- return ioutil.NopCloser(bytes.NewReader(archive)), nil
+ return io.NopCloser(bytes.NewReader(archive)), nil
}
// ensureGitAttributes makes sure export-subst and export-ignore features are
@@ -859,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) {
}
}()
- b, err := ioutil.ReadAll(f)
+ b, err := io.ReadAll(f)
if err != nil {
return err
}
diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go
index ba27c70f5a..981e3fe91f 100644
--- a/src/cmd/go/internal/modfetch/codehost/git_test.go
+++ b/src/cmd/go/internal/modfetch/codehost/git_test.go
@@ -10,6 +10,8 @@ import (
"flag"
"fmt"
"internal/testenv"
+ "io"
+ "io/fs"
"io/ioutil"
"log"
"os"
@@ -210,7 +212,7 @@ var readFileTests = []struct {
repo: gitrepo1,
rev: "v2.3.4",
file: "another.txt",
- err: os.ErrNotExist.Error(),
+ err: fs.ErrNotExist.Error(),
},
}
@@ -432,7 +434,7 @@ func TestReadZip(t *testing.T) {
if tt.err != "" {
t.Fatalf("ReadZip: no error, wanted %v", tt.err)
}
- zipdata, err := ioutil.ReadAll(rc)
+ zipdata, err := io.ReadAll(rc)
if err != nil {
t.Fatal(err)
}
diff --git a/src/cmd/go/internal/modfetch/codehost/shell.go b/src/cmd/go/internal/modfetch/codehost/shell.go
index 2762c55720..b9525adf5e 100644
--- a/src/cmd/go/internal/modfetch/codehost/shell.go
+++ b/src/cmd/go/internal/modfetch/codehost/shell.go
@@ -14,6 +14,7 @@ import (
"bytes"
"flag"
"fmt"
+ "io"
"io/ioutil"
"log"
"os"
@@ -115,7 +116,7 @@ func main() {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
- data, err := ioutil.ReadAll(rc)
+ data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go
index 7284557f4b..ec97fc7e1b 100644
--- a/src/cmd/go/internal/modfetch/codehost/vcs.go
+++ b/src/cmd/go/internal/modfetch/codehost/vcs.go
@@ -9,6 +9,7 @@ import (
"fmt"
"internal/lazyregexp"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -377,7 +378,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
if err != nil {
- return nil, os.ErrNotExist
+ return nil, fs.ErrNotExist
}
return out, nil
}
@@ -395,7 +396,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s
return nil, vcsErrorf("ReadFileRevs not implemented")
}
-func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
+func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
// We don't technically need to lock here since we're returning an error
// uncondititonally, but doing so anyway will help to avoid baking in
// lock-inversion bugs.
diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index d043903336..b6bcf83f1a 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path"
@@ -419,9 +420,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
tagPrefix = r.codeDir + "/"
}
+ isRetracted, err := r.retractedVersions()
+ if err != nil {
+ isRetracted = func(string) bool { return false }
+ }
+
// tagToVersion returns the version obtained by trimming tagPrefix from tag.
- // If the tag is invalid or a pseudo-version, tagToVersion returns an empty
- // version.
+ // If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns
+ // an empty version.
tagToVersion := func(tag string) (v string, tagIsCanonical bool) {
if !strings.HasPrefix(tag, tagPrefix) {
return "", false
@@ -436,6 +442,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
if v == "" || !strings.HasPrefix(trimmed, v) {
return "", false // Invalid or incomplete version (just vX or vX.Y).
}
+ if isRetracted(v) {
+ return "", false
+ }
if v == trimmed {
tagIsCanonical = true
}
@@ -500,15 +509,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
return checkGoMod()
}
+ // Find the highest tagged version in the revision's history, subject to
+ // major version and +incompatible constraints. Use that version as the
+ // pseudo-version base so that the pseudo-version sorts higher. Ignore
+ // retracted versions.
+ allowedMajor := func(major string) func(v string) bool {
+ return func(v string) bool {
+ return (major == "" || semver.Major(v) == major) && !isRetracted(v)
+ }
+ }
if pseudoBase == "" {
var tag string
if r.pseudoMajor != "" || canUseIncompatible() {
- tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor)
+ tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor))
} else {
// Allow either v1 or v0, but not incompatible higher versions.
- tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1")
+ tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1"))
if tag == "" {
- tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0")
+ tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0"))
}
}
pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid
@@ -869,6 +887,57 @@ func (r *codeRepo) modPrefix(rev string) string {
return r.modPath + "@" + rev
}
+func (r *codeRepo) retractedVersions() (func(string) bool, error) {
+ versions, err := r.Versions("")
+ if err != nil {
+ return nil, err
+ }
+
+ for i, v := range versions {
+ if strings.HasSuffix(v, "+incompatible") {
+ versions = versions[:i]
+ break
+ }
+ }
+ if len(versions) == 0 {
+ return func(string) bool { return false }, nil
+ }
+
+ var highest string
+ for i := len(versions) - 1; i >= 0; i-- {
+ v := versions[i]
+ if semver.Prerelease(v) == "" {
+ highest = v
+ break
+ }
+ }
+ if highest == "" {
+ highest = versions[len(versions)-1]
+ }
+
+ data, err := r.GoMod(highest)
+ if err != nil {
+ return nil, err
+ }
+ f, err := modfile.ParseLax("go.mod", data, nil)
+ if err != nil {
+ return nil, err
+ }
+ retractions := make([]modfile.VersionInterval, len(f.Retract))
+ for _, r := range f.Retract {
+ retractions = append(retractions, r.VersionInterval)
+ }
+
+ return func(v string) bool {
+ for _, r := range retractions {
+ if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 {
+ return true
+ }
+ }
+ return false
+ }, nil
+}
+
func (r *codeRepo) Zip(dst io.Writer, version string) error {
if version != module.CanonicalVersion(version) {
return fmt.Errorf("version %s is not canonical", version)
@@ -972,7 +1041,7 @@ type zipFile struct {
}
func (f zipFile) Path() string { return f.name }
-func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
+func (f zipFile) Lstat() (fs.FileInfo, error) { return f.f.FileInfo(), nil }
func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
type dataFile struct {
@@ -981,9 +1050,9 @@ type dataFile struct {
}
func (f dataFile) Path() string { return f.name }
-func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil }
+func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil }
func (f dataFile) Open() (io.ReadCloser, error) {
- return ioutil.NopCloser(bytes.NewReader(f.data)), nil
+ return io.NopCloser(bytes.NewReader(f.data)), nil
}
type dataFileInfo struct {
@@ -992,7 +1061,7 @@ type dataFileInfo struct {
func (fi dataFileInfo) Name() string { return path.Base(fi.f.name) }
func (fi dataFileInfo) Size() int64 { return int64(len(fi.f.data)) }
-func (fi dataFileInfo) Mode() os.FileMode { return 0644 }
+func (fi dataFileInfo) Mode() fs.FileMode { return 0644 }
func (fi dataFileInfo) ModTime() time.Time { return time.Time{} }
func (fi dataFileInfo) IsDir() bool { return false }
func (fi dataFileInfo) Sys() interface{} { return nil }
diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go
index f69c193b86..4364fef6d1 100644
--- a/src/cmd/go/internal/modfetch/coderepo_test.go
+++ b/src/cmd/go/internal/modfetch/coderepo_test.go
@@ -60,7 +60,6 @@ var altVgotests = map[string]string{
type codeRepoTest struct {
vcs string
path string
- lookErr string
mpath string
rev string
err string
@@ -332,9 +331,9 @@ var codeRepoTests = []codeRepoTest{
// package in subdirectory - custom domain
// In general we can't reject these definitively in Lookup,
// but gopkg.in is special.
- vcs: "git",
- path: "gopkg.in/yaml.v2/abc",
- lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"",
+ vcs: "git",
+ path: "gopkg.in/yaml.v2/abc",
+ err: "invalid module path \"gopkg.in/yaml.v2/abc\"",
},
{
// package in subdirectory - github
@@ -440,16 +439,7 @@ func TestCodeRepo(t *testing.T) {
testenv.MustHaveExecPath(t, tt.vcs)
}
- repo, err := Lookup("direct", tt.path)
- if tt.lookErr != "" {
- if err != nil && err.Error() == tt.lookErr {
- return
- }
- t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr)
- }
- if err != nil {
- t.Fatalf("Lookup(%q): %v", tt.path, err)
- }
+ repo := Lookup("direct", tt.path)
if tt.mpath == "" {
tt.mpath = tt.path
@@ -658,7 +648,7 @@ var codeRepoVersionsTests = []struct {
{
vcs: "git",
path: "gopkg.in/russross/blackfriday.v2",
- versions: []string{"v2.0.0", "v2.0.1"},
+ versions: []string{"v2.0.0", "v2.0.1", "v2.1.0-pre.1"},
},
{
vcs: "git",
@@ -685,10 +675,7 @@ func TestCodeRepoVersions(t *testing.T) {
testenv.MustHaveExecPath(t, tt.vcs)
}
- repo, err := Lookup("direct", tt.path)
- if err != nil {
- t.Fatalf("Lookup(%q): %v", tt.path, err)
- }
+ repo := Lookup("direct", tt.path)
list, err := repo.Versions(tt.prefix)
if err != nil {
t.Fatalf("Versions(%q): %v", tt.prefix, err)
@@ -763,10 +750,7 @@ func TestLatest(t *testing.T) {
testenv.MustHaveExecPath(t, tt.vcs)
}
- repo, err := Lookup("direct", tt.path)
- if err != nil {
- t.Fatalf("Lookup(%q): %v", tt.path, err)
- }
+ repo := Lookup("direct", tt.path)
info, err := repo.Latest()
if err != nil {
if tt.err != "" {
diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go
index 01d8f007ac..25e9fb62c1 100644
--- a/src/cmd/go/internal/modfetch/fetch.go
+++ b/src/cmd/go/internal/modfetch/fetch.go
@@ -11,6 +11,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -63,14 +64,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
defer span.Done()
- // If the directory exists, and no .partial file exists, the module has
- // already been completely extracted. .partial files may be created when a
- // module zip directory is extracted in place instead of being extracted to a
- // temporary directory and renamed.
dir, err = DownloadDir(mod)
if err == nil {
+ // The directory has already been completely extracted (no .partial file exists).
return dir, nil
- } else if dir == "" || !errors.Is(err, os.ErrNotExist) {
+ } else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
return "", err
}
@@ -88,6 +86,9 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
}
defer unlock()
+ ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
+ defer span.Done()
+
// Check whether the directory was populated while we were waiting on the lock.
_, dirErr := DownloadDir(mod)
if dirErr == nil {
@@ -95,10 +96,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
}
_, dirExists := dirErr.(*DownloadDirPartialError)
- // Clean up any remaining temporary directories from previous runs, as well
- // as partially extracted diectories created by future versions of cmd/go.
- // This is only safe to do because the lock file ensures that their writers
- // are no longer active.
+ // Clean up any remaining temporary directories created by old versions
+ // (before 1.16), as well as partially extracted directories (indicated by
+ // DownloadDirPartialError, usually because of a .partial file). This is only
+ // safe to do because the lock file ensures that their writers are no longer
+ // active.
parentDir := filepath.Dir(dir)
tmpPrefix := filepath.Base(dir) + ".tmp-"
if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
@@ -116,88 +118,44 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
if err != nil {
return "", err
}
- if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) {
- return "", err
- }
- // Extract the module zip directory.
+ // Extract the module zip directory at its final location.
//
- // By default, we extract to a temporary directory, then atomically rename to
- // its final location. We use the existence of the source directory to signal
- // that it has been extracted successfully (see DownloadDir). If someone
- // deletes the entire directory (e.g., as an attempt to prune out file
- // corruption), the module cache will still be left in a recoverable
- // state.
+ // To prevent other processes from reading the directory if we crash,
+ // create a .partial file before extracting the directory, and delete
+ // the .partial file afterward (all while holding the lock).
//
- // Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if
- // another process opens files in the temporary directory. This is partially
- // mitigated by using robustio.Rename, which retries os.Rename for a short
- // time.
+ // Before Go 1.16, we extracted to a temporary directory with a random name
+ // then renamed it into place with os.Rename. On Windows, this failed with
+ // ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
+ // opened files in the temporary directory.
//
- // To avoid this error completely, if unzipInPlace is set, we instead create a
- // .partial file (indicating the directory isn't fully extracted), then we
- // extract the directory at its final location, then we delete the .partial
- // file. This is not the default behavior because older versions of Go may
- // simply stat the directory to check whether it exists without looking for a
- // .partial file. If multiple versions run concurrently, the older version may
- // assume a partially extracted directory is complete.
- // TODO(golang.org/issue/36568): when these older versions are no longer
- // supported, remove the old default behavior and the unzipInPlace flag.
+ // Go 1.14.2 and higher respect .partial files. Older versions may use
+ // partially extracted directories. 'go mod verify' can detect this,
+ // and 'go clean -modcache' can fix it.
if err := os.MkdirAll(parentDir, 0777); err != nil {
return "", err
}
-
- ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
- if unzipInPlace {
- if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil {
- return "", err
- }
- if err := modzip.Unzip(dir, mod, zipfile); err != nil {
- fmt.Fprintf(os.Stderr, "-> %s\n", err)
- if rmErr := RemoveAll(dir); rmErr == nil {
- os.Remove(partialPath)
- }
- return "", err
- }
- if err := os.Remove(partialPath); err != nil {
- return "", err
- }
- } else {
- tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
- if err != nil {
- return "", err
- }
- if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
- fmt.Fprintf(os.Stderr, "-> %s\n", err)
- RemoveAll(tmpDir)
- return "", err
- }
- if err := robustio.Rename(tmpDir, dir); err != nil {
- RemoveAll(tmpDir)
- return "", err
+ if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil {
+ return "", err
+ }
+ if err := modzip.Unzip(dir, mod, zipfile); err != nil {
+ fmt.Fprintf(os.Stderr, "-> %s\n", err)
+ if rmErr := RemoveAll(dir); rmErr == nil {
+ os.Remove(partialPath)
}
+ return "", err
+ }
+ if err := os.Remove(partialPath); err != nil {
+ return "", err
}
- defer span.Done()
if !cfg.ModCacheRW {
- // Make dir read-only only *after* renaming it.
- // os.Rename was observed to fail for read-only directories on macOS.
makeDirsReadOnly(dir)
}
return dir, nil
}
-var unzipInPlace bool
-
-func init() {
- for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") {
- if f == "modcacheunzipinplace=1" {
- unzipInPlace = true
- break
- }
- }
-}
-
var downloadZipCache par.Cache
// DownloadZip downloads the specific module version to the
@@ -276,12 +234,28 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
}
}()
+ var unrecoverableErr error
err = TryProxies(func(proxy string) error {
- repo, err := Lookup(proxy, mod.Path)
+ if unrecoverableErr != nil {
+ return unrecoverableErr
+ }
+ repo := Lookup(proxy, mod.Path)
+ err := repo.Zip(f, mod.Version)
if err != nil {
- return err
+ // Zip may have partially written to f before failing.
+ // (Perhaps the server crashed while sending the file?)
+ // Since we allow fallback on error in some cases, we need to fix up the
+ // file to be empty again for the next attempt.
+ if _, err := f.Seek(0, io.SeekStart); err != nil {
+ unrecoverableErr = err
+ return err
+ }
+ if err := f.Truncate(0); err != nil {
+ unrecoverableErr = err
+ return err
+ }
}
- return repo.Zip(f, mod.Version)
+ return err
})
if err != nil {
return err
@@ -341,10 +315,10 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
func makeDirsReadOnly(dir string) {
type pathMode struct {
path string
- mode os.FileMode
+ mode fs.FileMode
}
var dirs []pathMode // in lexical order
- filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
if err == nil && info.Mode()&0222 != 0 {
if info.IsDir() {
dirs = append(dirs, pathMode{path, info.Mode()})
@@ -363,7 +337,7 @@ func makeDirsReadOnly(dir string) {
// any permission changes needed to do so.
func RemoveAll(dir string) error {
// Module cache has 0555 directories; make them writable in order to remove content.
- filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return nil // ignore errors walking in file system
}
@@ -454,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error
return nil
}
+// HaveSum returns true if the go.sum file contains an entry for mod.
+// The entry's hash must be generated with a known hash algorithm.
+// mod.Version may have a "/go.mod" suffix to distinguish sums for
+// .mod and .zip files.
+func HaveSum(mod module.Version) bool {
+ goSum.mu.Lock()
+ defer goSum.mu.Unlock()
+ inited, err := initGoSum()
+ if err != nil || !inited {
+ return false
+ }
+ for _, h := range goSum.m[mod] {
+ if !strings.HasPrefix(h, "h1:") {
+ continue
+ }
+ if !goSum.status[modSum{mod, h}].dirty {
+ return true
+ }
+ }
+ return false
+}
+
// checkMod checks the given module's checksum.
func checkMod(mod module.Version) {
if cfg.GOMODCACHE == "" {
@@ -468,7 +464,7 @@ func checkMod(mod module.Version) {
}
data, err := renameio.ReadFile(ziphash)
if err != nil {
- if errors.Is(err, os.ErrNotExist) {
+ if errors.Is(err, fs.ErrNotExist) {
// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
return
}
@@ -487,7 +483,7 @@ func checkMod(mod module.Version) {
// goModSum returns the checksum for the go.mod contents.
func goModSum(data []byte) (string, error) {
return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
- return ioutil.NopCloser(bytes.NewReader(data)), nil
+ return io.NopCloser(bytes.NewReader(data)), nil
})
}
diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go
index 4ac26650a9..d75b4da521 100644
--- a/src/cmd/go/internal/modfetch/proxy.go
+++ b/src/cmd/go/internal/modfetch/proxy.go
@@ -9,9 +9,8 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
+ "io/fs"
"net/url"
- "os"
"path"
pathpkg "path"
"path/filepath"
@@ -186,7 +185,7 @@ func proxyList() ([]proxySpec, error) {
// TryProxies iterates f over each configured proxy (including "noproxy" and
// "direct" if applicable) until f returns no error or until f returns an
-// error that is not equivalent to os.ErrNotExist on a proxy configured
+// error that is not equivalent to fs.ErrNotExist on a proxy configured
// not to fall back on errors.
//
// TryProxies then returns that final error.
@@ -222,7 +221,7 @@ func TryProxies(f func(proxy string) error) error {
if err == nil {
return nil
}
- isNotExistErr := errors.Is(err, os.ErrNotExist)
+ isNotExistErr := errors.Is(err, fs.ErrNotExist)
if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) {
bestErr = err
@@ -305,7 +304,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) {
return nil, err
}
defer body.Close()
- return ioutil.ReadAll(body)
+ return io.ReadAll(body)
}
func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
@@ -428,7 +427,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
func (p *proxyRepo) Latest() (*RevInfo, error) {
data, err := p.getBytes("@latest")
if err != nil {
- if !errors.Is(err, os.ErrNotExist) {
+ if !errors.Is(err, fs.ErrNotExist) {
return nil, p.versionError("", err)
}
return p.latest()
diff --git a/src/cmd/go/internal/modfetch/pseudo.go b/src/cmd/go/internal/modfetch/pseudo.go
index 20c0b060ab..93eb0fad96 100644
--- a/src/cmd/go/internal/modfetch/pseudo.go
+++ b/src/cmd/go/internal/modfetch/pseudo.go
@@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string {
return v + incDecimal(patch) + "-0." + segment + build
}
+// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and
+// revision, which may be used as a placeholder.
+func ZeroPseudoVersion(major string) string {
+ return PseudoVersion(major, "", time.Time{}, "000000000000")
+}
+
// incDecimal returns the decimal string incremented by 1.
func incDecimal(decimal string) string {
// Scan right to left turning 9s to 0s until you find a digit to increment.
@@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool {
return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
}
+// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base,
+// timestamp, and revision, as returned by ZeroPseudoVersion.
+func IsZeroPseudoVersion(v string) bool {
+ return v == ZeroPseudoVersion(semver.Major(v))
+}
+
// PseudoVersionTime returns the time stamp of the pseudo-version v.
// It returns an error if v is not a pseudo-version or if the time stamp
// embedded in the pseudo-version is not a valid time.
diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go
index eed4dd4258..af9e24cefd 100644
--- a/src/cmd/go/internal/modfetch/repo.go
+++ b/src/cmd/go/internal/modfetch/repo.go
@@ -7,6 +7,7 @@ package modfetch
import (
"fmt"
"io"
+ "io/fs"
"os"
"sort"
"strconv"
@@ -32,8 +33,17 @@ type Repo interface {
// Versions lists all known versions with the given prefix.
// Pseudo-versions are not included.
+ //
// Versions should be returned sorted in semver order
// (implementations can use SortVersions).
+ //
+ // Versions returns a non-nil error only if there was a problem
+ // fetching the list of versions: it may return an empty list
+ // along with a nil error if the list of matching versions
+ // is known to be empty.
+ //
+ // If the underlying repository does not exist,
+ // Versions returns an error matching errors.Is(_, os.NotExist).
Versions(prefix string) ([]string, error)
// Stat returns information about the revision rev.
@@ -188,27 +198,26 @@ type lookupCacheKey struct {
//
// A successful return does not guarantee that the module
// has any defined versions.
-func Lookup(proxy, path string) (Repo, error) {
+func Lookup(proxy, path string) Repo {
if traceRepo {
defer logCall("Lookup(%q, %q)", proxy, path)()
}
type cached struct {
- r Repo
- err error
+ r Repo
}
c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} {
- r, err := lookup(proxy, path)
- if err == nil {
- if traceRepo {
+ r := newCachingRepo(path, func() (Repo, error) {
+ r, err := lookup(proxy, path)
+ if err == nil && traceRepo {
r = newLoggingRepo(r)
}
- r = newCachingRepo(r)
- }
- return cached{r, err}
+ return r, err
+ })
+ return cached{r}
}).(cached)
- return c.r, c.err
+ return c.r
}
// lookup returns the module with the given module path.
@@ -228,7 +237,7 @@ func lookup(proxy, path string) (r Repo, err error) {
switch proxy {
case "off":
- return nil, errProxyOff
+ return errRepo{path, errProxyOff}, nil
case "direct":
return lookupDirect(path)
case "noproxy":
@@ -407,7 +416,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error {
return l.r.Zip(dst, version)
}
-// A notExistError is like os.ErrNotExist, but with a custom message
+// errRepo is a Repo that returns the same error for all operations.
+//
+// It is useful in conjunction with caching, since cache hits will not attempt
+// the prohibited operations.
+type errRepo struct {
+ modulePath string
+ err error
+}
+
+func (r errRepo) ModulePath() string { return r.modulePath }
+
+func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err }
+func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err }
+func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err }
+func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err }
+func (r errRepo) Zip(dst io.Writer, version string) error { return r.err }
+
+// A notExistError is like fs.ErrNotExist, but with a custom message
type notExistError struct {
err error
}
@@ -421,7 +447,7 @@ func (e notExistError) Error() string {
}
func (notExistError) Is(target error) bool {
- return target == os.ErrNotExist
+ return target == fs.ErrNotExist
}
func (e notExistError) Unwrap() error {
diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go
index 47a2571531..4fbc54d15c 100644
--- a/src/cmd/go/internal/modfetch/sumdb.go
+++ b/src/cmd/go/internal/modfetch/sumdb.go
@@ -12,7 +12,8 @@ import (
"bytes"
"errors"
"fmt"
- "io/ioutil"
+ "io"
+ "io/fs"
"net/url"
"os"
"path/filepath"
@@ -182,7 +183,7 @@ func (c *dbClient) initBase() {
return nil
}
})
- if errors.Is(err, os.ErrNotExist) {
+ if errors.Is(err, fs.ErrNotExist) {
// No proxies, or all proxies failed (with 404, 410, or were were allowed
// to fall back), or we reached an explicit "direct" or "off".
c.base = c.direct
@@ -203,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
}
targ := filepath.Join(cfg.SumdbDir, file)
data, err = lockedfile.Read(targ)
- if errors.Is(err, os.ErrNotExist) {
+ if errors.Is(err, fs.ErrNotExist) {
// Treat non-existent as empty, to bootstrap the "latest" file
// the first time we connect to a given database.
return []byte{}, nil
@@ -227,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error {
return err
}
defer f.Close()
- data, err := ioutil.ReadAll(f)
+ data, err := io.ReadAll(f)
if err != nil {
return err
}
@@ -257,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) {
// during which the empty file can be locked for reading.
// Treat observing an empty file as file not found.
if err == nil && len(data) == 0 {
- err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist}
+ err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist}
}
return data, err
}
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion