cmd/go: merge module support from x/vgo repo

This CL corresponds to CL 123361, the final manual CL in that repo,
making this the final manual sync.

All future commits will happen in this repo (the main Go repo),
and we'll update x/vgo automatically with a fixed patch+script.

Change-Id: I572243309c1809727604fd704705a23c30e85d1a
Reviewed-on: https://go-review.googlesource.com/123576
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

1 files changed, 175 insertions, 133 deletions

diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index bb6e8ac179..33be117de9 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -6,20 +6,16 @@ package modfetch
 
 import (
 	"archive/zip"
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"regexp"
-	"strconv"
 	"strings"
 	"time"
 
-	"cmd/go/internal/modconv"
 	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
@@ -28,7 +24,8 @@ import (
 
 // A codeRepo implements modfetch.Repo using an underlying codehost.Repo.
 type codeRepo struct {
-	modPath  string
+	modPath string
+
 	code     codehost.Repo
 	codeRoot string
 	codeDir  string
@@ -39,10 +36,9 @@ type codeRepo struct {
 	pseudoMajor string
 }
 
-func newCodeRepo(code codehost.Repo, path string) (Repo, error) {
-	codeRoot := code.Root()
-	if !hasPathPrefix(path, codeRoot) {
-		return nil, fmt.Errorf("mismatched repo: found %s for %s", codeRoot, path)
+func newCodeRepo(code codehost.Repo, root, path string) (Repo, error) {
+	if !hasPathPrefix(path, root) {
+		return nil, fmt.Errorf("mismatched repo: found %s for %s", root, path)
 	}
 	pathPrefix, pathMajor, ok := module.SplitPathVersion(path)
 	if !ok {
@@ -62,7 +58,7 @@ func newCodeRepo(code codehost.Repo, path string) (Repo, error) {
 	//
 	// Compute codeDir = bar, the subdirectory within the repo
 	// corresponding to the module root.
-	codeDir := strings.Trim(strings.TrimPrefix(pathPrefix, codeRoot), "/")
+	codeDir := strings.Trim(strings.TrimPrefix(pathPrefix, root), "/")
 	if strings.HasPrefix(path, "gopkg.in/") {
 		// But gopkg.in is a special legacy case, in which pathPrefix does not start with codeRoot.
 		// For example we might have:
@@ -78,7 +74,7 @@ func newCodeRepo(code codehost.Repo, path string) (Repo, error) {
 	r := &codeRepo{
 		modPath:     path,
 		code:        code,
-		codeRoot:    codeRoot,
+		codeRoot:    root,
 		codeDir:     codeDir,
 		pathPrefix:  pathPrefix,
 		pathMajor:   pathMajor,
@@ -110,7 +106,7 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
 		if r.codeDir != "" {
 			v = v[len(r.codeDir)+1:]
 		}
-		if !semver.IsValid(v) || v != semver.Canonical(v) || isPseudoVersion(v) || !module.MatchPathMajor(v, r.pathMajor) {
+		if !semver.IsValid(v) || v != semver.Canonical(v) || IsPseudoVersion(v) || !module.MatchPathMajor(v, r.pathMajor) {
 			continue
 		}
 		list = append(list, v)
@@ -131,7 +127,7 @@ func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return r.convert(info)
+	return r.convert(info, rev)
 }
 
 func (r *codeRepo) Latest() (*RevInfo, error) {
@@ -139,39 +135,70 @@ func (r *codeRepo) Latest() (*RevInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return r.convert(info)
+	return r.convert(info, "")
 }
 
-func (r *codeRepo) convert(info *codehost.RevInfo) (*RevInfo, error) {
-	versionOK := func(v string) bool {
-		return semver.IsValid(v) && v == semver.Canonical(v) && !isPseudoVersion(v) && module.MatchPathMajor(v, r.pathMajor)
+func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, error) {
+	info2 := &RevInfo{
+		Name:  info.Name,
+		Short: info.Short,
+		Time:  info.Time,
 	}
-	v := info.Version
-	if r.codeDir == "" {
-		if !versionOK(v) {
-			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
-		}
+
+	// Determine version.
+	if semver.IsValid(statVers) && statVers == semver.Canonical(statVers) && module.MatchPathMajor(statVers, r.pathMajor) {
+		// The original call was repo.Stat(statVers), and requestedVersion is OK, so use it.
+		info2.Version = statVers
 	} else {
-		p := r.codeDir + "/"
-		if strings.HasPrefix(v, p) && versionOK(v[len(p):]) {
+		// Otherwise derive a version from a code repo tag.
+		// Tag must have a prefix matching codeDir.
+		p := ""
+		if r.codeDir != "" {
+			p = r.codeDir + "/"
+		}
+
+		tagOK := func(v string) bool {
+			if !strings.HasPrefix(v, p) {
+				return false
+			}
 			v = v[len(p):]
+			return semver.IsValid(v) && v == semver.Canonical(v) && module.MatchPathMajor(v, r.pathMajor) && !IsPseudoVersion(v)
+		}
+
+		// If info.Version is OK, use it.
+		if tagOK(info.Version) {
+			info2.Version = info.Version[len(p):]
 		} else {
-			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
+			// Otherwise look through all known tags for latest in semver ordering.
+			for _, tag := range info.Tags {
+				if tagOK(tag) && semver.Compare(info2.Version, tag[len(p):]) < 0 {
+					info2.Version = tag[len(p):]
+				}
+			}
+			// Otherwise make a pseudo-version.
+			if info2.Version == "" {
+				info2.Version = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
+			}
 		}
 	}
 
-	info2 := &RevInfo{
-		Name:    info.Name,
-		Short:   info.Short,
-		Time:    info.Time,
-		Version: v,
+	// Do not allow a successful stat of a pseudo-version for a subdirectory
+	// unless the subdirectory actually does have a go.mod.
+	if IsPseudoVersion(info2.Version) && r.codeDir != "" {
+		_, _, _, err := r.findDir(info2.Version)
+		if err != nil {
+			// TODO: It would be nice to return an error like "not a module".
+			// Right now we return "missing go.mod", which is a little confusing.
+			return nil, err
+		}
 	}
+
 	return info2, nil
 }
 
 func (r *codeRepo) revToRev(rev string) string {
 	if semver.IsValid(rev) {
-		if isPseudoVersion(rev) {
+		if IsPseudoVersion(rev) {
 			i := strings.Index(rev, "-")
 			j := strings.Index(rev[i+1:], "-")
 			return rev[i+1+j+1:]
@@ -196,82 +223,100 @@ func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err e
 	if err != nil {
 		return "", "", nil, err
 	}
-	if r.pathMajor == "" || strings.HasPrefix(r.pathMajor, ".") {
-		if r.codeDir == "" {
-			return rev, "", nil, nil
-		}
-		file1 := path.Join(r.codeDir, "go.mod")
-		gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
-		if err1 != nil {
-			return "", "", nil, fmt.Errorf("missing go.mod")
-		}
-		return rev, r.codeDir, gomod1, nil
-	}
 
-	// Suppose pathMajor is "/v2".
-	// Either go.mod should claim v2 and v2/go.mod should not exist,
-	// or v2/go.mod should exist and claim v2. Not both.
-	// Note that we don't check the full path, just the major suffix,
-	// because of replacement modules. This might be a fork of
-	// the real module, found at a different path, usable only in
-	// a replace directive.
+	// Load info about go.mod but delay consideration
+	// (except I/O error) until we rule out v2/go.mod.
 	file1 := path.Join(r.codeDir, "go.mod")
-	file2 := path.Join(r.codeDir, r.pathMajor[1:], "go.mod")
 	gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
-	gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod)
-	found1 := err1 == nil && isMajor(gomod1, r.pathMajor)
-	found2 := err2 == nil && isMajor(gomod2, r.pathMajor)
-
-	if err2 == nil && !found2 {
-		return "", "", nil, fmt.Errorf("%s has non-...%s module path", file2, r.pathMajor)
-	}
-	if found1 && found2 {
-		return "", "", nil, fmt.Errorf("both %s and %s claim ...%s module", file1, file2, r.pathMajor)
-	}
-	if found2 {
-		return rev, filepath.Join(r.codeDir, r.pathMajor), gomod2, nil
+	if err1 != nil && !os.IsNotExist(err1) {
+		return "", "", nil, fmt.Errorf("reading %s/%s at revision %s: %v", r.pathPrefix, file1, rev, err1)
+	}
+	mpath1 := modfile.ModulePath(gomod1)
+	found1 := err1 == nil && isMajor(mpath1, r.pathMajor)
+
+	var file2 string
+	if r.pathMajor != "" && !strings.HasPrefix(r.pathMajor, ".") {
+		// Suppose pathMajor is "/v2".
+		// Either go.mod should claim v2 and v2/go.mod should not exist,
+		// or v2/go.mod should exist and claim v2. Not both.
+		// Note that we don't check the full path, just the major suffix,
+		// because of replacement modules. This might be a fork of
+		// the real module, found at a different path, usable only in
+		// a replace directive.
+		dir2 := path.Join(r.codeDir, r.pathMajor[1:])
+		file2 = path.Join(dir2, "go.mod")
+		gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod)
+		if err2 != nil && !os.IsNotExist(err2) {
+			return "", "", nil, fmt.Errorf("reading %s/%s at revision %s: %v", r.pathPrefix, file2, rev, err2)
+		}
+		mpath2 := modfile.ModulePath(gomod2)
+		found2 := err2 == nil && isMajor(mpath2, r.pathMajor)
+
+		if found1 && found2 {
+			return "", "", nil, fmt.Errorf("%s/%s and ...%s/go.mod both have ...%s module paths at revision %s", r.pathPrefix, file1, r.pathMajor, r.pathMajor, rev)
+		}
+		if found2 {
+			return rev, dir2, gomod2, nil
+		}
+		if err2 == nil {
+			if mpath2 == "" {
+				return "", "", nil, fmt.Errorf("%s/%s is missing module path at revision %s", r.pathPrefix, file2, rev)
+			}
+			return "", "", nil, fmt.Errorf("%s/%s has non-...%s module path %q at revision %s", r.pathPrefix, file2, r.pathMajor, mpath2, rev)
+		}
 	}
+
+	// Not v2/go.mod, so it's either go.mod or nothing. Which is it?
 	if found1 {
+		// Explicit go.mod with matching module path OK.
 		return rev, r.codeDir, gomod1, nil
 	}
-	return "", "", nil, fmt.Errorf("missing or invalid go.mod")
-}
-
-func isMajor(gomod []byte, pathMajor string) bool {
-	return strings.HasSuffix(modPath(gomod), pathMajor)
-}
-
-var moduleStr = []byte("module")
-
-func modPath(mod []byte) string {
-	for len(mod) > 0 {
-		line := mod
-		mod = nil
-		if i := bytes.IndexByte(line, '\n'); i >= 0 {
-			line, mod = line[:i], line[i+1:]
+	if err1 == nil {
+		// Explicit go.mod with non-matching module path disallowed.
+		suffix := ""
+		if file2 != "" {
+			suffix = fmt.Sprintf(" (and ...%s/go.mod does not exist)", r.pathMajor)
 		}
-		line = bytes.TrimSpace(line)
-		if !bytes.HasPrefix(line, moduleStr) {
-			continue
+		if mpath1 == "" {
+			return "", "", nil, fmt.Errorf("%s is missing module path%s at revision %s", file1, suffix, rev)
 		}
-		line = line[len(moduleStr):]
-		n := len(line)
-		line = bytes.TrimSpace(line)
-		if len(line) == n || len(line) == 0 {
-			continue
+		if r.pathMajor != "" { // ".v1", ".v2" for gopkg.in
+			return "", "", nil, fmt.Errorf("%s has non-...%s module path %q%s at revision %s", file1, r.pathMajor, mpath1, suffix, rev)
 		}
+		return "", "", nil, fmt.Errorf("%s has post-%s module path %q%s at revision %s", file1, semver.Major(version), mpath1, suffix, rev)
+	}
 
-		if line[0] == '"' || line[0] == '`' {
-			p, err := strconv.Unquote(string(line))
-			if err != nil {
-				return "" // malformed quoted string or multiline module path
-			}
-			return p
-		}
+	if r.codeDir == "" && (r.pathMajor == "" || strings.HasPrefix(r.pathMajor, ".")) {
+		// Implicit go.mod at root of repo OK for v0/v1 and for gopkg.in.
+		return rev, "", nil, nil
+	}
 
-		return string(line)
+	// Implicit go.mod below root of repo or at v2+ disallowed.
+	// Be clear about possibility of using either location for v2+.
+	if file2 != "" {
+		return "", "", nil, fmt.Errorf("missing %s/go.mod and ...%s/go.mod at revision %s", r.pathPrefix, r.pathMajor, rev)
 	}
-	return "" // missing module path
+	return "", "", nil, fmt.Errorf("missing %s/go.mod at revision %s", r.pathPrefix, rev)
+}
+
+func isMajor(mpath, pathMajor string) bool {
+	if mpath == "" {
+		return false
+	}
+	if pathMajor == "" {
+		// mpath must NOT have version suffix.
+		i := len(mpath)
+		for i > 0 && '0' <= mpath[i-1] && mpath[i-1] <= '9' {
+			i--
+		}
+		if i < len(mpath) && i >= 2 && mpath[i-1] == 'v' && mpath[i-2] == '/' {
+			// Found valid suffix.
+			return false
+		}
+		return true
+	}
+	// Otherwise pathMajor is ".v1", ".v2" (gopkg.in), or "/v2", "/v3" etc.
+	return strings.HasSuffix(mpath, pathMajor)
 }
 
 func (r *codeRepo) GoMod(version string) (data []byte, err error) {
@@ -284,7 +329,7 @@ func (r *codeRepo) GoMod(version string) (data []byte, err error) {
 	}
 	data, err = r.code.ReadFile(rev, path.Join(dir, "go.mod"), codehost.MaxGoMod)
 	if err != nil {
-		if e := strings.ToLower(err.Error()); strings.Contains(e, "not found") || strings.Contains(e, "404") { // TODO
+		if os.IsNotExist(err) {
 			return r.legacyGoMod(rev, dir), nil
 		}
 		return nil, err
@@ -292,41 +337,15 @@ func (r *codeRepo) GoMod(version string) (data []byte, err error) {
 	return data, nil
 }
 
-var altConfigs = []string{
-	"Gopkg.lock",
-
-	"GLOCKFILE",
-	"Godeps/Godeps.json",
-	"dependencies.tsv",
-	"glide.lock",
-	"vendor.conf",
-	"vendor.yml",
-	"vendor/manifest",
-	"vendor/vendor.json",
-}
-
 func (r *codeRepo) legacyGoMod(rev, dir string) []byte {
-	mf := new(modfile.File)
-	mf.AddModuleStmt(r.modPath)
-	for _, file := range altConfigs {
-		data, err := r.code.ReadFile(rev, path.Join(dir, file), codehost.MaxGoMod)
-		if err != nil {
-			continue
-		}
-		convert := modconv.Converters[file]
-		if convert == nil {
-			continue
-		}
-		if err := ConvertLegacyConfig(mf, file, data); err != nil {
-			continue
-		}
-		break
-	}
-	data, err := mf.Format()
-	if err != nil {
-		return []byte(fmt.Sprintf("%s\nmodule %q\n", modconv.Prefix, r.modPath))
-	}
-	return append([]byte(modconv.Prefix+"\n"), data...)
+	// We used to try to build a go.mod reflecting pre-existing
+	// package management metadata files, but the conversion
+	// was inherently imperfect (because those files don't have
+	// exactly the same semantics as go.mod) and, when done
+	// for dependencies in the middle of a build, impossible to
+	// correct. So we stopped.
+	// Return a fake go.mod that simply declares the module path.
+	return []byte(fmt.Sprintf("module %s\n", modfile.AutoQuote(r.modPath)))
 }
 
 func (r *codeRepo) modPrefix(rev string) string {
@@ -348,7 +367,7 @@ func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error
 	subdir := strings.Trim(strings.TrimPrefix(dir, actualDir), "/")
 
 	// Spool to local file.
-	f, err := ioutil.TempFile(tmpdir, "vgo-codehost-")
+	f, err := ioutil.TempFile(tmpdir, "go-codehost-")
 	if err != nil {
 		dl.Close()
 		return "", err
@@ -375,7 +394,7 @@ func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error
 	if err != nil {
 		return "", err
 	}
-	f2, err := ioutil.TempFile(tmpdir, "vgo-")
+	f2, err := ioutil.TempFile(tmpdir, "go-codezip-")
 	if err != nil {
 		return "", err
 	}
@@ -441,6 +460,11 @@ func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error
 		if !strings.HasPrefix(name, subdir) {
 			continue
 		}
+		if name == ".hg_archival.txt" {
+			// Inserted by hg archive.
+			// Not correct to drop from other version control systems, but too bad.
+			continue
+		}
 		name = strings.TrimPrefix(name, subdir)
 		if isVendoredPackage(name) {
 			continue
@@ -476,7 +500,8 @@ func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error
 	}
 
 	if !haveLICENSE && subdir != "" {
-		if data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE); err == nil {
+		data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE)
+		if err == nil {
 			w, err := zw.Create(r.modPrefix(version) + "/LICENSE")
 			if err != nil {
 				return "", err
@@ -571,6 +596,23 @@ func ParsePseudoVersion(repo Repo, version string) (rev string, err error) {
 
 var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.0\.0-[0-9]{14}-[A-Za-z0-9]+$`)
 
-func isPseudoVersion(v string) bool {
+// IsPseudoVersion reports whether v is a pseudo-version.
+func IsPseudoVersion(v string) bool {
 	return pseudoVersionRE.MatchString(v)
 }
+
+// PseudoVersionTime returns the time stamp of the pseudo-version v.
+// It returns an error if v is not a pseudo-version or if the time stamp
+// embedded in the pseudo-version is not a valid time.
+func PseudoVersionTime(v string) (time.Time, error) {
+	if !IsPseudoVersion(v) {
+		return time.Time{}, fmt.Errorf("not a pseudo-version")
+	}
+	i := strings.Index(v, "-") + 1
+	j := i + strings.Index(v[i:], "-")
+	t, err := time.Parse("20060102150405", v[i:j])
+	if err != nil {
+		return time.Time{}, fmt.Errorf("malformed pseudo-version %q", v)
+	}
+	return t, nil
+}