cmd/go: add dark copy of golang.org/x/vgo

This CL corresponds to golang.org/cl/118096 (7fbc8df48a7)
in the vgo repo.

It copies the bulk of the code from vgo back into the main repo,
but completely disabled - vgo.Init is a no-op and vgo.Enabled
returns false unconditionally.

The point of this CL is to make the two trees easier to diff and
to make future syncs smaller.

Change-Id: Ic34fd5ddd8272a70c5a3b3437b5169e967d0ed03
Reviewed-on: https://go-review.googlesource.com/118095
Reviewed-by: Bryan C. Mills <bcmills@google.com>

1 files changed, 576 insertions, 0 deletions

diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
new file mode 100644
index 0000000000..bb6e8ac179
--- /dev/null
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -0,0 +1,576 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package modfetch
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"cmd/go/internal/modconv"
+	"cmd/go/internal/modfetch/codehost"
+	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
+	"cmd/go/internal/semver"
+)
+
+// A codeRepo implements modfetch.Repo using an underlying codehost.Repo.
+type codeRepo struct {
+	modPath  string
+	code     codehost.Repo
+	codeRoot string
+	codeDir  string
+
+	path        string
+	pathPrefix  string
+	pathMajor   string
+	pseudoMajor string
+}
+
+func newCodeRepo(code codehost.Repo, path string) (Repo, error) {
+	codeRoot := code.Root()
+	if !hasPathPrefix(path, codeRoot) {
+		return nil, fmt.Errorf("mismatched repo: found %s for %s", codeRoot, path)
+	}
+	pathPrefix, pathMajor, ok := module.SplitPathVersion(path)
+	if !ok {
+		return nil, fmt.Errorf("invalid module path %q", path)
+	}
+	pseudoMajor := "v0"
+	if pathMajor != "" {
+		pseudoMajor = pathMajor[1:]
+	}
+
+	// At this point we might have:
+	//	codeRoot = github.com/rsc/foo
+	//	path = github.com/rsc/foo/bar/v2
+	//	pathPrefix = github.com/rsc/foo/bar
+	//	pathMajor = /v2
+	//	pseudoMajor = v2
+	//
+	// Compute codeDir = bar, the subdirectory within the repo
+	// corresponding to the module root.
+	codeDir := strings.Trim(strings.TrimPrefix(pathPrefix, codeRoot), "/")
+	if strings.HasPrefix(path, "gopkg.in/") {
+		// But gopkg.in is a special legacy case, in which pathPrefix does not start with codeRoot.
+		// For example we might have:
+		//	codeRoot = gopkg.in/yaml.v2
+		//	pathPrefix = gopkg.in/yaml
+		//	pathMajor = .v2
+		//	pseudoMajor = v2
+		//	codeDir = pathPrefix (because codeRoot is not a prefix of pathPrefix)
+		// Clear codeDir - the module root is the repo root for gopkg.in repos.
+		codeDir = ""
+	}
+
+	r := &codeRepo{
+		modPath:     path,
+		code:        code,
+		codeRoot:    codeRoot,
+		codeDir:     codeDir,
+		pathPrefix:  pathPrefix,
+		pathMajor:   pathMajor,
+		pseudoMajor: pseudoMajor,
+	}
+
+	return r, nil
+}
+
+func (r *codeRepo) ModulePath() string {
+	return r.modPath
+}
+
+func (r *codeRepo) Versions(prefix string) ([]string, error) {
+	p := prefix
+	if r.codeDir != "" {
+		p = r.codeDir + "/" + p
+	}
+	tags, err := r.code.Tags(p)
+	if err != nil {
+		return nil, err
+	}
+	list := []string{}
+	for _, tag := range tags {
+		if !strings.HasPrefix(tag, p) {
+			continue
+		}
+		v := tag
+		if r.codeDir != "" {
+			v = v[len(r.codeDir)+1:]
+		}
+		if !semver.IsValid(v) || v != semver.Canonical(v) || isPseudoVersion(v) || !module.MatchPathMajor(v, r.pathMajor) {
+			continue
+		}
+		list = append(list, v)
+	}
+	SortVersions(list)
+	return list, nil
+}
+
+func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
+	if rev == "latest" {
+		return r.Latest()
+	}
+	codeRev := r.revToRev(rev)
+	if semver.IsValid(codeRev) && r.codeDir != "" {
+		codeRev = r.codeDir + "/" + codeRev
+	}
+	info, err := r.code.Stat(codeRev)
+	if err != nil {
+		return nil, err
+	}
+	return r.convert(info)
+}
+
+func (r *codeRepo) Latest() (*RevInfo, error) {
+	info, err := r.code.Latest()
+	if err != nil {
+		return nil, err
+	}
+	return r.convert(info)
+}
+
+func (r *codeRepo) convert(info *codehost.RevInfo) (*RevInfo, error) {
+	versionOK := func(v string) bool {
+		return semver.IsValid(v) && v == semver.Canonical(v) && !isPseudoVersion(v) && module.MatchPathMajor(v, r.pathMajor)
+	}
+	v := info.Version
+	if r.codeDir == "" {
+		if !versionOK(v) {
+			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
+		}
+	} else {
+		p := r.codeDir + "/"
+		if strings.HasPrefix(v, p) && versionOK(v[len(p):]) {
+			v = v[len(p):]
+		} else {
+			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
+		}
+	}
+
+	info2 := &RevInfo{
+		Name:    info.Name,
+		Short:   info.Short,
+		Time:    info.Time,
+		Version: v,
+	}
+	return info2, nil
+}
+
+func (r *codeRepo) revToRev(rev string) string {
+	if semver.IsValid(rev) {
+		if isPseudoVersion(rev) {
+			i := strings.Index(rev, "-")
+			j := strings.Index(rev[i+1:], "-")
+			return rev[i+1+j+1:]
+		}
+		if r.codeDir == "" {
+			return rev
+		}
+		return r.codeDir + "/" + rev
+	}
+	return rev
+}
+
+func (r *codeRepo) versionToRev(version string) (rev string, err error) {
+	if !semver.IsValid(version) {
+		return "", fmt.Errorf("malformed semantic version %q", version)
+	}
+	return r.revToRev(version), nil
+}
+
+func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err error) {
+	rev, err = r.versionToRev(version)
+	if err != nil {
+		return "", "", nil, err
+	}
+	if r.pathMajor == "" || strings.HasPrefix(r.pathMajor, ".") {
+		if r.codeDir == "" {
+			return rev, "", nil, nil
+		}
+		file1 := path.Join(r.codeDir, "go.mod")
+		gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
+		if err1 != nil {
+			return "", "", nil, fmt.Errorf("missing go.mod")
+		}
+		return rev, r.codeDir, gomod1, nil
+	}
+
+	// Suppose pathMajor is "/v2".
+	// Either go.mod should claim v2 and v2/go.mod should not exist,
+	// or v2/go.mod should exist and claim v2. Not both.
+	// Note that we don't check the full path, just the major suffix,
+	// because of replacement modules. This might be a fork of
+	// the real module, found at a different path, usable only in
+	// a replace directive.
+	file1 := path.Join(r.codeDir, "go.mod")
+	file2 := path.Join(r.codeDir, r.pathMajor[1:], "go.mod")
+	gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
+	gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod)
+	found1 := err1 == nil && isMajor(gomod1, r.pathMajor)
+	found2 := err2 == nil && isMajor(gomod2, r.pathMajor)
+
+	if err2 == nil && !found2 {
+		return "", "", nil, fmt.Errorf("%s has non-...%s module path", file2, r.pathMajor)
+	}
+	if found1 && found2 {
+		return "", "", nil, fmt.Errorf("both %s and %s claim ...%s module", file1, file2, r.pathMajor)
+	}
+	if found2 {
+		return rev, filepath.Join(r.codeDir, r.pathMajor), gomod2, nil
+	}
+	if found1 {
+		return rev, r.codeDir, gomod1, nil
+	}
+	return "", "", nil, fmt.Errorf("missing or invalid go.mod")
+}
+
+func isMajor(gomod []byte, pathMajor string) bool {
+	return strings.HasSuffix(modPath(gomod), pathMajor)
+}
+
+var moduleStr = []byte("module")
+
+func modPath(mod []byte) string {
+	for len(mod) > 0 {
+		line := mod
+		mod = nil
+		if i := bytes.IndexByte(line, '\n'); i >= 0 {
+			line, mod = line[:i], line[i+1:]
+		}
+		line = bytes.TrimSpace(line)
+		if !bytes.HasPrefix(line, moduleStr) {
+			continue
+		}
+		line = line[len(moduleStr):]
+		n := len(line)
+		line = bytes.TrimSpace(line)
+		if len(line) == n || len(line) == 0 {
+			continue
+		}
+
+		if line[0] == '"' || line[0] == '`' {
+			p, err := strconv.Unquote(string(line))
+			if err != nil {
+				return "" // malformed quoted string or multiline module path
+			}
+			return p
+		}
+
+		return string(line)
+	}
+	return "" // missing module path
+}
+
+func (r *codeRepo) GoMod(version string) (data []byte, err error) {
+	rev, dir, gomod, err := r.findDir(version)
+	if err != nil {
+		return nil, err
+	}
+	if gomod != nil {
+		return gomod, nil
+	}
+	data, err = r.code.ReadFile(rev, path.Join(dir, "go.mod"), codehost.MaxGoMod)
+	if err != nil {
+		if e := strings.ToLower(err.Error()); strings.Contains(e, "not found") || strings.Contains(e, "404") { // TODO
+			return r.legacyGoMod(rev, dir), nil
+		}
+		return nil, err
+	}
+	return data, nil
+}
+
+var altConfigs = []string{
+	"Gopkg.lock",
+
+	"GLOCKFILE",
+	"Godeps/Godeps.json",
+	"dependencies.tsv",
+	"glide.lock",
+	"vendor.conf",
+	"vendor.yml",
+	"vendor/manifest",
+	"vendor/vendor.json",
+}
+
+func (r *codeRepo) legacyGoMod(rev, dir string) []byte {
+	mf := new(modfile.File)
+	mf.AddModuleStmt(r.modPath)
+	for _, file := range altConfigs {
+		data, err := r.code.ReadFile(rev, path.Join(dir, file), codehost.MaxGoMod)
+		if err != nil {
+			continue
+		}
+		convert := modconv.Converters[file]
+		if convert == nil {
+			continue
+		}
+		if err := ConvertLegacyConfig(mf, file, data); err != nil {
+			continue
+		}
+		break
+	}
+	data, err := mf.Format()
+	if err != nil {
+		return []byte(fmt.Sprintf("%s\nmodule %q\n", modconv.Prefix, r.modPath))
+	}
+	return append([]byte(modconv.Prefix+"\n"), data...)
+}
+
+func (r *codeRepo) modPrefix(rev string) string {
+	return r.modPath + "@" + rev
+}
+
+func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error) {
+	rev, dir, _, err := r.findDir(version)
+	if err != nil {
+		return "", err
+	}
+	dl, actualDir, err := r.code.ReadZip(rev, dir, codehost.MaxZipFile)
+	if err != nil {
+		return "", err
+	}
+	if actualDir != "" && !hasPathPrefix(dir, actualDir) {
+		return "", fmt.Errorf("internal error: downloading %v %v: dir=%q but actualDir=%q", r.path, rev, dir, actualDir)
+	}
+	subdir := strings.Trim(strings.TrimPrefix(dir, actualDir), "/")
+
+	// Spool to local file.
+	f, err := ioutil.TempFile(tmpdir, "vgo-codehost-")
+	if err != nil {
+		dl.Close()
+		return "", err
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+	maxSize := int64(codehost.MaxZipFile)
+	lr := &io.LimitedReader{R: dl, N: maxSize + 1}
+	if _, err := io.Copy(f, lr); err != nil {
+		dl.Close()
+		return "", err
+	}
+	dl.Close()
+	if lr.N <= 0 {
+		return "", fmt.Errorf("downloaded zip file too large")
+	}
+	size := (maxSize + 1) - lr.N
+	if _, err := f.Seek(0, 0); err != nil {
+		return "", err
+	}
+
+	// Translate from zip file we have to zip file we want.
+	zr, err := zip.NewReader(f, size)
+	if err != nil {
+		return "", err
+	}
+	f2, err := ioutil.TempFile(tmpdir, "vgo-")
+	if err != nil {
+		return "", err
+	}
+
+	zw := zip.NewWriter(f2)
+	newName := f2.Name()
+	defer func() {
+		f2.Close()
+		if err != nil {
+			os.Remove(newName)
+		}
+	}()
+	if subdir != "" {
+		subdir += "/"
+	}
+	haveLICENSE := false
+	topPrefix := ""
+	haveGoMod := make(map[string]bool)
+	for _, zf := range zr.File {
+		if topPrefix == "" {
+			i := strings.Index(zf.Name, "/")
+			if i < 0 {
+				return "", fmt.Errorf("missing top-level directory prefix")
+			}
+			topPrefix = zf.Name[:i+1]
+		}
+		if !strings.HasPrefix(zf.Name, topPrefix) {
+			return "", fmt.Errorf("zip file contains more than one top-level directory")
+		}
+		dir, file := path.Split(zf.Name)
+		if file == "go.mod" {
+			haveGoMod[dir] = true
+		}
+	}
+	root := topPrefix + subdir
+	inSubmodule := func(name string) bool {
+		for {
+			dir, _ := path.Split(name)
+			if len(dir) <= len(root) {
+				return false
+			}
+			if haveGoMod[dir] {
+				return true
+			}
+			name = dir[:len(dir)-1]
+		}
+	}
+	for _, zf := range zr.File {
+		if topPrefix == "" {
+			i := strings.Index(zf.Name, "/")
+			if i < 0 {
+				return "", fmt.Errorf("missing top-level directory prefix")
+			}
+			topPrefix = zf.Name[:i+1]
+		}
+		if strings.HasSuffix(zf.Name, "/") { // drop directory dummy entries
+			continue
+		}
+		if !strings.HasPrefix(zf.Name, topPrefix) {
+			return "", fmt.Errorf("zip file contains more than one top-level directory")
+		}
+		name := strings.TrimPrefix(zf.Name, topPrefix)
+		if !strings.HasPrefix(name, subdir) {
+			continue
+		}
+		name = strings.TrimPrefix(name, subdir)
+		if isVendoredPackage(name) {
+			continue
+		}
+		if inSubmodule(zf.Name) {
+			continue
+		}
+		base := path.Base(name)
+		if strings.ToLower(base) == "go.mod" && base != "go.mod" {
+			return "", fmt.Errorf("zip file contains %s, want all lower-case go.mod", zf.Name)
+		}
+		if name == "LICENSE" {
+			haveLICENSE = true
+		}
+		size := int64(zf.UncompressedSize)
+		if size < 0 || maxSize < size {
+			return "", fmt.Errorf("module source tree too big")
+		}
+		maxSize -= size
+
+		rc, err := zf.Open()
+		if err != nil {
+			return "", err
+		}
+		w, err := zw.Create(r.modPrefix(version) + "/" + name)
+		lr := &io.LimitedReader{R: rc, N: size + 1}
+		if _, err := io.Copy(w, lr); err != nil {
+			return "", err
+		}
+		if lr.N <= 0 {
+			return "", fmt.Errorf("individual file too large")
+		}
+	}
+
+	if !haveLICENSE && subdir != "" {
+		if data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE); err == nil {
+			w, err := zw.Create(r.modPrefix(version) + "/LICENSE")
+			if err != nil {
+				return "", err
+			}
+			if _, err := w.Write(data); err != nil {
+				return "", err
+			}
+		}
+	}
+	if err := zw.Close(); err != nil {
+		return "", err
+	}
+	if err := f2.Close(); err != nil {
+		return "", err
+	}
+
+	return f2.Name(), nil
+}
+
+// hasPathPrefix reports whether the path s begins with the
+// elements in prefix.
+func hasPathPrefix(s, prefix string) bool {
+	switch {
+	default:
+		return false
+	case len(s) == len(prefix):
+		return s == prefix
+	case len(s) > len(prefix):
+		if prefix != "" && prefix[len(prefix)-1] == '/' {
+			return strings.HasPrefix(s, prefix)
+		}
+		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
+	}
+}
+
+func isVendoredPackage(name string) bool {
+	var i int
+	if strings.HasPrefix(name, "vendor/") {
+		i += len("vendor/")
+	} else if j := strings.Index(name, "/vendor/"); j >= 0 {
+		i += len("/vendor/")
+	} else {
+		return false
+	}
+	return strings.Contains(name[i:], "/")
+}
+
+func PseudoVersion(major string, t time.Time, rev string) string {
+	if major == "" {
+		major = "v0"
+	}
+	return fmt.Sprintf("%s.0.0-%s-%s", major, t.UTC().Format("20060102150405"), rev)
+}
+
+var ErrNotPseudoVersion = errors.New("not a pseudo-version")
+
+/*
+func ParsePseudoVersion(repo Repo, version string) (rev string, err error) {
+	major := semver.Major(version)
+	if major == "" {
+		return "", ErrNotPseudoVersion
+	}
+	majorPrefix := major + ".0.0-"
+	if !strings.HasPrefix(version, majorPrefix) || !strings.Contains(version[len(majorPrefix):], "-") {
+		return "", ErrNotPseudoVersion
+	}
+	versionSuffix := version[len(majorPrefix):]
+	for i := 0; versionSuffix[i] != '-'; i++ {
+		c := versionSuffix[i]
+		if c < '0' || '9' < c {
+			return "", ErrNotPseudoVersion
+		}
+	}
+	rev = versionSuffix[strings.Index(versionSuffix, "-")+1:]
+	if rev == "" {
+		return "", ErrNotPseudoVersion
+	}
+	if proxyURL != "" {
+		return version, nil
+	}
+	fullRev, t, err := repo.CommitInfo(rev)
+	if err != nil {
+		return "", fmt.Errorf("unknown pseudo-version %s: loading %v: %v", version, rev, err)
+	}
+	v := PseudoVersion(major, t, repo.ShortRev(fullRev))
+	if v != version {
+		return "", fmt.Errorf("unknown pseudo-version %s: %v is %v", version, rev, v)
+	}
+	return fullRev, nil
+}
+*/
+
+var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.0\.0-[0-9]{14}-[A-Za-z0-9]+$`)
+
+func isPseudoVersion(v string) bool {
+	return pseudoVersionRE.MatchString(v)
+}