cmd/go/internal/module: add new +incompatible version build annotation

Repos written before the introduction of semantic import versioning
introduced tags like v2.0.0, v3.0.0, and so on, expecting that
(1) the import path would remain unchanged, and perhaps also
(2) there would be at most one copy of the package in a build.

We've always accommodated these by mapping them into the
v0/v1 version range, so that if you ran

    go get k8s.io/client-go@v8.0.0

it would not complain about v8.x.x being a non-v1 version and
instead would map that version to a pseudo-version in go.mod:

    require k8s.io/client-go v0.0.0-20180628043050-7d04d0e2a0a1

The pseudo-version fails to capture two important facts: first,
that this really is the v8.0.0 tag, and second, that it should be
preferred over any earlier v1 tags.

A related problem is that running "go get k8s.io/client-go"
with no version will choose the latest v1 tag (v1.5.1), which
is obsolete.

This CL introduces a new version suffix +incompatible that
indicates that the tag should be considered an (incompatible)
extension of the v1 version sequence instead of part of its
own major version with its own versioned module path.
The requirement above can now be written:

    require k8s.io/client-go v8.0.0+incompatible

(The +metadata suffix is a standard part of semantic versioning,
and that suffix is ignored when comparing two versions for
precedence or equality. As part of canonicalizing versions
recorded in go.mod, the go command has always stripped all
such suffixes. It still strips nearly all: only +incompatible is
preserved now.)

In addition to recognizing the +incompatible, the code that
maps a commit hash to a version will use that form when
appropriate, so that

    go get k8s.io/client-go@7d04d0

will choose k8s.io/client-go@v8.0.0+incompatible.

Also, the code that computes the list of available versions from
a given source code repository also maps old tags to +incompatible
versions, for any tagged commit in which a go.mod file does not exist.
Therefore

    go list -m -versions k8s.io/client-go@latest

will show

    k8s.io/client-go v1.4.0 v1.5.0 v1.5.1 v2.0.0-alpha.0+incompatible ... v8.0.0+incompatible

and similarly

    go get k8s.io/client-go

will now choose v8.0.0+incompatible as the meaning of "latest tagged version".

The extraction of +incompatible versions from source code repos
depends on a codehost.Repo method ReadFileRevs, to do a bulk read
of multiple revisions of a file. That method is only implemented for git in this CL.
Future CLs will need to add support for that method to the other repository
implementations.

Documentation for this change is in CL 124515.

Fixes #26238.

Change-Id: I5bb1d7a46b5fffde34a3c0e6f8d19d9608188cea
Reviewed-on: https://go-review.googlesource.com/124384
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

1 files changed, 165 insertions, 1 deletions

diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index ef23e53775..d021a13890 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -245,10 +245,12 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
 	}
 
 	// Fast path: maybe rev is a hash we already have locally.
+	didStatLocal := false
 	if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) {
 		if info, err := r.statLocal(rev, rev); err == nil {
 			return info, nil
 		}
+		didStatLocal = true
 	}
 
 	// Maybe rev is a tag we already have locally.
@@ -308,11 +310,25 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
+	// Perhaps r.localTags did not have the ref when we loaded local tags,
+	// but we've since done fetches that pulled down the hash we need
+	// (or already have the hash we need, just without its tag).
+	// Either way, try a local stat before falling back to network I/O.
+	if !didStatLocal {
+		if info, err := r.statLocal(rev, hash); err == nil {
+			if strings.HasPrefix(ref, "refs/tags/") {
+				// Make sure tag exists, so it will be in localTags next time the go command is run.
+				Run(r.dir, "git", "tag", strings.TrimPrefix(ref, "refs/tags/"), hash)
+			}
+			return info, nil
+		}
+	}
+
 	// If we know a specific commit we need, fetch it.
 	if r.fetchLevel <= fetchSome && hash != "" && !r.local {
 		r.fetchLevel = fetchSome
 		var refspec string
-		if ref != "" && ref != "head" {
+		if ref != "" && ref != "HEAD" {
 			// If we do know the ref name, save the mapping locally
 			// so that (if it is a tag) it can show up in localTags
 			// on a future call. Also, some servers refuse to allow
@@ -438,6 +454,154 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
 	return out, nil
 }
 
+func (r *gitRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) {
+	// Create space to hold results.
+	files := make(map[string]*FileRev)
+	for _, rev := range revs {
+		f := &FileRev{Rev: rev}
+		files[rev] = f
+	}
+
+	// Collect locally-known revs.
+	need, err := r.readFileRevs(revs, file, files)
+	if err != nil {
+		return nil, err
+	}
+	if len(need) == 0 {
+		return files, nil
+	}
+
+	// Build list of known remote refs that might help.
+	var redo []string
+	r.refsOnce.Do(r.loadRefs)
+	if r.refsErr != nil {
+		return nil, r.refsErr
+	}
+	for _, tag := range need {
+		if r.refs["refs/tags/"+tag] != "" {
+			redo = append(redo, tag)
+		}
+	}
+	if len(redo) == 0 {
+		return files, nil
+	}
+
+	// Protect r.fetchLevel and the "fetch more and more" sequence.
+	// See stat method above.
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	var refs []string
+	var protoFlag []string
+	var unshallowFlag []string
+	for _, tag := range redo {
+		refs = append(refs, "refs/tags/"+tag+":refs/tags/"+tag)
+	}
+	if len(refs) > 1 {
+		unshallowFlag = unshallow(r.dir)
+		if len(unshallowFlag) > 0 {
+			// To work around a protocol version 2 bug that breaks --unshallow,
+			// add -c protocol.version=0.
+			// TODO(rsc): The bug is believed to be server-side, meaning only
+			// on Google's Git servers. Once the servers are fixed, drop the
+			// protocol.version=0. See Google-internal bug b/110495752.
+			protoFlag = []string{"-c", "protocol.version=0"}
+		}
+	}
+	if _, err := Run(r.dir, "git", protoFlag, "fetch", unshallowFlag, "-f", r.remote, refs); err != nil {
+		return nil, err
+	}
+
+	if _, err := r.readFileRevs(redo, file, files); err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}
+
+func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*FileRev) (missing []string, err error) {
+	var stdin bytes.Buffer
+	for _, tag := range tags {
+		fmt.Fprintf(&stdin, "refs/tags/%s\n", tag)
+		fmt.Fprintf(&stdin, "refs/tags/%s:%s\n", tag, file)
+	}
+
+	data, err := RunWithStdin(r.dir, &stdin, "git", "cat-file", "--batch")
+	if err != nil {
+		return nil, err
+	}
+
+	next := func() (typ string, body []byte, ok bool) {
+		var line string
+		i := bytes.IndexByte(data, '\n')
+		if i < 0 {
+			return "", nil, false
+		}
+		line, data = string(bytes.TrimSpace(data[:i])), data[i+1:]
+		if strings.HasSuffix(line, " missing") {
+			return "missing", nil, true
+		}
+		f := strings.Fields(line)
+		if len(f) != 3 {
+			return "", nil, false
+		}
+		n, err := strconv.Atoi(f[2])
+		if err != nil || n > len(data) {
+			return "", nil, false
+		}
+		body, data = data[:n], data[n:]
+		if len(data) > 0 && data[0] == '\r' {
+			data = data[1:]
+		}
+		if len(data) > 0 && data[0] == '\n' {
+			data = data[1:]
+		}
+		return f[1], body, true
+	}
+
+	badGit := func() ([]string, error) {
+		return nil, fmt.Errorf("malformed output from git cat-file --batch")
+	}
+
+	for _, tag := range tags {
+		commitType, _, ok := next()
+		if !ok {
+			return badGit()
+		}
+		fileType, fileData, ok := next()
+		if !ok {
+			return badGit()
+		}
+		f := fileMap[tag]
+		f.Data = nil
+		f.Err = nil
+		switch commitType {
+		default:
+			f.Err = fmt.Errorf("unexpected non-commit type %q for rev %s", commitType, tag)
+
+		case "missing":
+			// Note: f.Err must not satisfy os.IsNotExist. That's reserved for the file not existing in a valid commit.
+			f.Err = fmt.Errorf("no such rev %s", tag)
+			missing = append(missing, tag)
+
+		case "tag", "commit":
+			switch fileType {
+			default:
+				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
+			case "missing":
+				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist}
+			case "blob":
+				f.Data = fileData
+			}
+		}
+	}
+	if len(bytes.TrimSpace(data)) != 0 {
+		return badGit()
+	}
+
+	return missing, nil
+}
+
 func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
 	// TODO: Use maxSize or drop it.
 	args := []string{}