cmd/go/internal/module: add new +incompatible version build annotation

Repos written before the introduction of semantic import versioning
introduced tags like v2.0.0, v3.0.0, and so on, expecting that
(1) the import path would remain unchanged, and perhaps also
(2) there would be at most one copy of the package in a build.

We've always accommodated these by mapping them into the
v0/v1 version range, so that if you ran

    go get k8s.io/client-go@v8.0.0

it would not complain about v8.x.x being a non-v1 version and
instead would map that version to a pseudo-version in go.mod:

    require k8s.io/client-go v0.0.0-20180628043050-7d04d0e2a0a1

The pseudo-version fails to capture two important facts: first,
that this really is the v8.0.0 tag, and second, that it should be
preferred over any earlier v1 tags.

A related problem is that running "go get k8s.io/client-go"
with no version will choose the latest v1 tag (v1.5.1), which
is obsolete.

This CL introduces a new version suffix +incompatible that
indicates that the tag should be considered an (incompatible)
extension of the v1 version sequence instead of part of its
own major version with its own versioned module path.
The requirement above can now be written:

    require k8s.io/client-go v8.0.0+incompatible

(The +metadata suffix is a standard part of semantic versioning,
and that suffix is ignored when comparing two versions for
precedence or equality. As part of canonicalizing versions
recorded in go.mod, the go command has always stripped all
such suffixes. It still strips nearly all: only +incompatible is
preserved now.)

In addition to recognizing the +incompatible, the code that
maps a commit hash to a version will use that form when
appropriate, so that

    go get k8s.io/client-go@7d04d0

will choose k8s.io/client-go@v8.0.0+incompatible.

Also, the code that computes the list of available versions from
a given source code repository also maps old tags to +incompatible
versions, for any tagged commit in which a go.mod file does not exist.
Therefore

    go list -m -versions k8s.io/client-go@latest

will show

    k8s.io/client-go v1.4.0 v1.5.0 v1.5.1 v2.0.0-alpha.0+incompatible ... v8.0.0+incompatible

and similarly

    go get k8s.io/client-go

will now choose v8.0.0+incompatible as the meaning of "latest tagged version".

The extraction of +incompatible versions from source code repos
depends on a codehost.Repo method ReadFileRevs, to do a bulk read
of multiple revisions of a file. That method is only implemented for git in this CL.
Future CLs will need to add support for that method to the other repository
implementations.

Documentation for this change is in CL 124515.

Fixes #26238.

Change-Id: I5bb1d7a46b5fffde34a3c0e6f8d19d9608188cea
Reviewed-on: https://go-review.googlesource.com/124384
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

1 files changed, 59 insertions, 9 deletions

diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index 33be117de9..f5d2e3e27f 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -97,7 +97,9 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	list := []string{}
+	var incompatible []string
 	for _, tag := range tags {
 		if !strings.HasPrefix(tag, p) {
 			continue
@@ -106,11 +108,34 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
 		if r.codeDir != "" {
 			v = v[len(r.codeDir)+1:]
 		}
-		if !semver.IsValid(v) || v != semver.Canonical(v) || IsPseudoVersion(v) || !module.MatchPathMajor(v, r.pathMajor) {
+		if v == "" || v != module.CanonicalVersion(v) || IsPseudoVersion(v) {
+			continue
+		}
+		if !module.MatchPathMajor(v, r.pathMajor) {
+			if r.codeDir == "" && r.pathMajor == "" && semver.Major(v) > "v1" {
+				incompatible = append(incompatible, v)
+			}
 			continue
 		}
 		list = append(list, v)
 	}
+
+	if len(incompatible) > 0 {
+		// Check for later versions that were created not following semantic import versioning,
+		// as indicated by the absence of a go.mod file. Those versions can be addressed
+		// by referring to them with a +incompatible suffix, as in v17.0.0+incompatible.
+		files, err := r.code.ReadFileRevs(incompatible, "go.mod", codehost.MaxGoMod)
+		if err != nil {
+			return nil, err
+		}
+		for _, rev := range incompatible {
+			f := files[rev]
+			if os.IsNotExist(f.Err) {
+				list = append(list, rev+"+incompatible")
+			}
+		}
+	}
+
 	SortVersions(list)
 	return list, nil
 }
@@ -146,7 +171,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 	}
 
 	// Determine version.
-	if semver.IsValid(statVers) && statVers == semver.Canonical(statVers) && module.MatchPathMajor(statVers, r.pathMajor) {
+	if module.CanonicalVersion(statVers) == statVers && module.MatchPathMajor(statVers, r.pathMajor) {
 		// The original call was repo.Stat(statVers), and requestedVersion is OK, so use it.
 		info2.Version = statVers
 	} else {
@@ -157,22 +182,43 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 			p = r.codeDir + "/"
 		}
 
-		tagOK := func(v string) bool {
+		// If this is a plain tag (no dir/ prefix)
+		// and the module path is unversioned,
+		// and if the underlying file tree has no go.mod,
+		// then allow using the tag with a +incompatible suffix.
+		canUseIncompatible := false
+		if r.codeDir == "" && r.pathMajor == "" {
+			_, errGoMod := r.code.ReadFile(info.Name, "go.mod", codehost.MaxGoMod)
+			if errGoMod != nil {
+				canUseIncompatible = true
+			}
+		}
+
+		tagOK := func(v string) string {
 			if !strings.HasPrefix(v, p) {
-				return false
+				return ""
 			}
 			v = v[len(p):]
-			return semver.IsValid(v) && v == semver.Canonical(v) && module.MatchPathMajor(v, r.pathMajor) && !IsPseudoVersion(v)
+			if module.CanonicalVersion(v) != v || IsPseudoVersion(v) {
+				return ""
+			}
+			if module.MatchPathMajor(v, r.pathMajor) {
+				return v
+			}
+			if canUseIncompatible {
+				return v + "+incompatible"
+			}
+			return ""
 		}
 
 		// If info.Version is OK, use it.
-		if tagOK(info.Version) {
-			info2.Version = info.Version[len(p):]
+		if v := tagOK(info.Version); v != "" {
+			info2.Version = v
 		} else {
 			// Otherwise look through all known tags for latest in semver ordering.
 			for _, tag := range info.Tags {
-				if tagOK(tag) && semver.Compare(info2.Version, tag[len(p):]) < 0 {
-					info2.Version = tag[len(p):]
+				if v := tagOK(tag); v != "" && semver.Compare(info2.Version, v) < 0 {
+					info2.Version = v
 				}
 			}
 			// Otherwise make a pseudo-version.
@@ -185,6 +231,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 	// Do not allow a successful stat of a pseudo-version for a subdirectory
 	// unless the subdirectory actually does have a go.mod.
 	if IsPseudoVersion(info2.Version) && r.codeDir != "" {
+		// TODO: git describe --first-parent --match 'v[0-9]*' --tags
 		_, _, _, err := r.findDir(info2.Version)
 		if err != nil {
 			// TODO: It would be nice to return an error like "not a module".
@@ -203,6 +250,9 @@ func (r *codeRepo) revToRev(rev string) string {
 			j := strings.Index(rev[i+1:], "-")
 			return rev[i+1+j+1:]
 		}
+		if semver.Build(rev) == "+incompatible" {
+			rev = rev[:len(rev)-len("+incompatible")]
+		}
 		if r.codeDir == "" {
 			return rev
 		}