[dev.regabi] cmd/dist: automatically bootstrap cmd subdirs

We want almost all cmd subdirectories anyway, and relative to the cost
of the rest of toolchain bootstrapping, copying/rewriting a few extra
source files is way cheaper than the engineering cost of forgetting to
maintain these lists as we split out new packages.

While here, also add cmd/internal/archive (and make it compile with Go
1.4) because it'll be needed in subsequent refactorings anyway; and
skip files starting with # (emacs temporary files) and test files
ending with _test.go.

Change-Id: Ic86e680a5fdfaecd617c36d5d04413293b2d6f52
Reviewed-on: https://go-review.googlesource.com/c/go/+/279832
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Russ Cox <rsc@golang.org>
Trust: Matthew Dempsky <mdempsky@google.com>

1 files changed, 44 insertions, 69 deletions

diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go
index 5e1647cbf0..eb8729149c 100644
--- a/src/cmd/dist/buildtool.go
+++ b/src/cmd/dist/buildtool.go
@@ -23,78 +23,35 @@ import (
 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
 // All directories in this list are relative to and must be below $GOROOT/src.
 //
-// The list has have two kinds of entries: names beginning with cmd/ with
+// The list has two kinds of entries: names beginning with cmd/ with
 // no other slashes, which are commands, and other paths, which are packages
 // supporting the commands. Packages in the standard library can be listed
 // if a newer copy needs to be substituted for the Go 1.4 copy when used
-// by the command packages.
+// by the command packages. Paths ending with /... automatically
+// include all packages within subdirectories as well.
 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
 var bootstrapDirs = []string{
 	"cmd/asm",
-	"cmd/asm/internal/arch",
-	"cmd/asm/internal/asm",
-	"cmd/asm/internal/flags",
-	"cmd/asm/internal/lex",
+	"cmd/asm/internal/...",
 	"cmd/cgo",
 	"cmd/compile",
-	"cmd/compile/internal/amd64",
-	"cmd/compile/internal/base",
-	"cmd/compile/internal/arm",
-	"cmd/compile/internal/arm64",
-	"cmd/compile/internal/gc",
-	"cmd/compile/internal/ir",
-	"cmd/compile/internal/logopt",
-	"cmd/compile/internal/mips",
-	"cmd/compile/internal/mips64",
-	"cmd/compile/internal/ppc64",
-	"cmd/compile/internal/riscv64",
-	"cmd/compile/internal/s390x",
-	"cmd/compile/internal/ssa",
-	"cmd/compile/internal/syntax",
-	"cmd/compile/internal/types",
-	"cmd/compile/internal/x86",
-	"cmd/compile/internal/wasm",
+	"cmd/compile/internal/...",
+	"cmd/internal/archive",
 	"cmd/internal/bio",
 	"cmd/internal/codesign",
-	"cmd/internal/gcprog",
 	"cmd/internal/dwarf",
 	"cmd/internal/edit",
+	"cmd/internal/gcprog",
 	"cmd/internal/goobj",
+	"cmd/internal/obj/...",
 	"cmd/internal/objabi",
-	"cmd/internal/obj",
-	"cmd/internal/obj/arm",
-	"cmd/internal/obj/arm64",
-	"cmd/internal/obj/mips",
-	"cmd/internal/obj/ppc64",
-	"cmd/internal/obj/riscv",
-	"cmd/internal/obj/s390x",
-	"cmd/internal/obj/x86",
-	"cmd/internal/obj/wasm",
 	"cmd/internal/pkgpath",
 	"cmd/internal/src",
 	"cmd/internal/sys",
 	"cmd/link",
-	"cmd/link/internal/amd64",
-	"cmd/compile/internal/base",
-	"cmd/link/internal/arm",
-	"cmd/link/internal/arm64",
-	"cmd/link/internal/benchmark",
-	"cmd/link/internal/ld",
-	"cmd/link/internal/loadelf",
-	"cmd/link/internal/loader",
-	"cmd/link/internal/loadmacho",
-	"cmd/link/internal/loadpe",
-	"cmd/link/internal/loadxcoff",
-	"cmd/link/internal/mips",
-	"cmd/link/internal/mips64",
-	"cmd/link/internal/ppc64",
-	"cmd/link/internal/riscv64",
-	"cmd/link/internal/s390x",
-	"cmd/link/internal/sym",
-	"cmd/link/internal/x86",
+	"cmd/link/internal/...",
 	"compress/flate",
 	"compress/zlib",
-	"cmd/link/internal/wasm",
 	"container/heap",
 	"debug/dwarf",
 	"debug/elf",
@@ -116,6 +73,7 @@ var bootstrapDirs = []string{
 var ignorePrefixes = []string{
 	".",
 	"_",
+	"#",
 }
 
 // File suffixes that use build tags introduced since Go 1.4.
@@ -129,6 +87,7 @@ var ignoreSuffixes = []string{
 	"_wasm.s",
 	"_wasm.go",
 	"_test.s",
+	"_test.go",
 }
 
 func bootstrapBuildTools() {
@@ -154,31 +113,47 @@ func bootstrapBuildTools() {
 	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 	for _, dir := range bootstrapDirs {
-		src := pathf("%s/src/%s", goroot, dir)
-		dst := pathf("%s/%s", base, dir)
-		xmkdirall(dst)
-		if dir == "cmd/cgo" {
-			// Write to src because we need the file both for bootstrap
-			// and for later in the main build.
-			mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
-		}
-	Dir:
-		for _, name := range xreaddirfiles(src) {
+		recurse := strings.HasSuffix(dir, "/...")
+		dir = strings.TrimSuffix(dir, "/...")
+		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
+			}
+
+			name := filepath.Base(path)
+			src := pathf("%s/src/%s", goroot, path)
+			dst := pathf("%s/%s", base, path)
+
+			if info.IsDir() {
+				if !recurse && path != dir || name == "testdata" {
+					return filepath.SkipDir
+				}
+
+				xmkdirall(dst)
+				if path == "cmd/cgo" {
+					// Write to src because we need the file both for bootstrap
+					// and for later in the main build.
+					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
+					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
+				}
+				return nil
+			}
+
 			for _, pre := range ignorePrefixes {
 				if strings.HasPrefix(name, pre) {
-					continue Dir
+					return nil
 				}
 			}
 			for _, suf := range ignoreSuffixes {
 				if strings.HasSuffix(name, suf) {
-					continue Dir
+					return nil
 				}
 			}
-			srcFile := pathf("%s/%s", src, name)
-			dstFile := pathf("%s/%s", dst, name)
-			text := bootstrapRewriteFile(srcFile)
-			writefile(text, dstFile, 0)
-		}
+
+			text := bootstrapRewriteFile(src)
+			writefile(text, dst, 0)
+			return nil
+		})
 	}
 
 	// Set up environment for invoking Go 1.4 go command.