cmd/go: add //go:embed support

The final piece of //go:embed support: have the go command stitch
together parsing in go/build, low-level data initialization in cmd/compile,
and the new data structures in package embed, to make the //go:embed
feature actually function.

And test, now that all the pieces are available to work together.

For #41191.
(Issue not fixed: still need to add a tool for use by Bazel.)

Change-Id: Ib1d198345c3b4d557d340f292eda13b984b65d65
Reviewed-on: https://go-review.googlesource.com/c/go/+/243945
Trust: Russ Cox <rsc@golang.org>
Trust: Jay Conrod <jayconrod@google.com>
Trust: Johan Brandhorst <johan.brandhorst@gmail.com>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Reviewed-by: Johan Brandhorst <johan.brandhorst@gmail.com>

1 files changed, 197 insertions, 5 deletions

diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go
index ff744ee9fa..30ca33b663 100644
--- a/src/cmd/go/internal/load/pkg.go
+++ b/src/cmd/go/internal/load/pkg.go
@@ -17,6 +17,7 @@ import (
 	"io/fs"
 	"io/ioutil"
 	"os"
+	"path"
 	pathpkg "path"
 	"path/filepath"
 	"runtime"
@@ -94,6 +95,10 @@ type PackagePublic struct {
 	SwigCXXFiles      []string `json:",omitempty"` // .swigcxx files
 	SysoFiles         []string `json:",omitempty"` // .syso system object files added to package
 
+	// Embedded files
+	EmbedPatterns []string `json:",omitempty"` // //go:embed patterns
+	EmbedFiles    []string `json:",omitempty"` // files and directories matched by EmbedPatterns
+
 	// Cgo directives
 	CgoCFLAGS    []string `json:",omitempty"` // cgo: flags for C compiler
 	CgoCPPFLAGS  []string `json:",omitempty"` // cgo: flags for C preprocessor
@@ -115,10 +120,14 @@ type PackagePublic struct {
 	// Test information
 	// If you add to this list you MUST add to p.AllFiles (below) too.
 	// Otherwise file name security lists will not apply to any new additions.
-	TestGoFiles  []string `json:",omitempty"` // _test.go files in package
-	TestImports  []string `json:",omitempty"` // imports from TestGoFiles
-	XTestGoFiles []string `json:",omitempty"` // _test.go files outside package
-	XTestImports []string `json:",omitempty"` // imports from XTestGoFiles
+	TestGoFiles        []string `json:",omitempty"` // _test.go files in package
+	TestImports        []string `json:",omitempty"` // imports from TestGoFiles
+	TestEmbedPatterns  []string `json:",omitempty"` // //go:embed patterns
+	TestEmbedFiles     []string `json:",omitempty"` // //files matched by EmbedPatterns
+	XTestGoFiles       []string `json:",omitempty"` // _test.go files outside package
+	XTestImports       []string `json:",omitempty"` // imports from XTestGoFiles
+	XTestEmbedPatterns []string `json:",omitempty"` // //go:embed patterns
+	XTestEmbedFiles    []string `json:",omitempty"` // //files matched by EmbedPatterns
 }
 
 // AllFiles returns the names of all the files considered for the package.
@@ -127,7 +136,7 @@ type PackagePublic struct {
 // The go/build package filtered others out (like foo_wrongGOARCH.s)
 // and that's OK.
 func (p *Package) AllFiles() []string {
-	return str.StringList(
+	files := str.StringList(
 		p.GoFiles,
 		p.CgoFiles,
 		// no p.CompiledGoFiles, because they are from GoFiles or generated by us
@@ -145,6 +154,27 @@ func (p *Package) AllFiles() []string {
 		p.TestGoFiles,
 		p.XTestGoFiles,
 	)
+
+	// EmbedFiles may overlap with the other files.
+	// Dedup, but delay building the map as long as possible.
+	// Only files in the current directory (no slash in name)
+	// need to be checked against the files variable above.
+	var have map[string]bool
+	for _, file := range p.EmbedFiles {
+		if !strings.Contains(file, "/") {
+			if have == nil {
+				have = make(map[string]bool)
+				for _, file := range files {
+					have[file] = true
+				}
+			}
+			if have[file] {
+				continue
+			}
+		}
+		files = append(files, file)
+	}
+	return files
 }
 
 // Desc returns the package "description", for use in b.showOutput.
@@ -174,6 +204,7 @@ type PackageInternal struct {
 	GobinSubdir       bool                 // install target would be subdir of GOBIN
 	BuildInfo         string               // add this info to package main
 	TestmainGo        *[]byte              // content for _testmain.go
+	Embed             map[string][]string  // //go:embed comment mapping
 
 	Asmflags   []string // -asmflags for this package
 	Gcflags    []string // -gcflags for this package
@@ -366,6 +397,9 @@ func (p *Package) copyBuild(pp *build.Package) {
 		p.TestImports = nil
 		p.XTestImports = nil
 	}
+	p.EmbedPatterns = pp.EmbedPatterns
+	p.TestEmbedPatterns = pp.TestEmbedPatterns
+	p.XTestEmbedPatterns = pp.XTestEmbedPatterns
 }
 
 // A PackageError describes an error loading information about a package.
@@ -960,6 +994,12 @@ func (pre *preload) preloadImports(imports []string, parent *build.Package) {
 // loadPackageData have completed. The preloader will not make any new calls
 // to loadPackageData.
 func (pre *preload) flush() {
+	// flush is usually deferred.
+	// Don't hang program waiting for workers on panic.
+	if v := recover(); v != nil {
+		panic(v)
+	}
+
 	close(pre.cancel)
 	for i := 0; i < preloadWorkerCount; i++ {
 		pre.sema <- struct{}{}
@@ -1624,6 +1664,11 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
 		p.setLoadPackageDataError(err, path, stk, importPos)
 	}
 
+	p.EmbedFiles, p.Internal.Embed, err = p.resolveEmbed(p.EmbedPatterns)
+	if err != nil {
+		setError(err)
+	}
+
 	useBindir := p.Name == "main"
 	if !p.Standard {
 		switch cfg.BuildBuildmode {
@@ -1865,6 +1910,153 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
 	}
 }
 
+// ResolveEmbed resolves //go:embed patterns and returns only the file list.
+// For use by go list to compute p.TestEmbedFiles and p.XTestEmbedFiles.
+func (p *Package) ResolveEmbed(patterns []string) []string {
+	files, _, _ := p.resolveEmbed(patterns)
+	return files
+}
+
+// resolveEmbed resolves //go:embed patterns to precise file lists.
+// It sets files to the list of unique files matched (for go list),
+// and it sets pmap to the more precise mapping from
+// patterns to files.
+// TODO(rsc): All these messages need position information for better error reports.
+func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[string][]string, err error) {
+	pmap = make(map[string][]string)
+	have := make(map[string]int)
+	dirOK := make(map[string]bool)
+	pid := 0 // pattern ID, to allow reuse of have map
+	for _, pattern := range patterns {
+		pid++
+
+		// Check pattern is valid for //go:embed.
+		if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) {
+			return nil, nil, fmt.Errorf("pattern %s: invalid pattern syntax", pattern)
+		}
+
+		// Glob to find matches.
+		match, err := fsys.Glob(p.Dir + string(filepath.Separator) + filepath.FromSlash(pattern))
+		if err != nil {
+			return nil, nil, fmt.Errorf("pattern %s: %v", pattern, err)
+		}
+
+		// Filter list of matches down to the ones that will still exist when
+		// the directory is packaged up as a module. (If p.Dir is in the module cache,
+		// only those files exist already, but if p.Dir is in the current module,
+		// then there may be other things lying around, like symbolic links or .git directories.)
+		var list []string
+		for _, file := range match {
+			rel := filepath.ToSlash(file[len(p.Dir)+1:]) // file, relative to p.Dir
+
+			what := "file"
+			info, err := fsys.Lstat(file)
+			if err != nil {
+				return nil, nil, err
+			}
+			if info.IsDir() {
+				what = "directory"
+			}
+
+			// Check that directories along path do not begin a new module
+			// (do not contain a go.mod).
+			for dir := file; len(dir) > len(p.Dir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
+				if _, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil {
+					return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in different module", pattern, what, rel)
+				}
+				if dir != file {
+					if info, err := fsys.Lstat(dir); err == nil && !info.IsDir() {
+						return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in non-directory %s", pattern, what, rel, dir[len(p.Dir)+1:])
+					}
+				}
+				dirOK[dir] = true
+				if elem := filepath.Base(dir); isBadEmbedName(elem) {
+					if dir == file {
+						return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: invalid name %s", pattern, what, rel, elem)
+					} else {
+						return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in invalid directory %s", pattern, what, rel, elem)
+					}
+				}
+			}
+
+			switch {
+			default:
+				return nil, nil, fmt.Errorf("pattern %s: cannot embed irregular file %s", pattern, rel)
+
+			case info.Mode().IsRegular():
+				if have[rel] != pid {
+					have[rel] = pid
+					list = append(list, rel)
+				}
+
+			case info.IsDir():
+				// Gather all files in the named directory, stopping at module boundaries
+				// and ignoring files that wouldn't be packaged into a module.
+				count := 0
+				err := fsys.Walk(file, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+					rel := filepath.ToSlash(path[len(p.Dir)+1:])
+					if info.IsDir() {
+						if _, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil {
+							return filepath.SkipDir
+						}
+						return nil
+					}
+					if !info.Mode().IsRegular() {
+						return nil
+					}
+					if isBadEmbedName(info.Name()) {
+						// Ignore bad names, assuming they won't go into modules.
+						return nil
+					}
+					count++
+					if have[rel] != pid {
+						have[rel] = pid
+						list = append(list, rel)
+					}
+					return nil
+				})
+				if err != nil {
+					return nil, nil, err
+				}
+				if count == 0 {
+					return nil, nil, fmt.Errorf("pattern %s: cannot embed directory %s: contains no embeddable files", pattern, rel)
+				}
+			}
+		}
+
+		if len(list) == 0 {
+			return nil, nil, fmt.Errorf("pattern %s: no matching files found", pattern)
+		}
+		sort.Strings(list)
+		pmap[pattern] = list
+	}
+
+	for file := range have {
+		files = append(files, file)
+	}
+	sort.Strings(files)
+	return files, pmap, nil
+}
+
+func validEmbedPattern(pattern string) bool {
+	return pattern != "." && fs.ValidPath(pattern)
+}
+
+// isBadEmbedName reports whether name is the base name of a file that
+// can't or won't be included in modules and therefore shouldn't be treated
+// as existing for embedding.
+func isBadEmbedName(name string) bool {
+	switch name {
+	// Version control directories won't be present in module.
+	case ".bzr", ".hg", ".git", ".svn":
+		return true
+	}
+	return false
+}
+
 // collectDeps populates p.Deps and p.DepsErrors by iterating over
 // p.Internal.Imports.
 //