cmd/compile/internal/syntax: implement comment reporting in scanner

R=go1.11

In order to collect comments in the AST and for error testing purposes,
the scanner needs to not only recognize and skip comments, but also be
able to report them if so desired. This change adds a mode flag to the
scanner's init function which controls the scanner behavior around
comments.

In the common case where comments are not needed, there must be no
significant overhead. Thus, comments are reported via a handler upcall
rather than being returned as a _Comment token (which the parser would
have to filter out with every scanner.next() call).

Because the handlers for error messages, directives, and comments all
look the same (they take a position and text), and because directives
look like comments, and errors never start with a '/', this change
simplifies the scanner's init call to only take one (error) handler
instead of 2 or 3 different handlers with identical signature. It is
trivial in the handler to determine if we have an error, directive,
or general comment.

Finally, because directives are comments, when reporting directives
the full comment text is returned now rather than just the directive
text. This simplifies the implementation and makes the scanner API
more regular. Furthermore, it provides important information about
the comment style used by a directive, which may matter eventually
when we fully implement /*line file:line:col*/ directives.

Change-Id: I2adbfcebecd615e4237ed3a832b6ceb9518bf09c
Reviewed-on: https://go-review.googlesource.com/88215
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

1 files changed, 64 insertions, 4 deletions

diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go
index 160bcbee26..4bfe5871fa 100644
--- a/src/cmd/compile/internal/syntax/scanner_test.go
+++ b/src/cmd/compile/internal/syntax/scanner_test.go
@@ -24,7 +24,7 @@ func TestScanner(t *testing.T) {
 	defer src.Close()
 
 	var s scanner
-	s.init(src, nil, nil)
+	s.init(src, nil, 0)
 	for {
 		s.next()
 		if s.tok == _EOF {
@@ -53,7 +53,7 @@ func TestTokens(t *testing.T) {
 
 	// scan source
 	var got scanner
-	got.init(&buf, nil, nil)
+	got.init(&buf, nil, 0)
 	got.next()
 	for i, want := range sampleTokens {
 		nlsemi := false
@@ -263,6 +263,66 @@ var sampleTokens = [...]struct {
 	{_Var, "var", 0, 0},
 }
 
+func TestComments(t *testing.T) {
+	type comment struct {
+		line, col uint // 0-based
+		text      string
+	}
+
+	for _, test := range []struct {
+		src  string
+		want comment
+	}{
+		// no comments
+		{"no comment here", comment{0, 0, ""}},
+		{" /", comment{0, 0, ""}},
+		{"\n /*/", comment{0, 0, ""}},
+
+		//-style comments
+		{"// line comment\n", comment{0, 0, "// line comment"}},
+		{"package p // line comment\n", comment{0, 10, "// line comment"}},
+		{"//\n//\n\t// want this one\r\n", comment{2, 1, "// want this one\r"}},
+		{"\n\n//\n", comment{2, 0, "//"}},
+		{"//", comment{0, 0, "//"}},
+
+		/*-style comments */
+		{"/* regular comment */", comment{0, 0, "/* regular comment */"}},
+		{"package p /* regular comment", comment{0, 0, ""}},
+		{"\n\n\n/*\n*//* want this one */", comment{4, 2, "/* want this one */"}},
+		{"\n\n/**/", comment{2, 0, "/**/"}},
+		{"/*", comment{0, 0, ""}},
+	} {
+		var s scanner
+		var got comment
+		s.init(strings.NewReader(test.src),
+			func(line, col uint, msg string) {
+				if msg[0] != '/' {
+					// error
+					if msg != "comment not terminated" {
+						t.Errorf("%q: %s", test.src, msg)
+					}
+					return
+				}
+				got = comment{line - linebase, col - colbase, msg} // keep last one
+			}, comments)
+
+		for {
+			s.next()
+			if s.tok == _EOF {
+				break
+			}
+		}
+
+		want := test.want
+		if got.line != want.line || got.col != want.col {
+			t.Errorf("%q: got position %d:%d; want %d:%d", test.src, got.line, got.col, want.line, want.col)
+		}
+		if got.text != want.text {
+			t.Errorf("%q: got %q; want %q", test.src, got.text, want.text)
+		}
+	}
+}
+
 func TestScanErrors(t *testing.T) {
 	for _, test := range []struct {
 		src, msg  string
@@ -354,7 +414,7 @@ func TestScanErrors(t *testing.T) {
 				// TODO(gri) make this use position info
 				t.Errorf("%q: got unexpected %q at line = %d", test.src, msg, line)
 			}
-		}, nil)
+		}, 0)
 
 		for {
 			s.next()
@@ -373,7 +433,7 @@ func TestIssue21938(t *testing.T) {
 	s := "/*" + strings.Repeat(" ", 4089) + "*/ .5"
 
 	var got scanner
-	got.init(strings.NewReader(s), nil, nil)
+	got.init(strings.NewReader(s), nil, 0)
 	got.next()
 
 	if got.tok != _Literal || got.lit != ".5" {