scanner: invalidate scanner.Position when no token is present

scanner.Position is the position of the most recently
scanned token. Make sure it is invalid if there is no
token scanned and update corresponding comment. This
is particularly important when reporting errors.

Fixes #2371.

R=r
CC=golang-dev
https://golang.org/cl/5294041

2 files changed, 20 insertions, 13 deletions

diff --git a/src/pkg/scanner/scanner.go b/src/pkg/scanner/scanner.go
index 8fbcb9c115..29e5f8c742 100644
--- a/src/pkg/scanner/scanner.go
+++ b/src/pkg/scanner/scanner.go
@@ -164,9 +164,12 @@ type Scanner struct {
 	// for values ch > ' '). The field may be changed at any time.
 	Whitespace uint64
 
-	// Current token position. The Offset, Line, and Column fields
-	// are set by Scan(); the Filename field is left untouched by the
-	// Scanner.
+	// Start position of most recently scanned token; set by Scan.
+	// Calling Init or Next invalidates the position (Line == 0).
+	// The Filename field is always left untouched by the Scanner.
+	// If an error is reported (via Error) and Position is invalid,
+	// the scanner is not inside a token. Call Pos to obtain an error
+	// position in that case.
 	Position
 }
 
@@ -201,6 +204,7 @@ func (s *Scanner) Init(src io.Reader) *Scanner {
 	s.ErrorCount = 0
 	s.Mode = GoTokens
 	s.Whitespace = GoWhitespace
+	s.Line = 0 // invalidate token position
 
 	return s
 }
@@ -302,6 +306,7 @@ func (s *Scanner) next() int {
 // get the current position.
 func (s *Scanner) Next() int {
 	s.tokPos = -1 // don't collect token text
+	s.Line = 0    // invalidate token position
 	ch := s.Peek()
 	s.ch = s.next()
 	return ch
@@ -323,7 +328,11 @@ func (s *Scanner) error(msg string) {
 		s.Error(s, msg)
 		return
 	}
-	fmt.Fprintf(os.Stderr, "%s: %s\n", s.Position, msg)
+	pos := s.Position
+	if !pos.IsValid() {
+		pos = s.Pos()
+	}
+	fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
 }
 
 func (s *Scanner) scanIdentifier() int {
@@ -520,6 +529,7 @@ func (s *Scanner) Scan() int {
 
 	// reset token text position
 	s.tokPos = -1
+	s.Line = 0
 
 redo:
 	// skip white space
diff --git a/src/pkg/scanner/scanner_test.go b/src/pkg/scanner/scanner_test.go
index 8403d61535..bbbba12834 100644
--- a/src/pkg/scanner/scanner_test.go
+++ b/src/pkg/scanner/scanner_test.go
@@ -100,11 +100,8 @@ var tokenList = []token{
 	{Ident, "_abc_123_"},
 	{Ident, "_äöü"},
 	{Ident, "_本"},
-	// TODO for unknown reasons these fail when checking the literals
-	/*
-		token{Ident, "äöü"},
-		token{Ident, "本"},
-	*/
+	{Ident, "äöü"},
+	{Ident, "本"},
 	{Ident, "a۰۱۸"},
 	{Ident, "foo६४"},
 	{Ident, "bar９８７６"},
@@ -365,14 +362,14 @@ func TestScanNext(t *testing.T) {
 	checkTok(t, s, 1, s.Scan(), Ident, "if")
 	checkTok(t, s, 1, s.Scan(), Ident, "a")
 	checkTok(t, s, 1, s.Scan(), '=', "=")
-	checkTok(t, s, 1, s.Next(), '=', "")
-	checkTok(t, s, 1, s.Next(), ' ', "")
-	checkTok(t, s, 1, s.Next(), 'b', "")
+	checkTok(t, s, 0, s.Next(), '=', "")
+	checkTok(t, s, 0, s.Next(), ' ', "")
+	checkTok(t, s, 0, s.Next(), 'b', "")
 	checkTok(t, s, 1, s.Scan(), Ident, "cd")
 	checkTok(t, s, 1, s.Scan(), '{', "{")
 	checkTok(t, s, 2, s.Scan(), Ident, "a")
 	checkTok(t, s, 2, s.Scan(), '+', "+")
-	checkTok(t, s, 2, s.Next(), '=', "")
+	checkTok(t, s, 0, s.Next(), '=', "")
 	checkTok(t, s, 2, s.Scan(), Ident, "c")
 	checkTok(t, s, 3, s.Scan(), '}', "}")
 	checkTok(t, s, 3, s.Scan(), -1, "")