cmd/compile/internal/syntax: match old parser errors and line numbers

This makes a bunch of changes to package syntax to tweak line numbers
for AST nodes. For example, short variable declaration statements are
now associated with the location of the ":=" token, and function calls
are associated with the location of the final ")" token. These help
satisfy many unit tests that assume the old parser's behavior.

Because many of these changes are questionable, they're guarded behind
a new "gcCompat" const to make them easy to identify and revisit in
the future.

A handful of remaining tests are too difficult to make behave
identically. These have been updated to execute with -newparser=0 and
comments explaining why they need to be fixed.

all.bash now passes with both the old and new parsers.

Change-Id: Iab834b71ca8698d39269f261eb5c92a0d55a3bf4
Reviewed-on: https://go-review.googlesource.com/27199
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

1 files changed, 38 insertions, 16 deletions

diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go
index 0f0f1ead9a..d02bb6d11b 100644
--- a/src/cmd/compile/internal/syntax/scanner.go
+++ b/src/cmd/compile/internal/syntax/scanner.go
@@ -46,7 +46,7 @@ redo:
 	// token start
 	s.pos, s.line = s.source.pos0(), s.source.line0
 
-	if isLetter(c) || c >= utf8.RuneSelf && unicode.IsLetter(c) {
+	if isLetter(c) || c >= utf8.RuneSelf && (unicode.IsLetter(c) || s.isCompatRune(c, true)) {
 		s.ident()
 		return
 	}
@@ -271,7 +271,7 @@ redo:
 
 	default:
 		s.tok = 0
-		s.error(fmt.Sprintf("invalid rune %q", c))
+		s.error(fmt.Sprintf("illegal character %#U", c))
 		goto redo
 	}
 
@@ -305,7 +305,7 @@ func (s *scanner) ident() {
 
 	// general case
 	if c >= utf8.RuneSelf {
-		for unicode.IsLetter(c) || c == '_' || unicode.IsDigit(c) {
+		for unicode.IsLetter(c) || c == '_' || unicode.IsDigit(c) || s.isCompatRune(c, false) {
 			c = s.getr()
 		}
 	}
@@ -327,6 +327,18 @@ func (s *scanner) ident() {
 	s.tok = _Name
 }
 
+func (s *scanner) isCompatRune(c rune, start bool) bool {
+	if !gcCompat || c < utf8.RuneSelf {
+		return false
+	}
+	if start && unicode.IsNumber(c) {
+		s.error(fmt.Sprintf("identifier cannot begin with digit %#U", c))
+	} else {
+		s.error(fmt.Sprintf("invalid identifier character %#U", c))
+	}
+	return true
+}
+
 // hash is a perfect hash function for keywords.
 // It assumes that s has at least length 2.
 func hash(s []byte) uint {
@@ -496,24 +508,26 @@ func (s *scanner) rune() {
 	s.startLit()
 
 	r := s.getr()
+	ok := false
 	if r == '\'' {
-		s.error("empty character literal")
+		s.error("empty character literal or unescaped ' in character literal")
 	} else if r == '\n' {
 		s.ungetr() // assume newline is not part of literal
 		s.error("newline in character literal")
 	} else {
-		ok := true
+		ok = true
 		if r == '\\' {
 			ok = s.escape('\'')
 		}
-		r = s.getr()
-		if r != '\'' {
-			// only report error if we're ok so far
-			if ok {
-				s.error("missing '")
-			}
-			s.ungetr()
+	}
+
+	r = s.getr()
+	if r != '\'' {
+		// only report error if we're ok so far
+		if ok {
+			s.error("missing '")
 		}
+		s.ungetr()
 	}
 
 	s.nlsemi = true
@@ -623,10 +637,18 @@ func (s *scanner) escape(quote rune) bool {
 			if c < 0 {
 				return true // complain in caller about EOF
 			}
-			if c != quote {
-				s.error(fmt.Sprintf("illegal character %#U in escape sequence", c))
+			if gcCompat {
+				name := "hex"
+				if base == 8 {
+					name = "octal"
+				}
+				s.error(fmt.Sprintf("non-%s character in escape sequence: %c", name, c))
 			} else {
-				s.error("escape sequence incomplete")
+				if c != quote {
+					s.error(fmt.Sprintf("illegal character %#U in escape sequence", c))
+				} else {
+					s.error("escape sequence incomplete")
+				}
 			}
 			s.ungetr()
 			return false
@@ -637,7 +659,7 @@ func (s *scanner) escape(quote rune) bool {
 	}
 	s.ungetr()
 
-	if x > max && n == 3 {
+	if x > max && base == 8 {
 		s.error(fmt.Sprintf("octal escape value > 255: %d", x))
 		return false
 	}