[dev.inline] cmd/compile/internal/syntax: report byte offset rather then rune count for column value

This will only become user-visible if error messages show column information.
Per the discussion in #10324.

For #10324.

Change-Id: I5959c1655aba74bb1a22fdc261cd728ffcfa6912
Reviewed-on: https://go-review.googlesource.com/34244
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

1 files changed, 64 insertions, 59 deletions

diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go
index 5532780399..c0943e7bce 100644
--- a/src/cmd/compile/internal/syntax/scanner_test.go
+++ b/src/cmd/compile/internal/syntax/scanner_test.go
@@ -263,71 +263,76 @@ func TestScanErrors(t *testing.T) {
 		// token.
 
 		// rune-level errors
-		{"fo\x00o", "invalid NUL character", 1, 3},
-		{"foo\n\ufeff bar", "invalid BOM in the middle of the file", 2, 1},
-		{"foo\n\n\xff    ", "invalid UTF-8 encoding", 3, 1},
+		{"fo\x00o", "invalid NUL character", 1, 2},
+		{"foo\n\ufeff bar", "invalid BOM in the middle of the file", 2, 0},
+		{"foo\n\n\xff    ", "invalid UTF-8 encoding", 3, 0},
 
 		// token-level errors
-		{"x + ~y", "bitwise complement operator is ^", 1, 5},
-		{"foo$bar = 0", "invalid character U+0024 '$'", 1, 4},
-		{"const x = 0xyz", "malformed hex constant", 1, 13},
-		{"0123456789", "malformed octal constant", 1, 11},
-		{"0123456789. /* foobar", "comment not terminated", 1, 13},   // valid float constant
-		{"0123456789e0 /*\nfoobar", "comment not terminated", 1, 14}, // valid float constant
-		{"var a, b = 08, 07\n", "malformed octal constant", 1, 14},
-		{"(x + 1.0e+x)", "malformed floating-point constant exponent", 1, 11},
-
-		{`''`, "empty character literal or unescaped ' in character literal", 1, 2},
-		{"'\n", "newline in character literal", 1, 2},
-		{`'\`, "missing '", 1, 3},
-		{`'\'`, "missing '", 1, 4},
-		{`'\x`, "missing '", 1, 4},
-		{`'\x'`, "non-hex character in escape sequence: '", 1, 4},
-		{`'\y'`, "unknown escape sequence", 1, 3},
-		{`'\x0'`, "non-hex character in escape sequence: '", 1, 5},
-		{`'\00'`, "non-octal character in escape sequence: '", 1, 5},
-		{`'\377' /*`, "comment not terminated", 1, 8}, // valid octal escape
-		{`'\378`, "non-octal character in escape sequence: 8", 1, 5},
-		{`'\400'`, "octal escape value > 255: 256", 1, 6},
-		{`'xx`, "missing '", 1, 3},
-
-		{"\"\n", "newline in string", 1, 2},
-		{`"`, "string not terminated", 1, 1},
-		{`"foo`, "string not terminated", 1, 1},
-		{"`", "string not terminated", 1, 1},
-		{"`foo", "string not terminated", 1, 1},
-		{"/*/", "comment not terminated", 1, 1},
-		{"/*\n\nfoo", "comment not terminated", 1, 1},
-		{"/*\n\nfoo", "comment not terminated", 1, 1},
-		{`"\`, "string not terminated", 1, 1},
-		{`"\"`, "string not terminated", 1, 1},
-		{`"\x`, "string not terminated", 1, 1},
-		{`"\x"`, "non-hex character in escape sequence: \"", 1, 4},
-		{`"\y"`, "unknown escape sequence", 1, 3},
-		{`"\x0"`, "non-hex character in escape sequence: \"", 1, 5},
-		{`"\00"`, "non-octal character in escape sequence: \"", 1, 5},
-		{`"\377" /*`, "comment not terminated", 1, 8}, // valid octal escape
-		{`"\378"`, "non-octal character in escape sequence: 8", 1, 5},
-		{`"\400"`, "octal escape value > 255: 256", 1, 6},
-
-		{`s := "foo\z"`, "unknown escape sequence", 1, 11},
-		{`s := "foo\z00\nbar"`, "unknown escape sequence", 1, 11},
-		{`"\x`, "string not terminated", 1, 1},
-		{`"\x"`, "non-hex character in escape sequence: \"", 1, 4},
-		{`var s string = "\x"`, "non-hex character in escape sequence: \"", 1, 19},
-		{`return "\Uffffffff"`, "escape sequence is invalid Unicode code point", 1, 19},
+		{"\u00BD" /* ½ */, "invalid identifier character U+00BD '½'", 1, 0},
+		{"\U0001d736\U0001d737\U0001d738_½" /* 𝜶𝜷𝜸_½ */, "invalid identifier character U+00BD '½'", 1, 13 /* byte offset */},
+		{"\U0001d7d8" /* 𝟘 */, "identifier cannot begin with digit U+1D7D8 '𝟘'", 1, 0},
+		{"foo\U0001d7d8_½" /* foo𝟘_½ */, "invalid identifier character U+00BD '½'", 1, 8 /* byte offset */},
+
+		{"x + ~y", "bitwise complement operator is ^", 1, 4},
+		{"foo$bar = 0", "invalid character U+0024 '$'", 1, 3},
+		{"const x = 0xyz", "malformed hex constant", 1, 12},
+		{"0123456789", "malformed octal constant", 1, 10},
+		{"0123456789. /* foobar", "comment not terminated", 1, 12},   // valid float constant
+		{"0123456789e0 /*\nfoobar", "comment not terminated", 1, 13}, // valid float constant
+		{"var a, b = 08, 07\n", "malformed octal constant", 1, 13},
+		{"(x + 1.0e+x)", "malformed floating-point constant exponent", 1, 10},
+
+		{`''`, "empty character literal or unescaped ' in character literal", 1, 1},
+		{"'\n", "newline in character literal", 1, 1},
+		{`'\`, "missing '", 1, 2},
+		{`'\'`, "missing '", 1, 3},
+		{`'\x`, "missing '", 1, 3},
+		{`'\x'`, "non-hex character in escape sequence: '", 1, 3},
+		{`'\y'`, "unknown escape sequence", 1, 2},
+		{`'\x0'`, "non-hex character in escape sequence: '", 1, 4},
+		{`'\00'`, "non-octal character in escape sequence: '", 1, 4},
+		{`'\377' /*`, "comment not terminated", 1, 7}, // valid octal escape
+		{`'\378`, "non-octal character in escape sequence: 8", 1, 4},
+		{`'\400'`, "octal escape value > 255: 256", 1, 5},
+		{`'xx`, "missing '", 1, 2},
+
+		{"\"\n", "newline in string", 1, 1},
+		{`"`, "string not terminated", 1, 0},
+		{`"foo`, "string not terminated", 1, 0},
+		{"`", "string not terminated", 1, 0},
+		{"`foo", "string not terminated", 1, 0},
+		{"/*/", "comment not terminated", 1, 0},
+		{"/*\n\nfoo", "comment not terminated", 1, 0},
+		{"/*\n\nfoo", "comment not terminated", 1, 0},
+		{`"\`, "string not terminated", 1, 0},
+		{`"\"`, "string not terminated", 1, 0},
+		{`"\x`, "string not terminated", 1, 0},
+		{`"\x"`, "non-hex character in escape sequence: \"", 1, 3},
+		{`"\y"`, "unknown escape sequence", 1, 2},
+		{`"\x0"`, "non-hex character in escape sequence: \"", 1, 4},
+		{`"\00"`, "non-octal character in escape sequence: \"", 1, 4},
+		{`"\377" /*`, "comment not terminated", 1, 7}, // valid octal escape
+		{`"\378"`, "non-octal character in escape sequence: 8", 1, 4},
+		{`"\400"`, "octal escape value > 255: 256", 1, 5},
+
+		{`s := "foo\z"`, "unknown escape sequence", 1, 10},
+		{`s := "foo\z00\nbar"`, "unknown escape sequence", 1, 10},
+		{`"\x`, "string not terminated", 1, 0},
+		{`"\x"`, "non-hex character in escape sequence: \"", 1, 3},
+		{`var s string = "\x"`, "non-hex character in escape sequence: \"", 1, 18},
+		{`return "\Uffffffff"`, "escape sequence is invalid Unicode code point", 1, 18},
 
 		// TODO(gri) move these test cases into an appropriate parser test
-		// {`//line :`, "invalid line number: ", 1, 9},
-		// {`//line :x`, "invalid line number: x", 1, 9},
-		// {`//line foo :`, "invalid line number: ", 1, 13},
-		// {`//line foo:123abc`, "invalid line number: 123abc", 1, 12},
-		// {`/**///line foo:x`, "invalid line number: x", 1, 16},
-		// {`//line foo:0`, "invalid line number: 0", 1, 12},
-		// {fmt.Sprintf(`//line foo:%d`, lineMax+1), fmt.Sprintf("invalid line number: %d", lineMax+1), 1, 12},
+		// {`//line :`, "invalid line number: ", 1, 8},
+		// {`//line :x`, "invalid line number: x", 1, 8},
+		// {`//line foo :`, "invalid line number: ", 1, 12},
+		// {`//line foo:123abc`, "invalid line number: 123abc", 1, 11},
+		// {`/**///line foo:x`, "invalid line number: x", 1, 15},
+		// {`//line foo:0`, "invalid line number: 0", 1, 11},
+		// {fmt.Sprintf(`//line foo:%d`, lineMax+1), fmt.Sprintf("invalid line number: %d", lineMax+1), 1, 11},
 
 		// former problem cases
-		{"package p\n\n\xef", "invalid UTF-8 encoding", 3, 1},
+		{"package p\n\n\xef", "invalid UTF-8 encoding", 3, 0},
 	} {
 		var s scanner
 		nerrors := 0