cmd/compile/internal/syntax: provide LitKind to identify literals

- various minor cleanups related to literal scanning
- report newlines in character and regular string literals (plus tests)

5 files changed, 56 insertions, 35 deletions

diff --git a/src/cmd/compile/internal/syntax/nodes.go b/src/cmd/compile/internal/syntax/nodes.go
index 087d7acafe..4e264c1e82 100644
--- a/src/cmd/compile/internal/syntax/nodes.go
+++ b/src/cmd/compile/internal/syntax/nodes.go
@@ -116,6 +116,7 @@ type (
 	// Value
 	BasicLit struct {
 		Value string
+		Kind  LitKind
 		expr
 	}
 
diff --git a/src/cmd/compile/internal/syntax/parser.go b/src/cmd/compile/internal/syntax/parser.go
index 39e7347849..7093a7a606 100644
--- a/src/cmd/compile/internal/syntax/parser.go
+++ b/src/cmd/compile/internal/syntax/parser.go
@@ -291,7 +291,7 @@ func (p *parser) importDecl(group *Group) Decl {
 		d.LocalPkgName = n
 		p.next()
 	}
-	if p.tok == _Literal && (p.lit[0] == '"' || p.lit[0] == '`') {
+	if p.tok == _Literal && p.kind == StringLit {
 		d.Path = p.oliteral()
 	} else {
 		p.syntax_error("missing import path; require quoted string")
@@ -570,16 +570,12 @@ func (p *parser) operand(keep_parens bool) Expr {
 	}
 
 	switch p.tok {
-	case _Literal:
-		x := new(BasicLit)
-		x.init(p)
-		x.Value = p.lit
-		p.next()
-		return x
-
 	case _Name:
 		return p.name()
 
+	case _Literal:
+		return p.oliteral()
+
 	case _Lparen:
 		p.next()
 		p.xnest++
@@ -1192,6 +1188,7 @@ func (p *parser) oliteral() *BasicLit {
 		b := new(BasicLit)
 		b.init(p)
 		b.Value = p.lit
+		b.Kind = p.kind
 		p.next()
 		return b
 	}
diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go
index b9d586d89f..f146c8cb6f 100644
--- a/src/cmd/compile/internal/syntax/scanner.go
+++ b/src/cmd/compile/internal/syntax/scanner.go
@@ -20,6 +20,7 @@ type scanner struct {
 	pos, line int
 	tok       token
 	lit       string   // valid if tok is _Name or _Literal
+	kind      LitKind  // valid if tok is _Literal
 	op        Operator // valid if tok is _Operator, _AssignOp, or _IncOp
 	prec      int      // valid if tok is _Operator, _AssignOp, or _IncOp
 
@@ -66,23 +67,15 @@ redo:
 
 	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
 		s.number(c)
-		s.nlsemi = true
-		s.tok = _Literal
 
 	case '"':
 		s.stdString()
-		s.nlsemi = true
-		s.tok = _Literal
 
 	case '`':
 		s.rawString()
-		s.nlsemi = true
-		s.tok = _Literal
 
 	case '\'':
 		s.rune()
-		s.nlsemi = true
-		s.tok = _Literal
 
 	case '(':
 		s.tok = _Lparen
@@ -125,8 +118,6 @@ redo:
 			s.ungetr()
 			s.source.r0-- // make sure '.' is part of literal (line cannot have changed)
 			s.number('.')
-			s.nlsemi = true
-			s.tok = _Literal
 			break
 		}
 		if c == '.' {
@@ -164,7 +155,7 @@ redo:
 		// don't goto assignop - want _Star token
 		if s.getr() == '=' {
 			s.tok = _AssignOp
-			return
+			break
 		}
 		s.ungetr()
 		s.tok = _Star
@@ -335,8 +326,8 @@ func (s *scanner) ident() {
 	}
 
 	s.nlsemi = true
-	s.tok = _Name
 	s.lit = string(lit)
+	s.tok = _Name
 }
 
 // hash is a perfect hash function for keywords.
@@ -374,6 +365,7 @@ func (s *scanner) number(c rune) {
 	s.startLit()
 
 	if c != '.' {
+		s.kind = IntLit // until proven otherwise
 		if c == '0' {
 			c = s.getr()
 			if c == 'x' || c == 'X' {
@@ -387,9 +379,7 @@ func (s *scanner) number(c rune) {
 				if !hasDigit {
 					s.error("malformed hex constant")
 				}
-				s.ungetr()
-				s.lit = string(s.stopLit())
-				return
+				goto done
 			}
 
 			// decimal 0, octal, or float
@@ -405,9 +395,7 @@ func (s *scanner) number(c rune) {
 				if has8or9 {
 					s.error("malformed octal constant")
 				}
-				s.ungetr()
-				s.lit = string(s.stopLit())
-				return
+				goto done
 			}
 
 		} else {
@@ -420,6 +408,7 @@ func (s *scanner) number(c rune) {
 
 	// float
 	if c == '.' {
+		s.kind = FloatLit
 		c = s.getr()
 		for isDigit(c) {
 			c = s.getr()
@@ -428,6 +417,7 @@ func (s *scanner) number(c rune) {
 
 	// exponent
 	if c == 'e' || c == 'E' {
+		s.kind = FloatLit
 		c = s.getr()
 		if c == '-' || c == '+' {
 			c = s.getr()
@@ -441,11 +431,16 @@ func (s *scanner) number(c rune) {
 	}
 
 	// complex
-	if c != 'i' {
-		s.ungetr() // not complex
+	if c == 'i' {
+		s.kind = ImagLit
+		s.getr()
 	}
 
+done:
+	s.ungetr()
+	s.nlsemi = true
 	s.lit = string(s.stopLit())
+	s.tok = _Literal
 }
 
 func (s *scanner) stdString() {
@@ -453,20 +448,28 @@ func (s *scanner) stdString() {
 
 	for {
 		r := s.getr()
+		if r == '"' {
+			break
+		}
 		if r == '\\' {
 			s.escape('"')
 			continue
 		}
-		if r == '"' {
+		if r == '\n' {
+			s.ungetr() // assume newline is not part of literal
+			s.error("newline in string")
 			break
 		}
 		if r < 0 {
-			s.error("string not terminated")
+			s.error("string not terminated") // TODO(gri) should point at start of string
 			break
 		}
 	}
 
+	s.nlsemi = true
 	s.lit = string(s.stopLit())
+	s.kind = StringLit
+	s.tok = _Literal
 }
 
 func (s *scanner) rawString() {
@@ -478,7 +481,7 @@ func (s *scanner) rawString() {
 			break
 		}
 		if r < 0 {
-			s.error("string not terminated")
+			s.error("string not terminated") // TODO(gri) should point at start of string
 			break
 		}
 	}
@@ -486,14 +489,22 @@ func (s *scanner) rawString() {
 	// literal (even though they are not part of the literal
 	// value).
 
+	s.nlsemi = true
 	s.lit = string(s.stopLit())
+	s.kind = StringLit
+	s.tok = _Literal
 }
 
 func (s *scanner) rune() {
 	s.startLit()
 
 	r := s.getr()
-	if r != '\'' {
+	if r == '\'' {
+		s.error("empty character literal")
+	} else if r == '\n' {
+		s.ungetr() // assume newline is not part of literal
+		s.error("newline in character literal")
+	} else {
 		ok := true
 		if r == '\\' {
 			ok = s.escape('\'')
@@ -506,12 +517,12 @@ func (s *scanner) rune() {
 			}
 			s.ungetr()
 		}
-	} else {
-		s.error("empty character literal")
 	}
 
+	s.nlsemi = true
 	s.lit = string(s.stopLit())
-	return
+	s.kind = RuneLit
+	s.tok = _Literal
 }
 
 func (s *scanner) lineComment() {
diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go
index 297b61428a..dca838dafa 100644
--- a/src/cmd/compile/internal/syntax/scanner_test.go
+++ b/src/cmd/compile/internal/syntax/scanner_test.go
@@ -278,6 +278,7 @@ func TestScanErrors(t *testing.T) {
 		{"(x + 1.0e+x)", "malformed floating-point constant exponent", 10, 1},
 
 		{`''`, "empty character literal", 1, 1},
+		{"'\n", "newline in character literal", 1, 1},
 		{`'\`, "missing '", 2, 1},
 		{`'\'`, "missing '", 3, 1},
 		{`'\x`, "missing '", 3, 1},
@@ -290,6 +291,7 @@ func TestScanErrors(t *testing.T) {
 		{`'\400'`, "octal escape value > 255: 256", 5, 1},
 		{`'xx`, "missing '", 2, 1},
 
+		{"\"\n", "newline in string", 1, 1},
 		{`"`, "string not terminated", 1, 1},
 		{`"foo`, "string not terminated", 4, 1},
 		{"`", "string not terminated", 1, 1},
diff --git a/src/cmd/compile/internal/syntax/tokens.go b/src/cmd/compile/internal/syntax/tokens.go
index 91a217655f..bd0118a141 100644
--- a/src/cmd/compile/internal/syntax/tokens.go
+++ b/src/cmd/compile/internal/syntax/tokens.go
@@ -157,6 +157,16 @@ func contains(tokset uint64, tok token) bool {
 	return tokset&(1<<tok) != 0
 }
 
+type LitKind uint
+
+const (
+	IntLit LitKind = iota
+	FloatLit
+	ImagLit
+	RuneLit
+	StringLit
+)
+
 type Operator uint
 
 const (