cmd/internal/gc: move check for large-hence-heap-allocated types into escape analysis

Before this change, the check for too-large arrays (and other large
types) occurred after escape analysis.  If the data moved off stack
and onto the heap contained any pointers, it would therefore escape,
but because the too-large check occurred after escape analysis this
would not be recorded and a stack pointer would leak to the heap
(see the modified escape_array.go for an example).

Some of these appear to remain, in calls to typecheck from within walk.

Also corrected a few comments in escape_array.go about "BAD"
analysis that is now done correctly.

Enhanced to move aditional EscNone-but-large-so-heap checks into esc.c.

Change-Id: I770c111baff28a9ed5f8beb601cf09dacc561b83
Reviewed-on: https://go-review.googlesource.com/10268
Reviewed-by: Russ Cox <rsc@golang.org>

1 files changed, 56 insertions, 6 deletions

diff --git a/test/escape_array.go b/test/escape_array.go
index ac51fe7ca6..5da77713d2 100644
--- a/test/escape_array.go
+++ b/test/escape_array.go
@@ -4,10 +4,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Test escape analysis for function parameters.
-
-// In this test almost everything is BAD except the simplest cases
-// where input directly flows to output.
+// Test escape analysis for arrays and some large things
 
 package foo
 
@@ -59,14 +56,67 @@ func fup(x *U, y *string) *string { // ERROR "leaking param: x to result ~r2 lev
 	return x[1]
 }
 
-// BAD: would be nice to record that *y (content) is what leaks, not y itself
 func fum(x *U, y **string) *string { // ERROR "leaking param: x to result ~r2 level=1$" "leaking param content: y$"
 	x[0] = *y
 	return x[1]
 }
 
-// BAD: would be nice to record that y[0] (content) is what leaks, not y itself
 func fuo(x *U, y *U) *string { // ERROR "leaking param: x to result ~r2 level=1$" "leaking param content: y$"
 	x[0] = y[0]
 	return x[1]
 }
+
+// These two tests verify that:
+// small array literals are stack allocated;
+// pointers stored in small array literals do not escape;
+// large array literals are heap allocated;
+// pointers stored in large array literals escape.
+func hugeLeaks1(x **string, y **string) { // ERROR "leaking param content: x" "hugeLeaks1 y does not escape" "mark escaped content: x"
+	a := [10]*string{*y}
+	_ = a
+	// 4 x 4,000,000 exceeds MaxStackVarSize, therefore it must be heap allocated if pointers are 4 bytes or larger.
+	b := [4000000]*string{*x} // ERROR "moved to heap: b"
+	_ = b
+}
+
+func hugeLeaks2(x *string, y *string) { // ERROR "leaking param: x" "hugeLeaks2 y does not escape"
+	a := [10]*string{y}
+	_ = a
+	// 4 x 4,000,000 exceeds MaxStackVarSize, therefore it must be heap allocated if pointers are 4 bytes or larger.
+	b := [4000000]*string{x} // ERROR "moved to heap: b"
+	_ = b
+}
+
+// BAD: x need not leak.
+func doesNew1(x *string, y *string) { // ERROR "leaking param: x" "leaking param: y"
+	a := new([10]*string) // ERROR "new\(\[10\]\*string\) does not escape"
+	a[0] = x
+	b := new([65537]*string) // ERROR "new\(\[65537\]\*string\) escapes to heap"
+	b[0] = y
+}
+
+type a10 struct {
+	s *string
+	i [10]int32
+}
+
+type a65537 struct {
+	s *string
+	i [65537]int32
+}
+
+// BAD: x need not leak.
+func doesNew2(x *string, y *string) { // ERROR "leaking param: x" "leaking param: y"
+	a := new(a10) // ERROR "new\(a10\) does not escape"
+	a.s = x
+	b := new(a65537) // ERROR "new\(a65537\) escapes to heap"
+	b.s = y
+}
+
+// BAD: x need not leak.
+func doesMakeSlice(x *string, y *string) { // ERROR "leaking param: x" "leaking param: y"
+	a := make([]*string, 10) // ERROR "make\(\[\]\*string, 10\) does not escape"
+	a[0] = x
+	b := make([]*string, 65537) // ERROR "make\(\[\]\*string, 65537\) escapes to heap"
+	b[0] = y
+}