cmd/compile: make HasHeapPointer recursive

Currently (*Type).HasHeapPointer only ignores pointers go:notinheap
types if the type itself is a pointer to a go:notinheap type. However,
if it's some other type that contains pointers where all of those
pointers are go:notinheap, it will conservatively return true. As a
result, we'll use write barriers where they aren't needed, for example
calling typedmemmove instead of just memmove on structs that contain
only go:notinheap pointers.

Fix this by making HasHeapPointer walk the whole type looking for
pointers that aren't marked go:notinheap.

Change-Id: Ib8c6abf6f7a20f34969d1d402c5498e0b990be59
Reviewed-on: https://go-review.googlesource.com/73412
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

1 files changed, 14 insertions, 7 deletions

diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go
index fbf0d88fb4..82829a1179 100644
--- a/src/cmd/compile/internal/types/type.go
+++ b/src/cmd/compile/internal/types/type.go
@@ -1364,7 +1364,14 @@ func (t *Type) IsUntyped() bool {
 	return false
 }
 
+// TODO(austin): We probably only need HasHeapPointer. See
+// golang.org/cl/73412 for discussion.
+
 func Haspointers(t *Type) bool {
+	return Haspointers1(t, false)
+}
+
+func Haspointers1(t *Type, ignoreNotInHeap bool) bool {
 	switch t.Etype {
 	case TINT, TUINT, TINT8, TUINT8, TINT16, TUINT16, TINT32, TUINT32, TINT64,
 		TUINT64, TUINTPTR, TFLOAT32, TFLOAT64, TCOMPLEX64, TCOMPLEX128, TBOOL:
@@ -1374,28 +1381,28 @@ func Haspointers(t *Type) bool {
 		if t.NumElem() == 0 { // empty array has no pointers
 			return false
 		}
-		return Haspointers(t.Elem())
+		return Haspointers1(t.Elem(), ignoreNotInHeap)
 
 	case TSTRUCT:
 		for _, t1 := range t.Fields().Slice() {
-			if Haspointers(t1.Type) {
+			if Haspointers1(t1.Type, ignoreNotInHeap) {
 				return true
 			}
 		}
 		return false
+
+	case TPTR32, TPTR64:
+		return !(ignoreNotInHeap && t.Elem().NotInHeap())
 	}
 
 	return true
 }
 
 // HasHeapPointer returns whether t contains a heap pointer.
-// This is used for write barrier insertion, so we ignore
+// This is used for write barrier insertion, so it ignores
 // pointers to go:notinheap types.
 func (t *Type) HasHeapPointer() bool {
-	if t.IsPtr() && t.Elem().NotInHeap() {
-		return false
-	}
-	return Haspointers(t)
+	return Haspointers1(t, true)
 }
 
 func (t *Type) Symbol() *obj.LSym {