runtime: replace reflectcall of defers with direct call

With GOEXPERIMENT=regabidefer, all deferred functions take no
arguments and have no results (their signature is always func()).
Since the signature is fixed, we can replace all of the reflectcalls
in the defer code with direct closure calls.

For #40724.

Change-Id: I3acd6742fe665610608a004c675f473b9d0e65ee
Reviewed-on: https://go-review.googlesource.com/c/go/+/306010
Trust: Austin Clements <austin@google.com>
Run-TryBot: Austin Clements <austin@google.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Than McIntosh <thanm@google.com>

1 files changed, 69 insertions, 14 deletions

diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index c265a5af79..bbf3ea473a 100644
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -382,6 +382,19 @@ func deferArgs(d *_defer) unsafe.Pointer {
 	return add(unsafe.Pointer(d), unsafe.Sizeof(*d))
 }
 
+// deferFunc returns d's deferred function. This is temporary while we
+// support both modes of GOEXPERIMENT=regabidefer. Once we commit to
+// that experiment, we should change the type of d.fn.
+//go:nosplit
+func deferFunc(d *_defer) func() {
+	if !experimentRegabiDefer {
+		throw("requires experimentRegabiDefer")
+	}
+	var fn func()
+	*(**funcval)(unsafe.Pointer(&fn)) = d.fn
+	return fn
+}
+
 var deferType *_type // type of _defer struct
 
 func init() {
@@ -635,10 +648,15 @@ func Goexit() {
 				addOneOpenDeferFrame(gp, 0, nil)
 			}
 		} else {
-
-			// Save the pc/sp in reflectcallSave(), so we can "recover" back to this
-			// loop if necessary.
-			reflectcallSave(&p, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz))
+			if experimentRegabiDefer {
+				// Save the pc/sp in deferCallSave(), so we can "recover" back to this
+				// loop if necessary.
+				deferCallSave(&p, deferFunc(d))
+			} else {
+				// Save the pc/sp in reflectcallSave(), so we can "recover" back to this
+				// loop if necessary.
+				reflectcallSave(&p, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz))
+			}
 		}
 		if p.aborted {
 			// We had a recursive panic in the defer d we started, and
@@ -860,7 +878,11 @@ func runOpenDeferFrame(gp *g, d *_defer) bool {
 		deferBits = deferBits &^ (1 << i)
 		*(*uint8)(unsafe.Pointer(d.varp - uintptr(deferBitsOffset))) = deferBits
 		p := d._panic
-		reflectcallSave(p, unsafe.Pointer(closure), deferArgs, argWidth)
+		if experimentRegabiDefer {
+			deferCallSave(p, deferFunc(d))
+		} else {
+			reflectcallSave(p, unsafe.Pointer(closure), deferArgs, argWidth)
+		}
 		if p != nil && p.aborted {
 			break
 		}
@@ -880,17 +902,20 @@ func runOpenDeferFrame(gp *g, d *_defer) bool {
 // panic record. This allows the runtime to return to the Goexit defer processing
 // loop, in the unusual case where the Goexit may be bypassed by a successful
 // recover.
+//
+// This is marked as a wrapper by the compiler so it doesn't appear in
+// tracebacks.
 func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) {
+	if experimentRegabiDefer {
+		throw("not allowed with experimentRegabiDefer")
+	}
 	if p != nil {
 		p.argp = unsafe.Pointer(getargp(0))
 		p.pc = getcallerpc()
 		p.sp = unsafe.Pointer(getcallersp())
 	}
-	// Pass a dummy RegArgs for now since no function actually implements
-	// the register-based ABI.
-	//
-	// TODO(mknyszek): Implement this properly, setting up arguments in
-	// registers as necessary in the caller.
+	// Pass a dummy RegArgs since we'll only take this path if
+	// we're not using the register ABI.
 	var regs abi.RegArgs
 	reflectcall(nil, fn, arg, argsize, argsize, argsize, &regs)
 	if p != nil {
@@ -899,6 +924,29 @@ func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) {
 	}
 }
 
+// deferCallSave calls fn() after saving the caller's pc and sp in the
+// panic record. This allows the runtime to return to the Goexit defer
+// processing loop, in the unusual case where the Goexit may be
+// bypassed by a successful recover.
+//
+// This is marked as a wrapper by the compiler so it doesn't appear in
+// tracebacks.
+func deferCallSave(p *_panic, fn func()) {
+	if !experimentRegabiDefer {
+		throw("only allowed with experimentRegabiDefer")
+	}
+	if p != nil {
+		p.argp = unsafe.Pointer(getargp(0))
+		p.pc = getcallerpc()
+		p.sp = unsafe.Pointer(getcallersp())
+	}
+	fn()
+	if p != nil {
+		p.pc = 0
+		p.sp = unsafe.Pointer(nil)
+	}
+}
+
 // The implementation of the predeclared function panic.
 func gopanic(e interface{}) {
 	gp := getg()
@@ -970,7 +1018,7 @@ func gopanic(e interface{}) {
 
 		// Mark defer as started, but keep on list, so that traceback
 		// can find and update the defer's argument frame if stack growth
-		// or a garbage collection happens before reflectcall starts executing d.fn.
+		// or a garbage collection happens before executing d.fn.
 		d.started = true
 
 		// Record the panic that is running the defer.
@@ -987,12 +1035,19 @@ func gopanic(e interface{}) {
 		} else {
 			p.argp = unsafe.Pointer(getargp(0))
 
-			var regs abi.RegArgs
-			reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz), uint32(d.siz), &regs)
+			if experimentRegabiDefer {
+				fn := deferFunc(d)
+				fn()
+			} else {
+				// Pass a dummy RegArgs since we'll only take this path if
+				// we're not using the register ABI.
+				var regs abi.RegArgs
+				reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz), uint32(d.siz), &regs)
+			}
 		}
 		p.argp = nil
 
-		// reflectcall did not panic. Remove d.
+		// Deferred function did not panic. Remove d.
 		if gp._defer != d {
 			throw("bad defer entry in panic")
 		}