diff options
Diffstat (limited to 'src/runtime/panic.go')
| -rw-r--r-- | src/runtime/panic.go | 439 |
1 files changed, 103 insertions, 336 deletions
diff --git a/src/runtime/panic.go b/src/runtime/panic.go index f6c38aafcc..e4bdceb32f 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -5,8 +5,7 @@ package runtime import ( - "internal/abi" - "internal/goexperiment" + "internal/goarch" "runtime/internal/atomic" "runtime/internal/sys" "unsafe" @@ -31,7 +30,7 @@ import ( // pc should be the program counter of the compiler-generated code that // triggered this panic. func panicCheck1(pc uintptr, msg string) { - if sys.GoarchWasm == 0 && hasPrefix(funcname(findfunc(pc)), "runtime.") { + if goarch.IsWasm == 0 && hasPrefix(funcname(findfunc(pc)), "runtime.") { // Note: wasm can't tail call, so we can't get the original caller's pc. throw(msg) } @@ -226,47 +225,27 @@ func panicmemAddr(addr uintptr) { panic(errorAddressString{msg: "invalid memory address or nil pointer dereference", addr: addr}) } -// Create a new deferred function fn with siz bytes of arguments. +// Create a new deferred function fn, which has no arguments and results. // The compiler turns a defer statement into a call to this. -//go:nosplit -func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn +func deferproc(fn func()) { gp := getg() if gp.m.curg != gp { // go code on the system stack can't defer throw("defer on system stack") } - if goexperiment.RegabiDefer && siz != 0 { - // TODO: Make deferproc just take a func(). - throw("defer with non-empty frame") - } - - // the arguments of fn are in a perilous state. The stack map - // for deferproc does not describe them. So we can't let garbage - // collection or stack copying trigger until we've copied them out - // to somewhere safe. The memmove below does that. - // Until the copy completes, we can only call nosplit routines. - sp := getcallersp() - argp := uintptr(unsafe.Pointer(&fn)) + unsafe.Sizeof(fn) - callerpc := getcallerpc() - - d := newdefer(siz) + d := newdefer() if d._panic != nil { throw("deferproc: d.panic != nil after newdefer") } d.link = gp._defer gp._defer = d d.fn = fn - d.pc = callerpc - d.sp = sp - switch siz { - case 0: - // Do nothing. - case sys.PtrSize: - *(*uintptr)(deferArgs(d)) = *(*uintptr)(unsafe.Pointer(argp)) - default: - memmove(deferArgs(d), unsafe.Pointer(argp), uintptr(siz)) - } + d.pc = getcallerpc() + // We must not be preempted between calling getcallersp and + // storing it to d.sp because getcallersp's result is a + // uintptr stack pointer. + d.sp = getcallersp() // deferproc returns 0 normally. // a deferred func that stops a panic @@ -280,12 +259,10 @@ func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn } // deferprocStack queues a new deferred function with a defer record on the stack. -// The defer record must have its siz and fn fields initialized. +// The defer record must have its fn field initialized. // All other fields can contain junk. -// The defer record must be immediately followed in memory by -// the arguments of the defer. -// Nosplit because the arguments on the stack won't be scanned -// until the defer record is spliced into the gp._defer list. +// Nosplit because of the uninitialized pointer fields on the stack. +// //go:nosplit func deferprocStack(d *_defer) { gp := getg() @@ -293,10 +270,7 @@ func deferprocStack(d *_defer) { // go code on the system stack can't defer throw("defer on system stack") } - if goexperiment.RegabiDefer && d.siz != 0 { - throw("defer with non-empty frame") - } - // siz and fn are already set. + // fn is already set. // The other fields are junk on entry to deferprocStack and // are initialized here. d.started = false @@ -327,132 +301,37 @@ func deferprocStack(d *_defer) { // been set and must not be clobbered. } -// Small malloc size classes >= 16 are the multiples of 16: 16, 32, 48, 64, 80, 96, 112, 128, 144, ... -// Each P holds a pool for defers with small arg sizes. -// Assign defer allocations to pools by rounding to 16, to match malloc size classes. - -const ( - deferHeaderSize = unsafe.Sizeof(_defer{}) - minDeferAlloc = (deferHeaderSize + 15) &^ 15 - minDeferArgs = minDeferAlloc - deferHeaderSize -) - -// defer size class for arg size sz -//go:nosplit -func deferclass(siz uintptr) uintptr { - if siz <= minDeferArgs { - return 0 - } - return (siz - minDeferArgs + 15) / 16 -} - -// total size of memory block for defer with arg size sz -func totaldefersize(siz uintptr) uintptr { - if siz <= minDeferArgs { - return minDeferAlloc - } - return deferHeaderSize + siz -} - -// Ensure that defer arg sizes that map to the same defer size class -// also map to the same malloc size class. -func testdefersizes() { - var m [len(p{}.deferpool)]int32 - - for i := range m { - m[i] = -1 - } - for i := uintptr(0); ; i++ { - defersc := deferclass(i) - if defersc >= uintptr(len(m)) { - break - } - siz := roundupsize(totaldefersize(i)) - if m[defersc] < 0 { - m[defersc] = int32(siz) - continue - } - if m[defersc] != int32(siz) { - print("bad defer size class: i=", i, " siz=", siz, " defersc=", defersc, "\n") - throw("bad defer size class") - } - } -} - -// The arguments associated with a deferred call are stored -// immediately after the _defer header in memory. -//go:nosplit -func deferArgs(d *_defer) unsafe.Pointer { - if d.siz == 0 { - // Avoid pointer past the defer allocation. - return nil - } - return add(unsafe.Pointer(d), unsafe.Sizeof(*d)) -} - -// deferFunc returns d's deferred function. This is temporary while we -// support both modes of GOEXPERIMENT=regabidefer. Once we commit to -// that experiment, we should change the type of d.fn. -//go:nosplit -func deferFunc(d *_defer) func() { - if !goexperiment.RegabiDefer { - throw("requires GOEXPERIMENT=regabidefer") - } - var fn func() - *(**funcval)(unsafe.Pointer(&fn)) = d.fn - return fn -} - -var deferType *_type // type of _defer struct - -func init() { - var x interface{} - x = (*_defer)(nil) - deferType = (*(**ptrtype)(unsafe.Pointer(&x))).elem -} +// Each P holds a pool for defers. // Allocate a Defer, usually using per-P pool. // Each defer must be released with freedefer. The defer is not // added to any defer chain yet. -// -// This must not grow the stack because there may be a frame without -// stack map information when this is called. -// -//go:nosplit -func newdefer(siz int32) *_defer { +func newdefer() *_defer { var d *_defer - sc := deferclass(uintptr(siz)) - gp := getg() - if sc < uintptr(len(p{}.deferpool)) { - pp := gp.m.p.ptr() - if len(pp.deferpool[sc]) == 0 && sched.deferpool[sc] != nil { - // Take the slow path on the system stack so - // we don't grow newdefer's stack. - systemstack(func() { - lock(&sched.deferlock) - for len(pp.deferpool[sc]) < cap(pp.deferpool[sc])/2 && sched.deferpool[sc] != nil { - d := sched.deferpool[sc] - sched.deferpool[sc] = d.link - d.link = nil - pp.deferpool[sc] = append(pp.deferpool[sc], d) - } - unlock(&sched.deferlock) - }) - } - if n := len(pp.deferpool[sc]); n > 0 { - d = pp.deferpool[sc][n-1] - pp.deferpool[sc][n-1] = nil - pp.deferpool[sc] = pp.deferpool[sc][:n-1] + mp := acquirem() + pp := mp.p.ptr() + if len(pp.deferpool) == 0 && sched.deferpool != nil { + lock(&sched.deferlock) + for len(pp.deferpool) < cap(pp.deferpool)/2 && sched.deferpool != nil { + d := sched.deferpool + sched.deferpool = d.link + d.link = nil + pp.deferpool = append(pp.deferpool, d) } + unlock(&sched.deferlock) + } + if n := len(pp.deferpool); n > 0 { + d = pp.deferpool[n-1] + pp.deferpool[n-1] = nil + pp.deferpool = pp.deferpool[:n-1] } + releasem(mp) + mp, pp = nil, nil + if d == nil { - // Allocate new defer+args. - systemstack(func() { - total := roundupsize(totaldefersize(uintptr(siz))) - d = (*_defer)(mallocgc(total, deferType, true)) - }) + // Allocate new defer. + d = new(_defer) } - d.siz = siz d.heap = true return d } @@ -460,11 +339,16 @@ func newdefer(siz int32) *_defer { // Free the given defer. // The defer cannot be used after this call. // -// This must not grow the stack because there may be a frame without a -// stack map when this is called. +// This is nosplit because the incoming defer is in a perilous state. +// It's not on any defer list, so stack copying won't adjust stack +// pointers in it (namely, d.link). Hence, if we were to copy the +// stack, d could then contain a stale pointer. // //go:nosplit func freedefer(d *_defer) { + d.link = nil + // After this point we can copy the stack. + if d._panic != nil { freedeferpanic() } @@ -474,53 +358,36 @@ func freedefer(d *_defer) { if !d.heap { return } - sc := deferclass(uintptr(d.siz)) - if sc >= uintptr(len(p{}.deferpool)) { - return - } - pp := getg().m.p.ptr() - if len(pp.deferpool[sc]) == cap(pp.deferpool[sc]) { + + mp := acquirem() + pp := mp.p.ptr() + if len(pp.deferpool) == cap(pp.deferpool) { // Transfer half of local cache to the central cache. - // - // Take this slow path on the system stack so - // we don't grow freedefer's stack. - systemstack(func() { - var first, last *_defer - for len(pp.deferpool[sc]) > cap(pp.deferpool[sc])/2 { - n := len(pp.deferpool[sc]) - d := pp.deferpool[sc][n-1] - pp.deferpool[sc][n-1] = nil - pp.deferpool[sc] = pp.deferpool[sc][:n-1] - if first == nil { - first = d - } else { - last.link = d - } - last = d + var first, last *_defer + for len(pp.deferpool) > cap(pp.deferpool)/2 { + n := len(pp.deferpool) + d := pp.deferpool[n-1] + pp.deferpool[n-1] = nil + pp.deferpool = pp.deferpool[:n-1] + if first == nil { + first = d + } else { + last.link = d } - lock(&sched.deferlock) - last.link = sched.deferpool[sc] - sched.deferpool[sc] = first - unlock(&sched.deferlock) - }) + last = d + } + lock(&sched.deferlock) + last.link = sched.deferpool + sched.deferpool = first + unlock(&sched.deferlock) } - // These lines used to be simply `*d = _defer{}` but that - // started causing a nosplit stack overflow via typedmemmove. - d.siz = 0 - d.started = false - d.openDefer = false - d.sp = 0 - d.pc = 0 - d.framepc = 0 - d.varp = 0 - d.fd = nil - // d._panic and d.fn must be nil already. - // If not, we would have called freedeferpanic or freedeferfn above, - // both of which throw. - d.link = nil + *d = _defer{} - pp.deferpool[sc] = append(pp.deferpool[sc], d) + pp.deferpool = append(pp.deferpool, d) + + releasem(mp) + mp, pp = nil, nil } // Separate function so that it can split stack. @@ -535,66 +402,39 @@ func freedeferfn() { throw("freedefer with d.fn != nil") } -// Run a deferred function if there is one. +// deferreturn runs deferred functions for the caller's frame. // The compiler inserts a call to this at the end of any // function which calls defer. -// If there is a deferred function, this will call runtime·jmpdefer, -// which will jump to the deferred function such that it appears -// to have been called by the caller of deferreturn at the point -// just before deferreturn was called. The effect is that deferreturn -// is called again and again until there are no more deferred functions. -// -// Declared as nosplit, because the function should not be preempted once we start -// modifying the caller's frame in order to reuse the frame to call the deferred -// function. -// -//go:nosplit func deferreturn() { gp := getg() - d := gp._defer - if d == nil { - return - } - sp := getcallersp() - if d.sp != sp { - return - } - if d.openDefer { - done := runOpenDeferFrame(gp, d) - if !done { - throw("unfinished open-coded defers in deferreturn") + for { + d := gp._defer + if d == nil { + return } + sp := getcallersp() + if d.sp != sp { + return + } + if d.openDefer { + done := runOpenDeferFrame(gp, d) + if !done { + throw("unfinished open-coded defers in deferreturn") + } + gp._defer = d.link + freedefer(d) + // If this frame uses open defers, then this + // must be the only defer record for the + // frame, so we can just return. + return + } + + fn := d.fn + d.fn = nil gp._defer = d.link freedefer(d) - return + fn() } - - // Moving arguments around. - // - // Everything called after this point must be recursively - // nosplit because the garbage collector won't know the form - // of the arguments until the jmpdefer can flip the PC over to - // fn. - argp := getcallersp() + sys.MinFrameSize - switch d.siz { - case 0: - // Do nothing. - case sys.PtrSize: - *(*uintptr)(unsafe.Pointer(argp)) = *(*uintptr)(deferArgs(d)) - default: - memmove(unsafe.Pointer(argp), deferArgs(d), uintptr(d.siz)) - } - fn := d.fn - d.fn = nil - gp._defer = d.link - freedefer(d) - // If the defer function pointer is nil, force the seg fault to happen - // here rather than in jmpdefer. gentraceback() throws an error if it is - // called with a callback on an LR architecture and jmpdefer is on the - // stack, because the stack trace can be incorrect in that case - see - // issue #8153). - _ = fn.fn - jmpdefer(fn, argp) } // Goexit terminates the goroutine that calls it. No other goroutine is affected. @@ -655,15 +495,9 @@ func Goexit() { addOneOpenDeferFrame(gp, 0, nil) } } else { - if goexperiment.RegabiDefer { - // Save the pc/sp in deferCallSave(), so we can "recover" back to this - // loop if necessary. - deferCallSave(&p, deferFunc(d)) - } else { - // Save the pc/sp in reflectcallSave(), so we can "recover" back to this - // loop if necessary. - reflectcallSave(&p, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz)) - } + // Save the pc/sp in deferCallSave(), so we can "recover" back to this + // loop if necessary. + deferCallSave(&p, d.fn) } if p.aborted { // We had a recursive panic in the defer d we started, and @@ -783,8 +617,7 @@ func addOneOpenDeferFrame(gp *g, pc uintptr, sp unsafe.Pointer) { throw("missing deferreturn") } - maxargsize, _ := readvarintUnsafe(fd) - d1 := newdefer(int32(maxargsize)) + d1 := newdefer() d1.openDefer = true d1._panic = nil // These are the pc/sp to set after we've @@ -845,57 +678,29 @@ func runOpenDeferFrame(gp *g, d *_defer) bool { done := true fd := d.fd - // Skip the maxargsize - _, fd = readvarintUnsafe(fd) deferBitsOffset, fd := readvarintUnsafe(fd) nDefers, fd := readvarintUnsafe(fd) deferBits := *(*uint8)(unsafe.Pointer(d.varp - uintptr(deferBitsOffset))) for i := int(nDefers) - 1; i >= 0; i-- { // read the funcdata info for this defer - var argWidth, closureOffset, nArgs uint32 - argWidth, fd = readvarintUnsafe(fd) + var closureOffset uint32 closureOffset, fd = readvarintUnsafe(fd) - nArgs, fd = readvarintUnsafe(fd) - if goexperiment.RegabiDefer && argWidth != 0 { - throw("defer with non-empty frame") - } if deferBits&(1<<i) == 0 { - for j := uint32(0); j < nArgs; j++ { - _, fd = readvarintUnsafe(fd) - _, fd = readvarintUnsafe(fd) - _, fd = readvarintUnsafe(fd) - } continue } - closure := *(**funcval)(unsafe.Pointer(d.varp - uintptr(closureOffset))) + closure := *(*func())(unsafe.Pointer(d.varp - uintptr(closureOffset))) d.fn = closure - deferArgs := deferArgs(d) - // If there is an interface receiver or method receiver, it is - // described/included as the first arg. - for j := uint32(0); j < nArgs; j++ { - var argOffset, argLen, argCallOffset uint32 - argOffset, fd = readvarintUnsafe(fd) - argLen, fd = readvarintUnsafe(fd) - argCallOffset, fd = readvarintUnsafe(fd) - memmove(unsafe.Pointer(uintptr(deferArgs)+uintptr(argCallOffset)), - unsafe.Pointer(d.varp-uintptr(argOffset)), - uintptr(argLen)) - } deferBits = deferBits &^ (1 << i) *(*uint8)(unsafe.Pointer(d.varp - uintptr(deferBitsOffset))) = deferBits p := d._panic - if goexperiment.RegabiDefer { - deferCallSave(p, deferFunc(d)) - } else { - reflectcallSave(p, unsafe.Pointer(closure), deferArgs, argWidth) - } + // Call the defer. Note that this can change d.varp if + // the stack moves. + deferCallSave(p, d.fn) if p != nil && p.aborted { break } d.fn = nil - // These args are just a copy, so can be cleared immediately - memclrNoHeapPointers(deferArgs, uintptr(argWidth)) if d._panic != nil && d._panic.recovered { done = deferBits == 0 break @@ -905,32 +710,6 @@ func runOpenDeferFrame(gp *g, d *_defer) bool { return done } -// reflectcallSave calls reflectcall after saving the caller's pc and sp in the -// panic record. This allows the runtime to return to the Goexit defer processing -// loop, in the unusual case where the Goexit may be bypassed by a successful -// recover. -// -// This is marked as a wrapper by the compiler so it doesn't appear in -// tracebacks. -func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) { - if goexperiment.RegabiDefer { - throw("not allowed with GOEXPERIMENT=regabidefer") - } - if p != nil { - p.argp = unsafe.Pointer(getargp()) - p.pc = getcallerpc() - p.sp = unsafe.Pointer(getcallersp()) - } - // Pass a dummy RegArgs since we'll only take this path if - // we're not using the register ABI. - var regs abi.RegArgs - reflectcall(nil, fn, arg, argsize, argsize, argsize, ®s) - if p != nil { - p.pc = 0 - p.sp = unsafe.Pointer(nil) - } -} - // deferCallSave calls fn() after saving the caller's pc and sp in the // panic record. This allows the runtime to return to the Goexit defer // processing loop, in the unusual case where the Goexit may be @@ -939,9 +718,6 @@ func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) { // This is marked as a wrapper by the compiler so it doesn't appear in // tracebacks. func deferCallSave(p *_panic, fn func()) { - if !goexperiment.RegabiDefer { - throw("only allowed with GOEXPERIMENT=regabidefer") - } if p != nil { p.argp = unsafe.Pointer(getargp()) p.pc = getcallerpc() @@ -1041,16 +817,7 @@ func gopanic(e interface{}) { } } else { p.argp = unsafe.Pointer(getargp()) - - if goexperiment.RegabiDefer { - fn := deferFunc(d) - fn() - } else { - // Pass a dummy RegArgs since we'll only take this path if - // we're not using the register ABI. - var regs abi.RegArgs - reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz), uint32(d.siz), ®s) - } + d.fn() } p.argp = nil |