diff options
30 files changed, 76 insertions, 52 deletions
diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index 4aba0ff64b..3ccce776ca 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -7,6 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "runtime/internal/atomic" "unsafe" ) @@ -184,7 +185,7 @@ func newosproc(mp *m) { } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Solaris because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 292ff94795..41352b3a5a 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -232,7 +233,7 @@ func newosproc(mp *m) { } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on AIX because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go index 9ed916705b..6f358d36d6 100644 --- a/src/runtime/os_js.go +++ b/src/runtime/os_js.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -35,7 +36,7 @@ func usleep_no_g(usec uint32) { usleep(usec) } -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) type mOS struct{} diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go index 99542fb2de..810d599508 100644 --- a/src/runtime/os_openbsd_syscall2.go +++ b/src/runtime/os_openbsd_syscall2.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -48,11 +49,11 @@ func open(name *byte, mode, perm int32) int32 // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) //go:noescape func obsdsigprocmask(how int32, new sigset) sigset diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 975d460a7d..2bb6ec0fce 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -458,7 +458,7 @@ func newosproc(mp *m) { } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Plan 9 because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index c76add7802..b49e9baec5 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -939,7 +939,7 @@ func newosproc0(mp *m, stk unsafe.Pointer) { throw("bad newosproc0") } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Windows because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index b997a467ba..cae15bc8e2 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1508,19 +1508,18 @@ func mexit(osStack bool) { } throw("m not found in allm") found: - if !osStack { - // Delay reaping m until it's done with the stack. - // - // If this is using an OS stack, the OS will free it - // so there's no need for reaping. - atomic.Store(&m.freeWait, 1) - // Put m on the free list, though it will not be reaped until - // freeWait is 0. Note that the free list must not be linked - // through alllink because some functions walk allm without - // locking, so may be using alllink. - m.freelink = sched.freem - sched.freem = m - } + // Delay reaping m until it's done with the stack. + // + // Put mp on the free list, though it will not be reaped while freeWait + // is freeMWait. mp is no longer reachable via allm, so even if it is + // on an OS stack, we must keep a reference to mp alive so that the GC + // doesn't free mp while we are still using it. + // + // Note that the free list must not be linked through alllink because + // some functions walk allm without locking, so may be using alllink. + m.freeWait.Store(freeMWait) + m.freelink = sched.freem + sched.freem = m unlock(&sched.lock) atomic.Xadd64(&ncgocall, int64(m.ncgocall)) @@ -1550,6 +1549,9 @@ found: mdestroy(m) if osStack { + // No more uses of mp, so it is safe to drop the reference. + m.freeWait.Store(freeMRef) + // Return from mstart and let the system thread // library free the g0 stack and terminate the thread. return @@ -1721,19 +1723,25 @@ func allocm(_p_ *p, fn func(), id int64) *m { lock(&sched.lock) var newList *m for freem := sched.freem; freem != nil; { - if freem.freeWait != 0 { + wait := freem.freeWait.Load() + if wait == freeMWait { next := freem.freelink freem.freelink = newList newList = freem freem = next continue } - // stackfree must be on the system stack, but allocm is - // reachable off the system stack transitively from - // startm. - systemstack(func() { - stackfree(freem.g0.stack) - }) + // Free the stack if needed. For freeMRef, there is + // nothing to do except drop freem from the sched.freem + // list. + if wait == freeMStack { + // stackfree must be on the system stack, but allocm is + // reachable off the system stack transitively from + // startm. + systemstack(func() { + stackfree(freem.g0.stack) + }) + } freem = freem.freelink } sched.freem = newList diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index b40045e4a5..9cd4777cf8 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -510,6 +510,13 @@ const ( tlsSize = tlsSlots * goarch.PtrSize ) +// Values for m.freeWait. +const ( + freeMStack = 0 // M done, free stack and reference. + freeMRef = 1 // M done, free reference. + freeMWait = 2 // M still in use. +) + type m struct { g0 *g // goroutine with scheduling stack morebuf gobuf // gobuf arg to morestack @@ -540,7 +547,7 @@ type m struct { newSigstack bool // minit on C thread called sigaltstack printlock int8 incgo bool // m is executing a cgo call - freeWait uint32 // if == 0, safe to free g0 and delete m (atomic) + freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) fastrand uint64 needextram bool traceback uint8 diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 9aa965454d..0d211e2db9 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "runtime/internal/atomic" + "unsafe" +) // read calls the read system call. // It returns a non-negative number of bytes written or a negative errno value. @@ -33,8 +36,8 @@ func open(name *byte, mode, perm int32) int32 // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 58b3a9171c..c90cc78968 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -473,7 +474,7 @@ func pthread_cond_signal(c *pthreadcond) int32 { func pthread_cond_signal_trampoline() // Not used on Darwin, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index d57bc2a7a4..3af0928828 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -65,7 +65,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index 97e6d9ab36..d4c4cc7fdb 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -63,7 +63,7 @@ GLOBL exitStack<>(SB),RODATA,$8 DATA exitStack<>+0x00(SB)/4, $0 DATA exitStack<>+0x04(SB)/4, $0 -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index 165e97c60d..57ae0399a5 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -60,7 +60,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s index b12e47c576..8546eba805 100644 --- a/src/runtime/sys_freebsd_arm.s +++ b/src/runtime/sys_freebsd_arm.s @@ -86,7 +86,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm64.s b/src/runtime/sys_freebsd_arm64.s index 1aa09e87ca..7dab3028de 100644 --- a/src/runtime/sys_freebsd_arm64.s +++ b/src/runtime/sys_freebsd_arm64.s @@ -98,7 +98,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 MOVD $0, R0 MOVD R0, (R0) -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 6df812234c..ddc5534352 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -78,7 +78,7 @@ TEXT exit1<>(SB),NOSPLIT,$0 INT $3 // not reached RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index f0e58e11db..5431dea4fc 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -60,7 +60,7 @@ TEXT runtime·exit(SB),NOSPLIT,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index ca443b699f..e41afdb066 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -131,7 +131,7 @@ TEXT exit1<>(SB),NOSPLIT|NOFRAME,$0 MOVW $1003, R1 MOVW R0, (R1) // fail hard -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 1276c077d7..47e1718252 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -59,7 +59,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SVC RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 0df2597993..315a6e4835 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -56,7 +56,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVV wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index 2207e9ab98..991b7ba60e 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -56,7 +56,7 @@ TEXT runtime·exit(SB),NOSPLIT,$0-4 UNDEF RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index dc3d89fae7..01d6c85597 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -55,7 +55,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL $SYS_exit_group RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index a3da46d136..8b1b26b03b 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -61,7 +61,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 ECALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOV wait+0(FP), A0 // We're done using the stack. diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 886add8b54..ca080b4225 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -52,7 +52,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 8a33894892..6c5386bcf1 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -53,7 +53,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-4 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index 02f5b4ba3b..c1cd95df14 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -122,7 +122,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s index 3a763b2a6a..2422b0282e 100644 --- a/src/runtime/sys_netbsd_arm.s +++ b/src/runtime/sys_netbsd_arm.s @@ -56,7 +56,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm64.s b/src/runtime/sys_netbsd_arm64.s index 8a0496e807..6d2c31631d 100644 --- a/src/runtime/sys_netbsd_arm64.s +++ b/src/runtime/sys_netbsd_arm64.s @@ -114,7 +114,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVD $0, R0 // If we're still running, MOVD R0, (R0) // crash -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go index 4d50b4f6b1..cba35cfee3 100644 --- a/src/runtime/sys_openbsd2.go +++ b/src/runtime/sys_openbsd2.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -250,7 +251,7 @@ func sigaltstack(new *stackt, old *stackt) { func sigaltstack_trampoline() // Not used on OpenBSD, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_openbsd_mips64.s b/src/runtime/sys_openbsd_mips64.s index f8ae8e7c30..bc392e4c54 100644 --- a/src/runtime/sys_openbsd_mips64.s +++ b/src/runtime/sys_openbsd_mips64.s @@ -24,7 +24,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVV R2, (R2) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0 MOVV wait+0(FP), R4 // arg 1 - notdead MOVV $302, R2 // sys___threxit |