From 2b655c0b928128de154a130876cdff03d973dd90 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 19 Feb 2015 15:48:40 -0500 Subject: runtime: tidy GC driver Change-Id: I0da26e89ae73272e49e82c6549c774e5bc97f64c Reviewed-on: https://go-review.googlesource.com/5331 Reviewed-by: Austin Clements --- src/runtime/malloc.go | 2 +- src/runtime/mgc.go | 210 ++++++++++++++++++++++---------------------------- src/runtime/mheap.go | 6 +- src/runtime/proc.go | 2 +- 4 files changed, 96 insertions(+), 124 deletions(-) diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index b65bf70656..475f97fd05 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -655,7 +655,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { } if shouldtriggergc() { - gogc(0) + startGC(gcBackgroundMode) } else if shouldhelpgc && atomicloaduint(&bggc.working) == 1 { // bggc.lock not taken since race on bggc.working is benign. // At worse we don't call gchelpwork. diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 75d6b9158e..079856ed70 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -206,9 +206,7 @@ func shouldtriggergc() bool { return triggerratio*(int64(memstats.next_gc)-int64(memstats.heap_alloc)) <= int64(memstats.next_gc) && atomicloaduint(&bggc.working) == 0 } -var work workdata - -type workdata struct { +var work struct { full uint64 // lock-free list of full blocks workbuf empty uint64 // lock-free list of empty blocks workbuf partial uint64 // lock-free list of partially filled blocks workbuf @@ -226,19 +224,21 @@ type workdata struct { // GC runs a garbage collection. func GC() { - gogc(2) + startGC(gcForceBlockMode) } -// force = 0 - start concurrent GC -// force = 1 - do STW GC regardless of current heap usage -// force = 2 - go STW GC and eager sweep -func gogc(force int32) { +const ( + gcBackgroundMode = iota // concurrent GC + gcForceMode // stop-the-world GC now + gcForceBlockMode // stop-the-world GC now and wait for sweep +) + +func startGC(mode int) { // The gc is turned off (via enablegc) until the bootstrap has completed. // Also, malloc gets called in the guts of a number of libraries that might be // holding locks. To avoid deadlocks during stoptheworld, don't bother // trying to run gc while holding a lock. The next mallocgc without a lock // will do the gc instead. - mp := acquirem() if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 { releasem(mp) @@ -247,20 +247,23 @@ func gogc(force int32) { releasem(mp) mp = nil - if force == 0 { - lock(&bggc.lock) - if !bggc.started { - bggc.working = 1 - bggc.started = true - go backgroundgc() - } else if bggc.working == 0 { - bggc.working = 1 - ready(bggc.g) - } - unlock(&bggc.lock) - } else { - gcwork(force) + if mode != gcBackgroundMode { + // special synchronous cases + gc(mode) + return } + + // trigger concurrent GC + lock(&bggc.lock) + if !bggc.started { + bggc.working = 1 + bggc.started = true + go backgroundgc() + } else if bggc.working == 0 { + bggc.working = 1 + ready(bggc.g) + } + unlock(&bggc.lock) } // State of the background concurrent GC goroutine. @@ -276,15 +279,15 @@ var bggc struct { func backgroundgc() { bggc.g = getg() for { - gcwork(0) + gc(gcBackgroundMode) lock(&bggc.lock) bggc.working = 0 goparkunlock(&bggc.lock, "Concurrent GC wait", traceEvGoBlock) } } -func gcwork(force int32) { - +func gc(mode int) { + // Ok, we're doing it! Stop everybody else semacquire(&worldsema, false) // Pick up the remaining unswept/not being swept spans concurrently @@ -292,13 +295,11 @@ func gcwork(force int32) { sweep.nbgsweep++ } - // Ok, we're doing it! Stop everybody else - mp := acquirem() mp.preemptoff = "gcing" releasem(mp) gctimer.count++ - if force == 0 { + if mode == gcBackgroundMode { gctimer.cycle.sweepterm = nanotime() } @@ -307,31 +308,40 @@ func gcwork(force int32) { traceGCStart() } - // Pick up the remaining unswept/not being swept spans before we STW - for gosweepone() != ^uintptr(0) { - sweep.nbgsweep++ - } systemstack(stoptheworld) systemstack(finishsweep_m) // finish sweep before we start concurrent scan. - if force == 0 { // Do as much work concurrently as possible - gcphase = _GCscan - systemstack(starttheworld) - gctimer.cycle.scan = nanotime() - // Do a concurrent heap scan before we stop the world. - systemstack(gcscan_m) - gctimer.cycle.installmarkwb = nanotime() - systemstack(stoptheworld) - systemstack(gcinstallmarkwb) - systemstack(harvestwbufs) - systemstack(starttheworld) - gctimer.cycle.mark = nanotime() - systemstack(gcmark_m) - gctimer.cycle.markterm = nanotime() - systemstack(stoptheworld) - systemstack(gcinstalloffwb_m) + + if mode == gcBackgroundMode { // Do as much work concurrently as possible + systemstack(func() { + gcphase = _GCscan + + // Concurrent scan. + starttheworld() + gctimer.cycle.scan = nanotime() + gcscan_m() + gctimer.cycle.installmarkwb = nanotime() + + // Sync. + stoptheworld() + gcphase = _GCmark + harvestwbufs() + + // Concurrent mark. + starttheworld() + gctimer.cycle.mark = nanotime() + var gcw gcWork + gcDrain(&gcw) + gcw.dispose() + + // Begin mark termination. + gctimer.cycle.markterm = nanotime() + stoptheworld() + gcphase = _GCoff + }) } else { - // For non-concurrent GC (force != 0) g stack have not been scanned so - // set gcscanvalid such that mark termination scans all stacks. + // For non-concurrent GC (mode != gcBackgroundMode) + // g stack have not been scanned so set gcscanvalid + // such that mark termination scans all stacks. // No races here since we are in a STW phase. for _, gp := range allgs { gp.gcworkdone = false // set to true in gcphasework @@ -341,9 +351,10 @@ func gcwork(force int32) { startTime := nanotime() if mp != acquirem() { - throw("gogc: rescheduled") + throw("gcwork: rescheduled") } + // TODO(rsc): Should the concurrent GC clear pools earlier? clearpools() // Run gc on the g0 stack. We do this so that the g stack @@ -355,7 +366,6 @@ func gcwork(force int32) { if debug.gctrace > 1 { n = 2 } - eagersweep := force >= 2 for i := 0; i < n; i++ { if i > 0 { // refresh start time if doing a second GC @@ -363,12 +373,28 @@ func gcwork(force int32) { } // switch to g0, call gc, then switch back systemstack(func() { - gc_m(startTime, eagersweep) + gc_m(startTime, mode == gcForceBlockMode) }) } systemstack(func() { - gccheckmark_m(startTime, eagersweep) + // Called from malloc.go using systemstack. + // The world is stopped. Rerun the scan and mark phases + // using the bitMarkedCheck bit instead of the + // bitMarked bit. If the marking encounters an + // bitMarked bit that is not set then we throw. + //go:nowritebarrier + if debug.gccheckmark == 0 { + return + } + + if checkmarkphase { + throw("gccheckmark_m, entered with checkmarkphase already true") + } + + checkmarkphase = true + initCheckmarks() + gc_m(startTime, mode == gcForceBlockMode) // turns off checkmarkphase + calls clearcheckmarkbits }) if trace.enabled { @@ -379,13 +405,13 @@ func gcwork(force int32) { // all done mp.preemptoff = "" - if force == 0 { + if mode == gcBackgroundMode { gctimer.cycle.sweep = nanotime() } semrelease(&worldsema) - if force == 0 { + if mode == gcBackgroundMode { if gctimer.verbose > 1 { GCprinttimes() } else if gctimer.verbose > 0 { @@ -405,76 +431,23 @@ func gcwork(force int32) { } } -// For now this must be bracketed with a stoptheworld and a starttheworld to ensure -// all go routines see the new barrier. -//go:nowritebarrier -func gcinstalloffwb_m() { - gcphase = _GCoff -} - -// For now this must be bracketed with a stoptheworld and a starttheworld to ensure -// all go routines see the new barrier. -//go:nowritebarrier -func gcinstallmarkwb() { - gcphase = _GCmark -} - -// Mark all objects that are known about. -// This is the concurrent mark phase. -//go:nowritebarrier -func gcmark_m() { - var gcw gcWork - gcDrain(&gcw) - gcw.dispose() - // TODO add another harvestwbuf and reset work.nwait=0, work.ndone=0, and work.nproc=1 - // and repeat the above gcDrain. -} - -// Called from malloc.go using systemstack. -// The world is stopped. Rerun the scan and mark phases -// using the bitMarkedCheck bit instead of the -// bitMarked bit. If the marking encounters an -// bitMarked bit that is not set then we throw. -//go:nowritebarrier -func gccheckmark_m(startTime int64, eagersweep bool) { - if debug.gccheckmark == 0 { - return - } - - if checkmarkphase { - throw("gccheckmark_m, entered with checkmarkphase already true") +// STW is in effect at this point. +//TODO go:nowritebarrier +func gc_m(start_time int64, eagersweep bool) { + if _DebugGCPtrs { + print("GC start\n") } - checkmarkphase = true - initCheckmarks() - gc_m(startTime, eagersweep) // turns off checkmarkphase + calls clearcheckmarkbits -} - -// Called from malloc.go using systemstack, stopping and starting the world handled in caller. -//go:nowritebarrier -func gc_m(start_time int64, eagersweep bool) { _g_ := getg() gp := _g_.m.curg casgstatus(gp, _Grunning, _Gwaiting) gp.waitreason = "garbage collection" - gc(start_time, eagersweep) - casgstatus(gp, _Gwaiting, _Grunning) -} - -// STW is in effect at this point. -//TODO go:nowritebarrier -func gc(start_time int64, eagersweep bool) { - if _DebugGCPtrs { - print("GC start\n") - } - gcphase = _GCmarktermination if debug.allocfreetrace > 0 { tracegc() } - _g_ := getg() _g_.m.traceback = 2 t0 := start_time work.tstart = start_time @@ -619,6 +592,7 @@ func gc(start_time int64, eagersweep bool) { if debug.gccheckmark > 0 { if !checkmarkphase { // first half of two-pass; don't set up sweep + casgstatus(gp, _Gwaiting, _Grunning) return } checkmarkphase = false // done checking marks @@ -666,16 +640,12 @@ func gc(start_time int64, eagersweep bool) { if _DebugGCPtrs { print("GC end\n") } + + casgstatus(gp, _Gwaiting, _Grunning) } // Hooks for other packages -//go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory -func runtime_debug_freeOSMemory() { - gogc(2) // force GC and do eager sweep - systemstack(scavenge_m) -} - var poolcleanup func() //go:linkname sync_runtime_registerPoolCleanup sync.runtime_registerPoolCleanup diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index d082f8e622..ba800aacef 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -747,8 +747,10 @@ func mHeap_Scavenge(k int32, now, limit uint64) { } } -func scavenge_m() { - mHeap_Scavenge(-1, ^uint64(0), 0) +//go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory +func runtime_debug_freeOSMemory() { + startGC(gcForceBlockMode) + systemstack(func() { mHeap_Scavenge(-1, ^uint64(0), 0) }) } // Initialize a new span with the given start and npages. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 027416a9ec..d251c314d4 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -123,7 +123,7 @@ func forcegchelper() { if debug.gctrace > 0 { println("GC forced") } - gogc(1) + startGC(gcForceMode) } } -- cgit v1.2.3-54-g00ecf